Refactor/docs

.devcontainer/devcontainer.json

@@ -9,8 +9,10 @@
     "ghcr.io/devcontainers-contrib/features/uv:1": {}
   },
 
-  // Install all deps (including cli + dev extras) and set up git hooks
-  "postCreateCommand": "uv sync --all-extras && pre-commit install",
+  "postCreateCommand": {
+    "python": "uv sync --all-extras && pre-commit install",
+    "claude": "curl -fsSL https://claude.ai/install.sh | bash"
+  },
 
   // Local MCP dev server
   "forwardPorts": [8000],

.github/workflows/ci.yml

@@ -12,8 +12,8 @@ on:
       - develop
 
 jobs:
-  code-quality:
-    name: Code quality
+  lint:
+    name: Lint & format
     runs-on: ubuntu-latest
 
     steps:
@@ -37,9 +37,6 @@ jobs:
       - name: Ruff lint
         run: uv run ruff check core/ plugins/ server/ tests/
 
-      - name: Audit dependencies for known CVEs
-        run: uv run pip-audit -r requirements.txt
-
       - name: Set up Go 1.21
         uses: actions/setup-go@v5
         with:
@@ -55,8 +52,35 @@ jobs:
             exit 1
           fi
 
-  test-suite:
+  security:
+    name: Security audit
+    needs: lint
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Audit dependencies for known CVEs
+        run: uv run pip-audit -r requirements.txt
+
+  test:
     name: Test suite
+    needs: lint
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/infra.yml

@@ -14,58 +14,63 @@ on:
       - "terraform/**"
 
 jobs:
-  terraform-validate:
-    name: Terraform validate
+  terraform-lint:
+    name: Terraform lint & validate
     runs-on: ubuntu-latest
-    env:
-      TF_PLUGIN_CACHE_DIR: ${{ runner.temp }}/.terraform.d/plugin-cache
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Cache Terraform providers
-        uses: actions/cache@v4
-        with:
-          path: ${{ runner.temp }}/.terraform.d/plugin-cache
-          key: ${{ runner.os }}-terraform-${{ hashFiles('terraform/**/.terraform.lock.hcl') }}
-
-      - name: Cache Terraform init
-        uses: actions/cache@v4
-        with:
-          path: terraform/aws/.terraform
-          key: ${{ runner.os }}-tf-init-${{ hashFiles('terraform/**/.terraform.lock.hcl') }}
-
       - name: Set up Terraform
         uses: hashicorp/setup-terraform@v3
 
       - name: Check Terraform formatting
         working-directory: ./terraform
         run: |
           if ! terraform fmt -check -recursive; then
-            echo "Terraform files are not properly formatted."
+            echo "::error::Terraform files are not properly formatted."
             echo "Run 'terraform fmt -recursive' locally and commit the result."
             exit 1
           fi
 
-      - name: Terraform init (no backend)
-        working-directory: ./terraform/aws
-        run: terraform init -backend=false
-
-      - name: Create placeholder Lambda zip for validation
-        working-directory: ./terraform/aws
+      - name: Create placeholder artifacts for validation
         run: |
-          python3 -c "import zipfile; zipfile.ZipFile('lambda-deployment.zip', 'w').close()"
+          python3 -c "import zipfile; zipfile.ZipFile('terraform/aws/lambda-deployment.zip', 'w').close()"
 
-      - name: Terraform validate
-        working-directory: ./terraform/aws
-        run: terraform validate
+      - name: Validate all Terraform directories
+        run: |
+          failed=0
+          for dir in terraform/*/; do
+            if ls "$dir"*.tf 1>/dev/null 2>&1; then
+              echo "--- Validating $dir ---"
+              terraform -chdir="$dir" init -backend=false -input=false
+              if ! terraform -chdir="$dir" validate; then
+                echo "::error::Validation failed in $dir"
+                failed=1
+              fi
+            else
+              echo "--- Skipping $dir (no .tf files) ---"
+            fi
+          done
+          exit $failed
 
       - name: Set up TFLint
         uses: terraform-linters/setup-tflint@v4
 
-      - name: Run TFLint
-        run: tflint --chdir=terraform/aws
+      - name: Run TFLint on all directories
+        run: |
+          failed=0
+          for dir in terraform/*/; do
+            if ls "$dir"*.tf 1>/dev/null 2>&1; then
+              echo "--- TFLint: $dir ---"
+              if ! tflint --chdir="$dir"; then
+                echo "::warning::TFLint issues in $dir"
+                failed=1
+              fi
+            fi
+          done
+          exit $failed
 
       - name: Run tfsec
         uses: aquasecurity/tfsec-action@v1.0.3

.github/workflows/release.yml

@@ -4,8 +4,8 @@ name: Release
 on:
   push:
     tags:
-      - "v*.*.*" # Triggers on version tags like v1.0.0, v2.1.3, etc.
-  workflow_dispatch: # Allows manual triggering
+      - "v*.*.*"
+  workflow_dispatch:
     inputs:
       version:
         description: "Version tag (e.g., v1.0.0)"
@@ -17,11 +17,68 @@ env:
   GO_VERSION: "1.21"
 
 jobs:
+  validate:
+    name: Pre-release validation
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Ruff lint
+        run: uv run ruff check core/ plugins/ server/ tests/
+
+      - name: Audit dependencies for known CVEs
+        run: uv run pip-audit -r requirements.txt
+
+      - name: Run Python tests with coverage
+        run: |
+          uv run pytest tests/ \
+            -n auto \
+            --cov=core \
+            --cov=plugins \
+            --cov-report=term-missing \
+            --cov-fail-under=80
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Check Go formatting
+        working-directory: ./client
+        run: |
+          UNFORMATTED=$(gofmt -l .)
+          if [ -n "$UNFORMATTED" ]; then
+            echo "The following files need formatting:"
+            echo "$UNFORMATTED"
+            exit 1
+          fi
+
+      - name: Run Go tests
+        working-directory: ./client
+        run: go test ./...
+
   build:
     name: Build binaries
+    needs: validate
     runs-on: ubuntu-latest
     permissions:
-      contents: write # Required to create releases and upload assets
+      contents: write
 
     strategy:
       matrix:
@@ -46,7 +103,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0 # Fetch all history for proper versioning
+          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v5
@@ -84,6 +141,7 @@ jobs:
 
   build-lambda-zip:
     name: Build Lambda ZIP
+    needs: validate
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -172,7 +230,6 @@ jobs:
         run: |
           cd artifacts
           echo "=== Flattening artifacts ==="
-          # Find all release files (binaries + lambda zip) in subdirectories and move to root
           find . -mindepth 2 -type f \( -name "opencontext-client-*" -o -name "opencontext-lambda-*" \) | while read file; do
             filename=$(basename "$file")
             dirname=$(dirname "$file")
@@ -182,7 +239,6 @@ jobs:
             rmdir "$dirname" 2>/dev/null || true
             mv "$tempname" "$filename"
           done
-          # Remove any remaining empty subdirectories
           find . -mindepth 1 -type d -empty -delete
           echo "=== Flattening complete ==="
 

.gitignore

@@ -231,3 +231,4 @@ examples/
 
 # Terraform variable files (may contain real secrets — use *.tfvars.example as templates)
 terraform/**/*.tfvars
+.claude/

.vscode/tasks.json

@@ -38,7 +38,7 @@
     {
       "label": "Dev: start local server",
       "type": "shell",
-      "command": "python3 scripts/local_server.py",
+      "command": "opencontext serve",
       "presentation": {
         "reveal": "always",
         "panel": "dedicated"

CLAUDE.md

@@ -18,10 +18,10 @@ pre-commit install          # set up git hooks
 
 ```bash
 # Run local dev server (http://localhost:8000/mcp)
-python3 scripts/local_server.py
+opencontext serve
 
 # Test the running server
-./scripts/test_streamable_http.sh
+opencontext test --url http://localhost:8000/mcp
 
 # Tests
 pytest tests/ -n auto --cov=core --cov=plugins --cov-fail-under=80
@@ -31,10 +31,13 @@ ruff check core/ plugins/ server/ tests/ --fix --unsafe-fixes
 ruff format core/ plugins/ server/ tests/
 
 # CLI
-opencontext validate        # validate config.yaml
+opencontext validate --env staging    # validate config + Terraform before deploy
 opencontext deploy --env staging
-opencontext status
-opencontext logs
+opencontext status --env staging
+opencontext logs --env staging
+opencontext plugin list               # list enabled/disabled plugins
+opencontext security                  # pip-audit vulnerability scan
+opencontext architecture              # print AWS infra diagram in terminal
 ```
 
 ## Project Layout
@@ -45,7 +48,7 @@ plugins/        # Built-in plugins: ckan/, arcgis/, socrata/
 custom_plugins/ # Drop user plugins here — auto-discovered at startup
 cli/            # Typer CLI (opencontext command)
 server/         # HTTP adapters: local aiohttp + AWS Lambda entry point
-scripts/        # local_server.py, deploy.sh, test_streamable_http.sh
+server/adapters/ # local aiohttp dev server + AWS Lambda entry point
 tests/          # pytest suite (80% coverage required)
 terraform/aws/  # Lambda + API Gateway + IAM IaC
 examples/       # Per-city config.yaml examples (Boston, Chicago, Seattle, etc.)
@@ -58,7 +61,7 @@ Key files:
 - `core/plugin_manager.py` — discovery, loading, one-plugin enforcement
 - `core/validators.py` — config validation; enforces the one-plugin rule
 - `server/adapters/aws_lambda.py` — Lambda entry point
-- `scripts/local_server.py` — aiohttp dev server
+- `cli/commands/serve.py` — aiohttp dev server (started via `opencontext serve`)
 
 ## Plugin System
 
@@ -115,6 +118,6 @@ uv run pytest tests/ -n auto --cov=core --cov=plugins --cov-fail-under=80
 - **Tool prefix required** → call `ckan__search_datasets`, not `search_datasets`.
 - **`config.yaml` is gitignored** → changes to it won't be committed. Use `config-example.yaml` for template changes.
 - **Coverage < 80%** → CI fails. New code needs tests; check gaps with `--cov-report=html`.
-- **Lambda size limit** → 250 MB max. `scripts/deploy.sh` validates before packaging.
+- **Lambda size limit** → 250 MB max. `opencontext deploy` validates package size before uploading.
 - **Python 3.11+ required** → match this in any new tooling or containers.
 - **Go client** (`client/`) is an optional stdio-to-HTTP bridge for tools that only speak stdio MCP.

README.md

@@ -25,7 +25,7 @@ opencontext authenticate
 opencontext configure
 
 # 3. Test locally (optional)
-python3 scripts/local_server.py
+opencontext serve
 
 # 4. Deploy
 opencontext deploy --env staging
@@ -47,7 +47,10 @@ See [Getting Started](docs/GETTING_STARTED.md) for full setup.
 | Doc                                        | Description                                     |
 | ------------------------------------------ | ----------------------------------------------- |
 | [Getting Started](docs/GETTING_STARTED.md) | Setup and usage                                 |
+| [CLI Reference](docs/CLI.md)               | All CLI commands and flags                      |
 | [Architecture](docs/ARCHITECTURE.md)       | System design and plugins                       |
+| [Built-in Plugins](docs/BUILT_IN_PLUGINS.md) | CKAN, ArcGIS Hub, Socrata plugin details      |
+| [Custom Plugins](docs/CUSTOM_PLUGINS.md)   | How to write your own plugin                    |
 | [Deployment](docs/DEPLOYMENT.md)           | AWS, Terraform, monitoring                      |
 | [Testing](docs/TESTING.md)                 | Local testing (Terminal, Claude, MCP Inspector) |