diff --git a/.env.example b/.env.example index 5eacc242..0e0ce96f 100644 --- a/.env.example +++ b/.env.example @@ -34,3 +34,21 @@ PYTHONDONTWRITEBYTECODE=1 # Optional: Database selection # DATABASE_TYPE=falkordb # or falkordb-remote or neo4j + +# ── Plugin Configuration ─────────────────────────────────────────────────── +# Required when using docker-compose.plugins.yml or docker-compose.dev.yml + +# Your ingress domain (used by Traefik labels on plugin services) +# DOMAIN=localhost + +# OTEL Plugin — span receiver and processor +# OTEL_RECEIVER_PORT=5317 +# OTEL_FILTER_ROUTES=/health,/metrics,/ping,/favicon.ico + +# Xdebug Plugin — DBGp TCP listener (dev only) +# XDEBUG_LISTEN_HOST=0.0.0.0 +# XDEBUG_LISTEN_PORT=9003 +# XDEBUG_DEDUP_CACHE_SIZE=10000 + +# Log level for plugin containers (DEBUG, INFO, WARNING, ERROR) +# LOG_LEVEL=INFO diff --git a/.github/services.json b/.github/services.json new file mode 100644 index 00000000..9130a27e --- /dev/null +++ b/.github/services.json @@ -0,0 +1,34 @@ +[ + { + "name": "cgc-core", + "path": ".", + "dockerfile": "Dockerfile", + "image": "cgc-core", + "health_check": "version", + "description": "CodeGraphContext MCP server core" + }, + { + "name": "cgc-plugin-otel", + "path": "plugins/cgc-plugin-otel", + "dockerfile": "Dockerfile", + "image": "cgc-plugin-otel", + "health_check": "grpc_ping", + "description": "OpenTelemetry span receiver and graph writer" + }, + { + "name": "cgc-plugin-xdebug", + "path": "plugins/cgc-plugin-xdebug", + "dockerfile": "Dockerfile", + "image": "cgc-plugin-xdebug", + "health_check": "tcp_connect", + "description": "Xdebug DBGp call-stack listener" + }, + { + "name": "cgc-mcp", + "path": ".", + "dockerfile": "Dockerfile.mcp", + "image": "cgc-mcp", + "health_check": "http_get", + "description": "CGC hosted MCP server — HTTP transport with bundled plugins" + } +] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3481810c..6953f454 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,7 @@ jobs: name: Build on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] include: @@ -48,20 +49,17 @@ jobs: run: | pyinstaller cgc.spec --clean - - name: Build with PyInstaller (Linux - Manylinux) + - name: Install dependencies (Linux) if: runner.os == 'Linux' run: | - docker run --rm \ - -v ${{ github.workspace }}:/src \ - quay.io/pypa/manylinux2014_x86_64 \ - /bin/bash -c " - set -e - cd /src - /opt/python/cp312-cp312/bin/python -m pip install --upgrade pip - /opt/python/cp312-cp312/bin/pip install . pyinstaller - /opt/python/cp312-cp312/bin/pyinstaller cgc.spec --clean - " - sudo chown -R $USER:$USER dist build + python -m pip install --upgrade pip + pip install . + pip install pyinstaller + + - name: Build with PyInstaller (Linux) + if: runner.os == 'Linux' + run: | + pyinstaller cgc.spec --clean - name: Rename artifact (Linux/Mac) if: runner.os != 'Windows' diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 2907fbab..a07e2d4f 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -10,14 +10,26 @@ jobs: test: runs-on: ubuntu-latest + services: + falkordb: + image: falkordb/falkordb:latest + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.12' + cache: 'pip' - name: Install dependencies run: | @@ -26,6 +38,10 @@ jobs: pip install pytest - name: Run end-to-end tests + env: + FALKORDB_HOST: localhost + FALKORDB_PORT: 6379 + DATABASE_TYPE: falkordb-remote run: | chmod +x tests/run_tests.sh ./tests/run_tests.sh e2e diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 5ee4297c..f62dabe9 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -19,9 +19,10 @@ jobs: python-version: "3.12" - name: Install system deps + continue-on-error: true run: | brew update - brew install ripgrep || true + brew install ripgrep - name: Install CodeGraphContext run: | @@ -35,12 +36,14 @@ jobs: df -h - name: Run index (verbose) + continue-on-error: true run: | - cgc index -f --debug || true + cgc index -f --debug - name: Try find + continue-on-error: true run: | - cgc find content "def" --debug || true + cgc find content "def" --debug - name: Upload logs if: always() diff --git a/.github/workflows/plugin-publish.yml b/.github/workflows/plugin-publish.yml new file mode 100644 index 00000000..25be6c28 --- /dev/null +++ b/.github/workflows/plugin-publish.yml @@ -0,0 +1,123 @@ +name: Build and Publish Plugin Images + +on: + push: + tags: + - 'v*.*.*' + pull_request: + branches: [main] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ${{ github.repository_owner }} + +jobs: + # ── Read the service matrix from services.json ────────────────────────── + setup: + name: Load service matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + + - name: Load services.json into matrix + id: load + run: | + # Filter to plugin services only (skip cgc-core — handled by docker-publish.yml) + MATRIX=$(cat .github/services.json | jq -c '[.[] | select(.name != "cgc-core")]') + echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT" + + # ── Build, smoke-test, and optionally push each plugin image ──────────── + build-plugins: + name: Build ${{ matrix.name }} + needs: setup + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + include: ${{ fromJson(needs.setup.outputs.matrix) }} + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.image }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }} + type=ref,event=pr + type=sha,prefix=sha- + labels: | + org.opencontainers.image.title=${{ matrix.name }} + org.opencontainers.image.description=${{ matrix.description }} + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + + - name: Build image (load locally for smoke test) + id: build-local + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.path }} + file: ${{ matrix.path }}/${{ matrix.dockerfile }} + push: false + load: true + tags: smoke-test/${{ matrix.image }}:ci + cache-from: type=gha,scope=${{ matrix.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.name }} + + - name: Smoke test — gRPC import + if: matrix.health_check == 'grpc_ping' + run: docker run --rm smoke-test/${{ matrix.image }}:ci python -c "import grpc; print('gRPC OK')" + + - name: Smoke test — socket + if: matrix.health_check == 'tcp_connect' + run: docker run --rm smoke-test/${{ matrix.image }}:ci python -c "import socket; socket.socket(); print('socket OK')" + + - name: Push image to GHCR + if: github.event_name != 'pull_request' + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.path }} + file: ${{ matrix.path }}/${{ matrix.dockerfile }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.name }} + platforms: linux/amd64,linux/arm64 + + # ── Summary ────────────────────────────────────────────────────────────── + build-summary: + name: Plugin build summary + needs: build-plugins + runs-on: ubuntu-latest + if: always() + steps: + - name: Report overall status + run: | + if [ "${{ needs.build-plugins.result }}" = "success" ]; then + echo "✅ All plugin images built successfully." + else + echo "⚠️ One or more plugin images failed. Check individual job logs." + exit 1 + fi diff --git a/.github/workflows/test-plugins.yml b/.github/workflows/test-plugins.yml new file mode 100644 index 00000000..9db1348f --- /dev/null +++ b/.github/workflows/test-plugins.yml @@ -0,0 +1,84 @@ +name: Plugin Tests + +on: + pull_request: + branches: [main] + paths: + - 'plugins/**' + - 'src/codegraphcontext/plugin_registry.py' + - 'tests/unit/plugin/**' + - 'tests/integration/plugin/**' + push: + branches: [main] + paths: + - 'plugins/**' + - 'src/codegraphcontext/plugin_registry.py' + workflow_dispatch: + +jobs: + plugin-unit-tests: + name: Plugin unit + integration tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + + - name: Install core CGC (no extras) and dev dependencies + run: | + pip install --no-cache-dir packaging pytest pytest-mock + pip install --no-cache-dir -e ".[dev]" + + - name: Install stub plugin (editable) + run: pip install --no-cache-dir -e plugins/cgc-plugin-stub + + - name: Run plugin unit tests + env: + PYTHONPATH: src + run: pytest tests/unit/plugin/ -v --tb=short + + - name: Run plugin integration tests + env: + PYTHONPATH: src + run: pytest tests/integration/plugin/ -v --tb=short + + plugin-import-check: + name: Verify plugin packages import cleanly + runs-on: ubuntu-latest + strategy: + matrix: + plugin: [cgc-plugin-stub, cgc-plugin-otel, cgc-plugin-xdebug] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install plugin + run: | + pip install --no-cache-dir typer neo4j packaging || true + pip install --no-cache-dir -e plugins/${{ matrix.plugin }} || true + + - name: Check plugin PLUGIN_METADATA + env: + PYTHONPATH: src + run: | + PLUGIN_MOD=$(echo "${{ matrix.plugin }}" | tr '-' '_') + python -c " + import importlib + mod = importlib.import_module('${PLUGIN_MOD}') + meta = getattr(mod, 'PLUGIN_METADATA', None) + assert meta is not None, 'PLUGIN_METADATA missing' + for field in ('name', 'version', 'cgc_version_constraint', 'description'): + assert field in meta, f'PLUGIN_METADATA missing field: {field}' + print(f'✅ ${PLUGIN_MOD} PLUGIN_METADATA OK: {meta[\"name\"]} v{meta[\"version\"]}') + " diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4429a088..807342eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + cache: 'pip' - name: Install dependencies run: | diff --git a/.gitignore b/.gitignore index 129b46b3..7a8f7eec 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,6 @@ venv/ ./.env .venv311/ venv311/ -docker-compose.yml - # PyPI dist/ build/ @@ -31,8 +29,9 @@ coverage.xml htmlcov/ # MCP mcp.json -# Docker -docker-compose.yml +# Docker — root docker-compose.yml is generated per-environment, not tracked. +# Named compose files (plugin-stack, dev, template) and samples/ ARE tracked. +/docker-compose.yml # macOS system files .DS_Store src/.DS_Store diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md new file mode 100644 index 00000000..a4670ff4 --- /dev/null +++ b/.specify/memory/constitution.md @@ -0,0 +1,50 @@ +# [PROJECT_NAME] Constitution + + +## Core Principles + +### [PRINCIPLE_1_NAME] + +[PRINCIPLE_1_DESCRIPTION] + + +### [PRINCIPLE_2_NAME] + +[PRINCIPLE_2_DESCRIPTION] + + +### [PRINCIPLE_3_NAME] + +[PRINCIPLE_3_DESCRIPTION] + + +### [PRINCIPLE_4_NAME] + +[PRINCIPLE_4_DESCRIPTION] + + +### [PRINCIPLE_5_NAME] + +[PRINCIPLE_5_DESCRIPTION] + + +## [SECTION_2_NAME] + + +[SECTION_2_CONTENT] + + +## [SECTION_3_NAME] + + +[SECTION_3_CONTENT] + + +## Governance + + +[GOVERNANCE_RULES] + + +**Version**: [CONSTITUTION_VERSION] | **Ratified**: [RATIFICATION_DATE] | **Last Amended**: [LAST_AMENDED_DATE] + diff --git a/.specify/scripts/bash/check-prerequisites.sh b/.specify/scripts/bash/check-prerequisites.sh new file mode 100755 index 00000000..88a55594 --- /dev/null +++ b/.specify/scripts/bash/check-prerequisites.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +# Consolidated prerequisite checking script +# +# This script provides unified prerequisite checking for Spec-Driven Development workflow. +# It replaces the functionality previously spread across multiple scripts. +# +# Usage: ./check-prerequisites.sh [OPTIONS] +# +# OPTIONS: +# --json Output in JSON format +# --require-tasks Require tasks.md to exist (for implementation phase) +# --include-tasks Include tasks.md in AVAILABLE_DOCS list +# --paths-only Only output path variables (no validation) +# --help, -h Show help message +# +# OUTPUTS: +# JSON mode: {"FEATURE_DIR":"...", "AVAILABLE_DOCS":["..."]} +# Text mode: FEATURE_DIR:... \n AVAILABLE_DOCS: \n ✓/✗ file.md +# Paths only: REPO_ROOT: ... \n BRANCH: ... \n FEATURE_DIR: ... etc. + +set -e + +# Parse command line arguments +JSON_MODE=false +REQUIRE_TASKS=false +INCLUDE_TASKS=false +PATHS_ONLY=false + +for arg in "$@"; do + case "$arg" in + --json) + JSON_MODE=true + ;; + --require-tasks) + REQUIRE_TASKS=true + ;; + --include-tasks) + INCLUDE_TASKS=true + ;; + --paths-only) + PATHS_ONLY=true + ;; + --help|-h) + cat << 'EOF' +Usage: check-prerequisites.sh [OPTIONS] + +Consolidated prerequisite checking for Spec-Driven Development workflow. + +OPTIONS: + --json Output in JSON format + --require-tasks Require tasks.md to exist (for implementation phase) + --include-tasks Include tasks.md in AVAILABLE_DOCS list + --paths-only Only output path variables (no prerequisite validation) + --help, -h Show this help message + +EXAMPLES: + # Check task prerequisites (plan.md required) + ./check-prerequisites.sh --json + + # Check implementation prerequisites (plan.md + tasks.md required) + ./check-prerequisites.sh --json --require-tasks --include-tasks + + # Get feature paths only (no validation) + ./check-prerequisites.sh --paths-only + +EOF + exit 0 + ;; + *) + echo "ERROR: Unknown option '$arg'. Use --help for usage information." >&2 + exit 1 + ;; + esac +done + +# Source common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get feature paths and validate branch +_paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +eval "$_paths_output" +unset _paths_output +check_feature_branch "$CURRENT_BRANCH" "$HAS_GIT" || exit 1 + +# If paths-only mode, output paths and exit (support JSON + paths-only combined) +if $PATHS_ONLY; then + if $JSON_MODE; then + # Minimal JSON paths payload (no validation performed) + if has_jq; then + jq -cn \ + --arg repo_root "$REPO_ROOT" \ + --arg branch "$CURRENT_BRANCH" \ + --arg feature_dir "$FEATURE_DIR" \ + --arg feature_spec "$FEATURE_SPEC" \ + --arg impl_plan "$IMPL_PLAN" \ + --arg tasks "$TASKS" \ + '{REPO_ROOT:$repo_root,BRANCH:$branch,FEATURE_DIR:$feature_dir,FEATURE_SPEC:$feature_spec,IMPL_PLAN:$impl_plan,TASKS:$tasks}' + else + printf '{"REPO_ROOT":"%s","BRANCH":"%s","FEATURE_DIR":"%s","FEATURE_SPEC":"%s","IMPL_PLAN":"%s","TASKS":"%s"}\n' \ + "$(json_escape "$REPO_ROOT")" "$(json_escape "$CURRENT_BRANCH")" "$(json_escape "$FEATURE_DIR")" "$(json_escape "$FEATURE_SPEC")" "$(json_escape "$IMPL_PLAN")" "$(json_escape "$TASKS")" + fi + else + echo "REPO_ROOT: $REPO_ROOT" + echo "BRANCH: $CURRENT_BRANCH" + echo "FEATURE_DIR: $FEATURE_DIR" + echo "FEATURE_SPEC: $FEATURE_SPEC" + echo "IMPL_PLAN: $IMPL_PLAN" + echo "TASKS: $TASKS" + fi + exit 0 +fi + +# Validate required directories and files +if [[ ! -d "$FEATURE_DIR" ]]; then + echo "ERROR: Feature directory not found: $FEATURE_DIR" >&2 + echo "Run /speckit.specify first to create the feature structure." >&2 + exit 1 +fi + +if [[ ! -f "$IMPL_PLAN" ]]; then + echo "ERROR: plan.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit.plan first to create the implementation plan." >&2 + exit 1 +fi + +# Check for tasks.md if required +if $REQUIRE_TASKS && [[ ! -f "$TASKS" ]]; then + echo "ERROR: tasks.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit.tasks first to create the task list." >&2 + exit 1 +fi + +# Build list of available documents +docs=() + +# Always check these optional docs +[[ -f "$RESEARCH" ]] && docs+=("research.md") +[[ -f "$DATA_MODEL" ]] && docs+=("data-model.md") + +# Check contracts directory (only if it exists and has files) +if [[ -d "$CONTRACTS_DIR" ]] && [[ -n "$(ls -A "$CONTRACTS_DIR" 2>/dev/null)" ]]; then + docs+=("contracts/") +fi + +[[ -f "$QUICKSTART" ]] && docs+=("quickstart.md") + +# Include tasks.md if requested and it exists +if $INCLUDE_TASKS && [[ -f "$TASKS" ]]; then + docs+=("tasks.md") +fi + +# Output results +if $JSON_MODE; then + # Build JSON array of documents + if has_jq; then + if [[ ${#docs[@]} -eq 0 ]]; then + json_docs="[]" + else + json_docs=$(printf '%s\n' "${docs[@]}" | jq -R . | jq -s .) + fi + jq -cn \ + --arg feature_dir "$FEATURE_DIR" \ + --argjson docs "$json_docs" \ + '{FEATURE_DIR:$feature_dir,AVAILABLE_DOCS:$docs}' + else + if [[ ${#docs[@]} -eq 0 ]]; then + json_docs="[]" + else + json_docs=$(for d in "${docs[@]}"; do printf '"%s",' "$(json_escape "$d")"; done) + json_docs="[${json_docs%,}]" + fi + printf '{"FEATURE_DIR":"%s","AVAILABLE_DOCS":%s}\n' "$(json_escape "$FEATURE_DIR")" "$json_docs" + fi +else + # Text output + echo "FEATURE_DIR:$FEATURE_DIR" + echo "AVAILABLE_DOCS:" + + # Show status of each potential document + check_file "$RESEARCH" "research.md" + check_file "$DATA_MODEL" "data-model.md" + check_dir "$CONTRACTS_DIR" "contracts/" + check_file "$QUICKSTART" "quickstart.md" + + if $INCLUDE_TASKS; then + check_file "$TASKS" "tasks.md" + fi +fi diff --git a/.specify/scripts/bash/common.sh b/.specify/scripts/bash/common.sh new file mode 100755 index 00000000..40f1c96e --- /dev/null +++ b/.specify/scripts/bash/common.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# Common functions and variables for all scripts + +# Get repository root, with fallback for non-git repositories +get_repo_root() { + if git rev-parse --show-toplevel >/dev/null 2>&1; then + git rev-parse --show-toplevel + else + # Fall back to script location for non-git repos + local script_dir="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + (cd "$script_dir/../../.." && pwd) + fi +} + +# Get current branch, with fallback for non-git repositories +get_current_branch() { + # First check if SPECIFY_FEATURE environment variable is set + if [[ -n "${SPECIFY_FEATURE:-}" ]]; then + echo "$SPECIFY_FEATURE" + return + fi + + # Then check git if available + if git rev-parse --abbrev-ref HEAD >/dev/null 2>&1; then + git rev-parse --abbrev-ref HEAD + return + fi + + # For non-git repos, try to find the latest feature directory + local repo_root=$(get_repo_root) + local specs_dir="$repo_root/specs" + + if [[ -d "$specs_dir" ]]; then + local latest_feature="" + local highest=0 + + for dir in "$specs_dir"/*; do + if [[ -d "$dir" ]]; then + local dirname=$(basename "$dir") + if [[ "$dirname" =~ ^([0-9]{3})- ]]; then + local number=${BASH_REMATCH[1]} + number=$((10#$number)) + if [[ "$number" -gt "$highest" ]]; then + highest=$number + latest_feature=$dirname + fi + fi + fi + done + + if [[ -n "$latest_feature" ]]; then + echo "$latest_feature" + return + fi + fi + + echo "main" # Final fallback +} + +# Check if we have git available +has_git() { + git rev-parse --show-toplevel >/dev/null 2>&1 +} + +check_feature_branch() { + local branch="$1" + local has_git_repo="$2" + + # For non-git repos, we can't enforce branch naming but still provide output + if [[ "$has_git_repo" != "true" ]]; then + echo "[specify] Warning: Git repository not detected; skipped branch validation" >&2 + return 0 + fi + + if [[ ! "$branch" =~ ^[0-9]{3}- ]]; then + echo "ERROR: Not on a feature branch. Current branch: $branch" >&2 + echo "Feature branches should be named like: 001-feature-name" >&2 + return 1 + fi + + return 0 +} + +get_feature_dir() { echo "$1/specs/$2"; } + +# Find feature directory by numeric prefix instead of exact branch match +# This allows multiple branches to work on the same spec (e.g., 004-fix-bug, 004-add-feature) +find_feature_dir_by_prefix() { + local repo_root="$1" + local branch_name="$2" + local specs_dir="$repo_root/specs" + + # Extract numeric prefix from branch (e.g., "004" from "004-whatever") + if [[ ! "$branch_name" =~ ^([0-9]{3})- ]]; then + # If branch doesn't have numeric prefix, fall back to exact match + echo "$specs_dir/$branch_name" + return + fi + + local prefix="${BASH_REMATCH[1]}" + + # Search for directories in specs/ that start with this prefix + local matches=() + if [[ -d "$specs_dir" ]]; then + for dir in "$specs_dir"/"$prefix"-*; do + if [[ -d "$dir" ]]; then + matches+=("$(basename "$dir")") + fi + done + fi + + # Handle results + if [[ ${#matches[@]} -eq 0 ]]; then + # No match found - return the branch name path (will fail later with clear error) + echo "$specs_dir/$branch_name" + elif [[ ${#matches[@]} -eq 1 ]]; then + # Exactly one match - perfect! + echo "$specs_dir/${matches[0]}" + else + # Multiple matches - this shouldn't happen with proper naming convention + echo "ERROR: Multiple spec directories found with prefix '$prefix': ${matches[*]}" >&2 + echo "Please ensure only one spec directory exists per numeric prefix." >&2 + return 1 + fi +} + +get_feature_paths() { + local repo_root=$(get_repo_root) + local current_branch=$(get_current_branch) + local has_git_repo="false" + + if has_git; then + has_git_repo="true" + fi + + # Use prefix-based lookup to support multiple branches per spec + local feature_dir + if ! feature_dir=$(find_feature_dir_by_prefix "$repo_root" "$current_branch"); then + echo "ERROR: Failed to resolve feature directory" >&2 + return 1 + fi + + # Use printf '%q' to safely quote values, preventing shell injection + # via crafted branch names or paths containing special characters + printf 'REPO_ROOT=%q\n' "$repo_root" + printf 'CURRENT_BRANCH=%q\n' "$current_branch" + printf 'HAS_GIT=%q\n' "$has_git_repo" + printf 'FEATURE_DIR=%q\n' "$feature_dir" + printf 'FEATURE_SPEC=%q\n' "$feature_dir/spec.md" + printf 'IMPL_PLAN=%q\n' "$feature_dir/plan.md" + printf 'TASKS=%q\n' "$feature_dir/tasks.md" + printf 'RESEARCH=%q\n' "$feature_dir/research.md" + printf 'DATA_MODEL=%q\n' "$feature_dir/data-model.md" + printf 'QUICKSTART=%q\n' "$feature_dir/quickstart.md" + printf 'CONTRACTS_DIR=%q\n' "$feature_dir/contracts" +} + +# Check if jq is available for safe JSON construction +has_jq() { + command -v jq >/dev/null 2>&1 +} + +# Escape a string for safe embedding in a JSON value (fallback when jq is unavailable). +# Handles backslash, double-quote, and JSON-required control character escapes (RFC 8259). +json_escape() { + local s="$1" + s="${s//\\/\\\\}" + s="${s//\"/\\\"}" + s="${s//$'\n'/\\n}" + s="${s//$'\t'/\\t}" + s="${s//$'\r'/\\r}" + s="${s//$'\b'/\\b}" + s="${s//$'\f'/\\f}" + # Escape any remaining U+0001-U+001F control characters as \uXXXX. + # (U+0000/NUL cannot appear in bash strings and is excluded.) + # LC_ALL=C ensures ${#s} counts bytes and ${s:$i:1} yields single bytes, + # so multi-byte UTF-8 sequences (first byte >= 0xC0) pass through intact. + local LC_ALL=C + local i char code + for (( i=0; i<${#s}; i++ )); do + char="${s:$i:1}" + printf -v code '%d' "'$char" 2>/dev/null || code=256 + if (( code >= 1 && code <= 31 )); then + printf '\\u%04x' "$code" + else + printf '%s' "$char" + fi + done +} + +check_file() { [[ -f "$1" ]] && echo " ✓ $2" || echo " ✗ $2"; } +check_dir() { [[ -d "$1" && -n $(ls -A "$1" 2>/dev/null) ]] && echo " ✓ $2" || echo " ✗ $2"; } + +# Resolve a template name to a file path using the priority stack: +# 1. .specify/templates/overrides/ +# 2. .specify/presets//templates/ (sorted by priority from .registry) +# 3. .specify/extensions//templates/ +# 4. .specify/templates/ (core) +resolve_template() { + local template_name="$1" + local repo_root="$2" + local base="$repo_root/.specify/templates" + + # Priority 1: Project overrides + local override="$base/overrides/${template_name}.md" + [ -f "$override" ] && echo "$override" && return 0 + + # Priority 2: Installed presets (sorted by priority from .registry) + local presets_dir="$repo_root/.specify/presets" + if [ -d "$presets_dir" ]; then + local registry_file="$presets_dir/.registry" + if [ -f "$registry_file" ] && command -v python3 >/dev/null 2>&1; then + # Read preset IDs sorted by priority (lower number = higher precedence). + # The python3 call is wrapped in an if-condition so that set -e does not + # abort the function when python3 exits non-zero (e.g. invalid JSON). + local sorted_presets="" + if sorted_presets=$(SPECKIT_REGISTRY="$registry_file" python3 -c " +import json, sys, os +try: + with open(os.environ['SPECKIT_REGISTRY']) as f: + data = json.load(f) + presets = data.get('presets', {}) + for pid, meta in sorted(presets.items(), key=lambda x: x[1].get('priority', 10)): + print(pid) +except Exception: + sys.exit(1) +" 2>/dev/null); then + if [ -n "$sorted_presets" ]; then + # python3 succeeded and returned preset IDs — search in priority order + while IFS= read -r preset_id; do + local candidate="$presets_dir/$preset_id/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done <<< "$sorted_presets" + fi + # python3 succeeded but registry has no presets — nothing to search + else + # python3 failed (missing, or registry parse error) — fall back to unordered directory scan + for preset in "$presets_dir"/*/; do + [ -d "$preset" ] || continue + local candidate="$preset/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done + fi + else + # Fallback: alphabetical directory order (no python3 available) + for preset in "$presets_dir"/*/; do + [ -d "$preset" ] || continue + local candidate="$preset/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done + fi + fi + + # Priority 3: Extension-provided templates + local ext_dir="$repo_root/.specify/extensions" + if [ -d "$ext_dir" ]; then + for ext in "$ext_dir"/*/; do + [ -d "$ext" ] || continue + # Skip hidden directories (e.g. .backup, .cache) + case "$(basename "$ext")" in .*) continue;; esac + local candidate="$ext/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done + fi + + # Priority 4: Core templates + local core="$base/${template_name}.md" + [ -f "$core" ] && echo "$core" && return 0 + + # Template not found in any location. + # Return 1 so callers can distinguish "not found" from "found". + # Callers running under set -e should use: TEMPLATE=$(resolve_template ...) || true + return 1 +} + diff --git a/.specify/scripts/bash/create-new-feature.sh b/.specify/scripts/bash/create-new-feature.sh new file mode 100755 index 00000000..58c5c86c --- /dev/null +++ b/.specify/scripts/bash/create-new-feature.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash + +set -e + +JSON_MODE=false +SHORT_NAME="" +BRANCH_NUMBER="" +ARGS=() +i=1 +while [ $i -le $# ]; do + arg="${!i}" + case "$arg" in + --json) + JSON_MODE=true + ;; + --short-name) + if [ $((i + 1)) -gt $# ]; then + echo 'Error: --short-name requires a value' >&2 + exit 1 + fi + i=$((i + 1)) + next_arg="${!i}" + # Check if the next argument is another option (starts with --) + if [[ "$next_arg" == --* ]]; then + echo 'Error: --short-name requires a value' >&2 + exit 1 + fi + SHORT_NAME="$next_arg" + ;; + --number) + if [ $((i + 1)) -gt $# ]; then + echo 'Error: --number requires a value' >&2 + exit 1 + fi + i=$((i + 1)) + next_arg="${!i}" + if [[ "$next_arg" == --* ]]; then + echo 'Error: --number requires a value' >&2 + exit 1 + fi + BRANCH_NUMBER="$next_arg" + ;; + --help|-h) + echo "Usage: $0 [--json] [--short-name ] [--number N] " + echo "" + echo "Options:" + echo " --json Output in JSON format" + echo " --short-name Provide a custom short name (2-4 words) for the branch" + echo " --number N Specify branch number manually (overrides auto-detection)" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 'Add user authentication system' --short-name 'user-auth'" + echo " $0 'Implement OAuth2 integration for API' --number 5" + exit 0 + ;; + *) + ARGS+=("$arg") + ;; + esac + i=$((i + 1)) +done + +FEATURE_DESCRIPTION="${ARGS[*]}" +if [ -z "$FEATURE_DESCRIPTION" ]; then + echo "Usage: $0 [--json] [--short-name ] [--number N] " >&2 + exit 1 +fi + +# Trim whitespace and validate description is not empty (e.g., user passed only whitespace) +FEATURE_DESCRIPTION=$(echo "$FEATURE_DESCRIPTION" | xargs) +if [ -z "$FEATURE_DESCRIPTION" ]; then + echo "Error: Feature description cannot be empty or contain only whitespace" >&2 + exit 1 +fi + +# Function to find the repository root by searching for existing project markers +find_repo_root() { + local dir="$1" + while [ "$dir" != "/" ]; do + if [ -d "$dir/.git" ] || [ -d "$dir/.specify" ]; then + echo "$dir" + return 0 + fi + dir="$(dirname "$dir")" + done + return 1 +} + +# Function to get highest number from specs directory +get_highest_from_specs() { + local specs_dir="$1" + local highest=0 + + if [ -d "$specs_dir" ]; then + for dir in "$specs_dir"/*; do + [ -d "$dir" ] || continue + dirname=$(basename "$dir") + number=$(echo "$dirname" | grep -o '^[0-9]\+' || echo "0") + number=$((10#$number)) + if [ "$number" -gt "$highest" ]; then + highest=$number + fi + done + fi + + echo "$highest" +} + +# Function to get highest number from git branches +get_highest_from_branches() { + local highest=0 + + # Get all branches (local and remote) + branches=$(git branch -a 2>/dev/null || echo "") + + if [ -n "$branches" ]; then + while IFS= read -r branch; do + # Clean branch name: remove leading markers and remote prefixes + clean_branch=$(echo "$branch" | sed 's/^[* ]*//; s|^remotes/[^/]*/||') + + # Extract feature number if branch matches pattern ###-* + if echo "$clean_branch" | grep -q '^[0-9]\{3\}-'; then + number=$(echo "$clean_branch" | grep -o '^[0-9]\{3\}' || echo "0") + number=$((10#$number)) + if [ "$number" -gt "$highest" ]; then + highest=$number + fi + fi + done <<< "$branches" + fi + + echo "$highest" +} + +# Function to check existing branches (local and remote) and return next available number +check_existing_branches() { + local specs_dir="$1" + + # Fetch all remotes to get latest branch info (suppress errors if no remotes) + git fetch --all --prune >/dev/null 2>&1 || true + + # Get highest number from ALL branches (not just matching short name) + local highest_branch=$(get_highest_from_branches) + + # Get highest number from ALL specs (not just matching short name) + local highest_spec=$(get_highest_from_specs "$specs_dir") + + # Take the maximum of both + local max_num=$highest_branch + if [ "$highest_spec" -gt "$max_num" ]; then + max_num=$highest_spec + fi + + # Return next number + echo $((max_num + 1)) +} + +# Function to clean and format a branch name +clean_branch_name() { + local name="$1" + echo "$name" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/-\+/-/g' | sed 's/^-//' | sed 's/-$//' +} + +# Resolve repository root. Prefer git information when available, but fall back +# to searching for repository markers so the workflow still functions in repositories that +# were initialised with --no-git. +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +if git rev-parse --show-toplevel >/dev/null 2>&1; then + REPO_ROOT=$(git rev-parse --show-toplevel) + HAS_GIT=true +else + REPO_ROOT="$(find_repo_root "$SCRIPT_DIR")" + if [ -z "$REPO_ROOT" ]; then + echo "Error: Could not determine repository root. Please run this script from within the repository." >&2 + exit 1 + fi + HAS_GIT=false +fi + +cd "$REPO_ROOT" + +SPECS_DIR="$REPO_ROOT/specs" +mkdir -p "$SPECS_DIR" + +# Function to generate branch name with stop word filtering and length filtering +generate_branch_name() { + local description="$1" + + # Common stop words to filter out + local stop_words="^(i|a|an|the|to|for|of|in|on|at|by|with|from|is|are|was|were|be|been|being|have|has|had|do|does|did|will|would|should|could|can|may|might|must|shall|this|that|these|those|my|your|our|their|want|need|add|get|set)$" + + # Convert to lowercase and split into words + local clean_name=$(echo "$description" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/ /g') + + # Filter words: remove stop words and words shorter than 3 chars (unless they're uppercase acronyms in original) + local meaningful_words=() + for word in $clean_name; do + # Skip empty words + [ -z "$word" ] && continue + + # Keep words that are NOT stop words AND (length >= 3 OR are potential acronyms) + if ! echo "$word" | grep -qiE "$stop_words"; then + if [ ${#word} -ge 3 ]; then + meaningful_words+=("$word") + elif echo "$description" | grep -q "\b${word^^}\b"; then + # Keep short words if they appear as uppercase in original (likely acronyms) + meaningful_words+=("$word") + fi + fi + done + + # If we have meaningful words, use first 3-4 of them + if [ ${#meaningful_words[@]} -gt 0 ]; then + local max_words=3 + if [ ${#meaningful_words[@]} -eq 4 ]; then max_words=4; fi + + local result="" + local count=0 + for word in "${meaningful_words[@]}"; do + if [ $count -ge $max_words ]; then break; fi + if [ -n "$result" ]; then result="$result-"; fi + result="$result$word" + count=$((count + 1)) + done + echo "$result" + else + # Fallback to original logic if no meaningful words found + local cleaned=$(clean_branch_name "$description") + echo "$cleaned" | tr '-' '\n' | grep -v '^$' | head -3 | tr '\n' '-' | sed 's/-$//' + fi +} + +# Generate branch name +if [ -n "$SHORT_NAME" ]; then + # Use provided short name, just clean it up + BRANCH_SUFFIX=$(clean_branch_name "$SHORT_NAME") +else + # Generate from description with smart filtering + BRANCH_SUFFIX=$(generate_branch_name "$FEATURE_DESCRIPTION") +fi + +# Determine branch number +if [ -z "$BRANCH_NUMBER" ]; then + if [ "$HAS_GIT" = true ]; then + # Check existing branches on remotes + BRANCH_NUMBER=$(check_existing_branches "$SPECS_DIR") + else + # Fall back to local directory check + HIGHEST=$(get_highest_from_specs "$SPECS_DIR") + BRANCH_NUMBER=$((HIGHEST + 1)) + fi +fi + +# Force base-10 interpretation to prevent octal conversion (e.g., 010 → 8 in octal, but should be 10 in decimal) +FEATURE_NUM=$(printf "%03d" "$((10#$BRANCH_NUMBER))") +BRANCH_NAME="${FEATURE_NUM}-${BRANCH_SUFFIX}" + +# GitHub enforces a 244-byte limit on branch names +# Validate and truncate if necessary +MAX_BRANCH_LENGTH=244 +if [ ${#BRANCH_NAME} -gt $MAX_BRANCH_LENGTH ]; then + # Calculate how much we need to trim from suffix + # Account for: feature number (3) + hyphen (1) = 4 chars + MAX_SUFFIX_LENGTH=$((MAX_BRANCH_LENGTH - 4)) + + # Truncate suffix at word boundary if possible + TRUNCATED_SUFFIX=$(echo "$BRANCH_SUFFIX" | cut -c1-$MAX_SUFFIX_LENGTH) + # Remove trailing hyphen if truncation created one + TRUNCATED_SUFFIX=$(echo "$TRUNCATED_SUFFIX" | sed 's/-$//') + + ORIGINAL_BRANCH_NAME="$BRANCH_NAME" + BRANCH_NAME="${FEATURE_NUM}-${TRUNCATED_SUFFIX}" + + >&2 echo "[specify] Warning: Branch name exceeded GitHub's 244-byte limit" + >&2 echo "[specify] Original: $ORIGINAL_BRANCH_NAME (${#ORIGINAL_BRANCH_NAME} bytes)" + >&2 echo "[specify] Truncated to: $BRANCH_NAME (${#BRANCH_NAME} bytes)" +fi + +if [ "$HAS_GIT" = true ]; then + if ! git checkout -b "$BRANCH_NAME" 2>/dev/null; then + # Check if branch already exists + if git branch --list "$BRANCH_NAME" | grep -q .; then + >&2 echo "Error: Branch '$BRANCH_NAME' already exists. Please use a different feature name or specify a different number with --number." + exit 1 + else + >&2 echo "Error: Failed to create git branch '$BRANCH_NAME'. Please check your git configuration and try again." + exit 1 + fi + fi +else + >&2 echo "[specify] Warning: Git repository not detected; skipped branch creation for $BRANCH_NAME" +fi + +FEATURE_DIR="$SPECS_DIR/$BRANCH_NAME" +mkdir -p "$FEATURE_DIR" + +TEMPLATE=$(resolve_template "spec-template" "$REPO_ROOT") || true +SPEC_FILE="$FEATURE_DIR/spec.md" +if [ -n "$TEMPLATE" ] && [ -f "$TEMPLATE" ]; then + cp "$TEMPLATE" "$SPEC_FILE" +else + echo "Warning: Spec template not found; created empty spec file" >&2 + touch "$SPEC_FILE" +fi + +# Inform the user how to persist the feature variable in their own shell +printf '# To persist: export SPECIFY_FEATURE=%q\n' "$BRANCH_NAME" >&2 + +if $JSON_MODE; then + if command -v jq >/dev/null 2>&1; then + jq -cn \ + --arg branch_name "$BRANCH_NAME" \ + --arg spec_file "$SPEC_FILE" \ + --arg feature_num "$FEATURE_NUM" \ + '{BRANCH_NAME:$branch_name,SPEC_FILE:$spec_file,FEATURE_NUM:$feature_num}' + else + printf '{"BRANCH_NAME":"%s","SPEC_FILE":"%s","FEATURE_NUM":"%s"}\n' "$(json_escape "$BRANCH_NAME")" "$(json_escape "$SPEC_FILE")" "$(json_escape "$FEATURE_NUM")" + fi +else + echo "BRANCH_NAME: $BRANCH_NAME" + echo "SPEC_FILE: $SPEC_FILE" + echo "FEATURE_NUM: $FEATURE_NUM" + printf '# To persist in your shell: export SPECIFY_FEATURE=%q\n' "$BRANCH_NAME" +fi diff --git a/.specify/scripts/bash/setup-plan.sh b/.specify/scripts/bash/setup-plan.sh new file mode 100755 index 00000000..9f552314 --- /dev/null +++ b/.specify/scripts/bash/setup-plan.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -e + +# Parse command line arguments +JSON_MODE=false +ARGS=() + +for arg in "$@"; do + case "$arg" in + --json) + JSON_MODE=true + ;; + --help|-h) + echo "Usage: $0 [--json]" + echo " --json Output results in JSON format" + echo " --help Show this help message" + exit 0 + ;; + *) + ARGS+=("$arg") + ;; + esac +done + +# Get script directory and load common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get all paths and variables from common functions +_paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +eval "$_paths_output" +unset _paths_output + +# Check if we're on a proper feature branch (only for git repos) +check_feature_branch "$CURRENT_BRANCH" "$HAS_GIT" || exit 1 + +# Ensure the feature directory exists +mkdir -p "$FEATURE_DIR" + +# Copy plan template if it exists +TEMPLATE=$(resolve_template "plan-template" "$REPO_ROOT") || true +if [[ -n "$TEMPLATE" ]] && [[ -f "$TEMPLATE" ]]; then + cp "$TEMPLATE" "$IMPL_PLAN" + echo "Copied plan template to $IMPL_PLAN" +else + echo "Warning: Plan template not found" + # Create a basic plan file if template doesn't exist + touch "$IMPL_PLAN" +fi + +# Output results +if $JSON_MODE; then + if has_jq; then + jq -cn \ + --arg feature_spec "$FEATURE_SPEC" \ + --arg impl_plan "$IMPL_PLAN" \ + --arg specs_dir "$FEATURE_DIR" \ + --arg branch "$CURRENT_BRANCH" \ + --arg has_git "$HAS_GIT" \ + '{FEATURE_SPEC:$feature_spec,IMPL_PLAN:$impl_plan,SPECS_DIR:$specs_dir,BRANCH:$branch,HAS_GIT:$has_git}' + else + printf '{"FEATURE_SPEC":"%s","IMPL_PLAN":"%s","SPECS_DIR":"%s","BRANCH":"%s","HAS_GIT":"%s"}\n' \ + "$(json_escape "$FEATURE_SPEC")" "$(json_escape "$IMPL_PLAN")" "$(json_escape "$FEATURE_DIR")" "$(json_escape "$CURRENT_BRANCH")" "$(json_escape "$HAS_GIT")" + fi +else + echo "FEATURE_SPEC: $FEATURE_SPEC" + echo "IMPL_PLAN: $IMPL_PLAN" + echo "SPECS_DIR: $FEATURE_DIR" + echo "BRANCH: $CURRENT_BRANCH" + echo "HAS_GIT: $HAS_GIT" +fi + diff --git a/.specify/scripts/bash/update-agent-context.sh b/.specify/scripts/bash/update-agent-context.sh new file mode 100755 index 00000000..74a98669 --- /dev/null +++ b/.specify/scripts/bash/update-agent-context.sh @@ -0,0 +1,832 @@ +#!/usr/bin/env bash + +# Update agent context files with information from plan.md +# +# This script maintains AI agent context files by parsing feature specifications +# and updating agent-specific configuration files with project information. +# +# MAIN FUNCTIONS: +# 1. Environment Validation +# - Verifies git repository structure and branch information +# - Checks for required plan.md files and templates +# - Validates file permissions and accessibility +# +# 2. Plan Data Extraction +# - Parses plan.md files to extract project metadata +# - Identifies language/version, frameworks, databases, and project types +# - Handles missing or incomplete specification data gracefully +# +# 3. Agent File Management +# - Creates new agent context files from templates when needed +# - Updates existing agent files with new project information +# - Preserves manual additions and custom configurations +# - Supports multiple AI agent formats and directory structures +# +# 4. Content Generation +# - Generates language-specific build/test commands +# - Creates appropriate project directory structures +# - Updates technology stacks and recent changes sections +# - Maintains consistent formatting and timestamps +# +# 5. Multi-Agent Support +# - Handles agent-specific file paths and naming conventions +# - Supports: Claude, Gemini, Copilot, Cursor, Qwen, opencode, Codex, Windsurf, Kilo Code, Auggie CLI, Roo Code, CodeBuddy CLI, Qoder CLI, Amp, SHAI, Tabnine CLI, Kiro CLI, Mistral Vibe, Kimi Code, Pi Coding Agent, iFlow CLI, Antigravity or Generic +# - Can update single agents or all existing agent files +# - Creates default Claude file if no agent files exist +# +# Usage: ./update-agent-context.sh [agent_type] +# Agent types: claude|gemini|copilot|cursor-agent|qwen|opencode|codex|windsurf|kilocode|auggie|roo|codebuddy|amp|shai|tabnine|kiro-cli|agy|bob|vibe|qodercli|kimi|trae|pi|iflow|generic +# Leave empty to update all existing agent files + +set -e + +# Enable strict error handling +set -u +set -o pipefail + +#============================================================================== +# Configuration and Global Variables +#============================================================================== + +# Get script directory and load common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get all paths and variables from common functions +_paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +eval "$_paths_output" +unset _paths_output + +NEW_PLAN="$IMPL_PLAN" # Alias for compatibility with existing code +AGENT_TYPE="${1:-}" + +# Agent-specific file paths +CLAUDE_FILE="$REPO_ROOT/CLAUDE.md" +GEMINI_FILE="$REPO_ROOT/GEMINI.md" +COPILOT_FILE="$REPO_ROOT/.github/agents/copilot-instructions.md" +CURSOR_FILE="$REPO_ROOT/.cursor/rules/specify-rules.mdc" +QWEN_FILE="$REPO_ROOT/QWEN.md" +AGENTS_FILE="$REPO_ROOT/AGENTS.md" +WINDSURF_FILE="$REPO_ROOT/.windsurf/rules/specify-rules.md" +KILOCODE_FILE="$REPO_ROOT/.kilocode/rules/specify-rules.md" +AUGGIE_FILE="$REPO_ROOT/.augment/rules/specify-rules.md" +ROO_FILE="$REPO_ROOT/.roo/rules/specify-rules.md" +CODEBUDDY_FILE="$REPO_ROOT/CODEBUDDY.md" +QODER_FILE="$REPO_ROOT/QODER.md" +# Amp, Kiro CLI, IBM Bob, and Pi all share AGENTS.md — use AGENTS_FILE to avoid +# updating the same file multiple times. +AMP_FILE="$AGENTS_FILE" +SHAI_FILE="$REPO_ROOT/SHAI.md" +TABNINE_FILE="$REPO_ROOT/TABNINE.md" +KIRO_FILE="$AGENTS_FILE" +AGY_FILE="$REPO_ROOT/.agent/rules/specify-rules.md" +BOB_FILE="$AGENTS_FILE" +VIBE_FILE="$REPO_ROOT/.vibe/agents/specify-agents.md" +KIMI_FILE="$REPO_ROOT/KIMI.md" +TRAE_FILE="$REPO_ROOT/.trae/rules/AGENTS.md" +IFLOW_FILE="$REPO_ROOT/IFLOW.md" + +# Template file +TEMPLATE_FILE="$REPO_ROOT/.specify/templates/agent-file-template.md" + +# Global variables for parsed plan data +NEW_LANG="" +NEW_FRAMEWORK="" +NEW_DB="" +NEW_PROJECT_TYPE="" + +#============================================================================== +# Utility Functions +#============================================================================== + +log_info() { + echo "INFO: $1" +} + +log_success() { + echo "✓ $1" +} + +log_error() { + echo "ERROR: $1" >&2 +} + +log_warning() { + echo "WARNING: $1" >&2 +} + +# Cleanup function for temporary files +cleanup() { + local exit_code=$? + # Disarm traps to prevent re-entrant loop + trap - EXIT INT TERM + rm -f /tmp/agent_update_*_$$ + rm -f /tmp/manual_additions_$$ + exit $exit_code +} + +# Set up cleanup trap +trap cleanup EXIT INT TERM + +#============================================================================== +# Validation Functions +#============================================================================== + +validate_environment() { + # Check if we have a current branch/feature (git or non-git) + if [[ -z "$CURRENT_BRANCH" ]]; then + log_error "Unable to determine current feature" + if [[ "$HAS_GIT" == "true" ]]; then + log_info "Make sure you're on a feature branch" + else + log_info "Set SPECIFY_FEATURE environment variable or create a feature first" + fi + exit 1 + fi + + # Check if plan.md exists + if [[ ! -f "$NEW_PLAN" ]]; then + log_error "No plan.md found at $NEW_PLAN" + log_info "Make sure you're working on a feature with a corresponding spec directory" + if [[ "$HAS_GIT" != "true" ]]; then + log_info "Use: export SPECIFY_FEATURE=your-feature-name or create a new feature first" + fi + exit 1 + fi + + # Check if template exists (needed for new files) + if [[ ! -f "$TEMPLATE_FILE" ]]; then + log_warning "Template file not found at $TEMPLATE_FILE" + log_warning "Creating new agent files will fail" + fi +} + +#============================================================================== +# Plan Parsing Functions +#============================================================================== + +extract_plan_field() { + local field_pattern="$1" + local plan_file="$2" + + grep "^\*\*${field_pattern}\*\*: " "$plan_file" 2>/dev/null | \ + head -1 | \ + sed "s|^\*\*${field_pattern}\*\*: ||" | \ + sed 's/^[ \t]*//;s/[ \t]*$//' | \ + grep -v "NEEDS CLARIFICATION" | \ + grep -v "^N/A$" || echo "" +} + +parse_plan_data() { + local plan_file="$1" + + if [[ ! -f "$plan_file" ]]; then + log_error "Plan file not found: $plan_file" + return 1 + fi + + if [[ ! -r "$plan_file" ]]; then + log_error "Plan file is not readable: $plan_file" + return 1 + fi + + log_info "Parsing plan data from $plan_file" + + NEW_LANG=$(extract_plan_field "Language/Version" "$plan_file") + NEW_FRAMEWORK=$(extract_plan_field "Primary Dependencies" "$plan_file") + NEW_DB=$(extract_plan_field "Storage" "$plan_file") + NEW_PROJECT_TYPE=$(extract_plan_field "Project Type" "$plan_file") + + # Log what we found + if [[ -n "$NEW_LANG" ]]; then + log_info "Found language: $NEW_LANG" + else + log_warning "No language information found in plan" + fi + + if [[ -n "$NEW_FRAMEWORK" ]]; then + log_info "Found framework: $NEW_FRAMEWORK" + fi + + if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]]; then + log_info "Found database: $NEW_DB" + fi + + if [[ -n "$NEW_PROJECT_TYPE" ]]; then + log_info "Found project type: $NEW_PROJECT_TYPE" + fi +} + +format_technology_stack() { + local lang="$1" + local framework="$2" + local parts=() + + # Add non-empty parts + [[ -n "$lang" && "$lang" != "NEEDS CLARIFICATION" ]] && parts+=("$lang") + [[ -n "$framework" && "$framework" != "NEEDS CLARIFICATION" && "$framework" != "N/A" ]] && parts+=("$framework") + + # Join with proper formatting + if [[ ${#parts[@]} -eq 0 ]]; then + echo "" + elif [[ ${#parts[@]} -eq 1 ]]; then + echo "${parts[0]}" + else + # Join multiple parts with " + " + local result="${parts[0]}" + for ((i=1; i<${#parts[@]}; i++)); do + result="$result + ${parts[i]}" + done + echo "$result" + fi +} + +#============================================================================== +# Template and Content Generation Functions +#============================================================================== + +get_project_structure() { + local project_type="$1" + + if [[ "$project_type" == *"web"* ]]; then + echo "backend/\\nfrontend/\\ntests/" + else + echo "src/\\ntests/" + fi +} + +get_commands_for_language() { + local lang="$1" + + case "$lang" in + *"Python"*) + echo "cd src && pytest && ruff check ." + ;; + *"Rust"*) + echo "cargo test && cargo clippy" + ;; + *"JavaScript"*|*"TypeScript"*) + echo "npm test \\&\\& npm run lint" + ;; + *) + echo "# Add commands for $lang" + ;; + esac +} + +get_language_conventions() { + local lang="$1" + echo "$lang: Follow standard conventions" +} + +create_new_agent_file() { + local target_file="$1" + local temp_file="$2" + local project_name="$3" + local current_date="$4" + + if [[ ! -f "$TEMPLATE_FILE" ]]; then + log_error "Template not found at $TEMPLATE_FILE" + return 1 + fi + + if [[ ! -r "$TEMPLATE_FILE" ]]; then + log_error "Template file is not readable: $TEMPLATE_FILE" + return 1 + fi + + log_info "Creating new agent context file from template..." + + if ! cp "$TEMPLATE_FILE" "$temp_file"; then + log_error "Failed to copy template file" + return 1 + fi + + # Replace template placeholders + local project_structure + project_structure=$(get_project_structure "$NEW_PROJECT_TYPE") + + local commands + commands=$(get_commands_for_language "$NEW_LANG") + + local language_conventions + language_conventions=$(get_language_conventions "$NEW_LANG") + + # Perform substitutions with error checking using safer approach + # Escape special characters for sed by using a different delimiter or escaping + local escaped_lang=$(printf '%s\n' "$NEW_LANG" | sed 's/[\[\.*^$()+{}|]/\\&/g') + local escaped_framework=$(printf '%s\n' "$NEW_FRAMEWORK" | sed 's/[\[\.*^$()+{}|]/\\&/g') + local escaped_branch=$(printf '%s\n' "$CURRENT_BRANCH" | sed 's/[\[\.*^$()+{}|]/\\&/g') + + # Build technology stack and recent change strings conditionally + local tech_stack + if [[ -n "$escaped_lang" && -n "$escaped_framework" ]]; then + tech_stack="- $escaped_lang + $escaped_framework ($escaped_branch)" + elif [[ -n "$escaped_lang" ]]; then + tech_stack="- $escaped_lang ($escaped_branch)" + elif [[ -n "$escaped_framework" ]]; then + tech_stack="- $escaped_framework ($escaped_branch)" + else + tech_stack="- ($escaped_branch)" + fi + + local recent_change + if [[ -n "$escaped_lang" && -n "$escaped_framework" ]]; then + recent_change="- $escaped_branch: Added $escaped_lang + $escaped_framework" + elif [[ -n "$escaped_lang" ]]; then + recent_change="- $escaped_branch: Added $escaped_lang" + elif [[ -n "$escaped_framework" ]]; then + recent_change="- $escaped_branch: Added $escaped_framework" + else + recent_change="- $escaped_branch: Added" + fi + + local substitutions=( + "s|\[PROJECT NAME\]|$project_name|" + "s|\[DATE\]|$current_date|" + "s|\[EXTRACTED FROM ALL PLAN.MD FILES\]|$tech_stack|" + "s|\[ACTUAL STRUCTURE FROM PLANS\]|$project_structure|g" + "s|\[ONLY COMMANDS FOR ACTIVE TECHNOLOGIES\]|$commands|" + "s|\[LANGUAGE-SPECIFIC, ONLY FOR LANGUAGES IN USE\]|$language_conventions|" + "s|\[LAST 3 FEATURES AND WHAT THEY ADDED\]|$recent_change|" + ) + + for substitution in "${substitutions[@]}"; do + if ! sed -i.bak -e "$substitution" "$temp_file"; then + log_error "Failed to perform substitution: $substitution" + rm -f "$temp_file" "$temp_file.bak" + return 1 + fi + done + + # Convert \n sequences to actual newlines + newline=$(printf '\n') + sed -i.bak2 "s/\\\\n/${newline}/g" "$temp_file" + + # Clean up backup files + rm -f "$temp_file.bak" "$temp_file.bak2" + + # Prepend Cursor frontmatter for .mdc files so rules are auto-included + if [[ "$target_file" == *.mdc ]]; then + local frontmatter_file + frontmatter_file=$(mktemp) || return 1 + printf '%s\n' "---" "description: Project Development Guidelines" "globs: [\"**/*\"]" "alwaysApply: true" "---" "" > "$frontmatter_file" + cat "$temp_file" >> "$frontmatter_file" + mv "$frontmatter_file" "$temp_file" + fi + + return 0 +} + + + + +update_existing_agent_file() { + local target_file="$1" + local current_date="$2" + + log_info "Updating existing agent context file..." + + # Use a single temporary file for atomic update + local temp_file + temp_file=$(mktemp) || { + log_error "Failed to create temporary file" + return 1 + } + + # Process the file in one pass + local tech_stack=$(format_technology_stack "$NEW_LANG" "$NEW_FRAMEWORK") + local new_tech_entries=() + local new_change_entry="" + + # Prepare new technology entries + if [[ -n "$tech_stack" ]] && ! grep -q "$tech_stack" "$target_file"; then + new_tech_entries+=("- $tech_stack ($CURRENT_BRANCH)") + fi + + if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]] && [[ "$NEW_DB" != "NEEDS CLARIFICATION" ]] && ! grep -q "$NEW_DB" "$target_file"; then + new_tech_entries+=("- $NEW_DB ($CURRENT_BRANCH)") + fi + + # Prepare new change entry + if [[ -n "$tech_stack" ]]; then + new_change_entry="- $CURRENT_BRANCH: Added $tech_stack" + elif [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]] && [[ "$NEW_DB" != "NEEDS CLARIFICATION" ]]; then + new_change_entry="- $CURRENT_BRANCH: Added $NEW_DB" + fi + + # Check if sections exist in the file + local has_active_technologies=0 + local has_recent_changes=0 + + if grep -q "^## Active Technologies" "$target_file" 2>/dev/null; then + has_active_technologies=1 + fi + + if grep -q "^## Recent Changes" "$target_file" 2>/dev/null; then + has_recent_changes=1 + fi + + # Process file line by line + local in_tech_section=false + local in_changes_section=false + local tech_entries_added=false + local changes_entries_added=false + local existing_changes_count=0 + local file_ended=false + + while IFS= read -r line || [[ -n "$line" ]]; do + # Handle Active Technologies section + if [[ "$line" == "## Active Technologies" ]]; then + echo "$line" >> "$temp_file" + in_tech_section=true + continue + elif [[ $in_tech_section == true ]] && [[ "$line" =~ ^##[[:space:]] ]]; then + # Add new tech entries before closing the section + if [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then + printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file" + tech_entries_added=true + fi + echo "$line" >> "$temp_file" + in_tech_section=false + continue + elif [[ $in_tech_section == true ]] && [[ -z "$line" ]]; then + # Add new tech entries before empty line in tech section + if [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then + printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file" + tech_entries_added=true + fi + echo "$line" >> "$temp_file" + continue + fi + + # Handle Recent Changes section + if [[ "$line" == "## Recent Changes" ]]; then + echo "$line" >> "$temp_file" + # Add new change entry right after the heading + if [[ -n "$new_change_entry" ]]; then + echo "$new_change_entry" >> "$temp_file" + fi + in_changes_section=true + changes_entries_added=true + continue + elif [[ $in_changes_section == true ]] && [[ "$line" =~ ^##[[:space:]] ]]; then + echo "$line" >> "$temp_file" + in_changes_section=false + continue + elif [[ $in_changes_section == true ]] && [[ "$line" == "- "* ]]; then + # Keep only first 2 existing changes + if [[ $existing_changes_count -lt 2 ]]; then + echo "$line" >> "$temp_file" + ((existing_changes_count++)) + fi + continue + fi + + # Update timestamp + if [[ "$line" =~ (\*\*)?Last\ updated(\*\*)?:.*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] ]]; then + echo "$line" | sed "s/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]/$current_date/" >> "$temp_file" + else + echo "$line" >> "$temp_file" + fi + done < "$target_file" + + # Post-loop check: if we're still in the Active Technologies section and haven't added new entries + if [[ $in_tech_section == true ]] && [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then + printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file" + tech_entries_added=true + fi + + # If sections don't exist, add them at the end of the file + if [[ $has_active_technologies -eq 0 ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then + echo "" >> "$temp_file" + echo "## Active Technologies" >> "$temp_file" + printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file" + tech_entries_added=true + fi + + if [[ $has_recent_changes -eq 0 ]] && [[ -n "$new_change_entry" ]]; then + echo "" >> "$temp_file" + echo "## Recent Changes" >> "$temp_file" + echo "$new_change_entry" >> "$temp_file" + changes_entries_added=true + fi + + # Ensure Cursor .mdc files have YAML frontmatter for auto-inclusion + if [[ "$target_file" == *.mdc ]]; then + if ! head -1 "$temp_file" | grep -q '^---'; then + local frontmatter_file + frontmatter_file=$(mktemp) || { rm -f "$temp_file"; return 1; } + printf '%s\n' "---" "description: Project Development Guidelines" "globs: [\"**/*\"]" "alwaysApply: true" "---" "" > "$frontmatter_file" + cat "$temp_file" >> "$frontmatter_file" + mv "$frontmatter_file" "$temp_file" + fi + fi + + # Move temp file to target atomically + if ! mv "$temp_file" "$target_file"; then + log_error "Failed to update target file" + rm -f "$temp_file" + return 1 + fi + + return 0 +} +#============================================================================== +# Main Agent File Update Function +#============================================================================== + +update_agent_file() { + local target_file="$1" + local agent_name="$2" + + if [[ -z "$target_file" ]] || [[ -z "$agent_name" ]]; then + log_error "update_agent_file requires target_file and agent_name parameters" + return 1 + fi + + log_info "Updating $agent_name context file: $target_file" + + local project_name + project_name=$(basename "$REPO_ROOT") + local current_date + current_date=$(date +%Y-%m-%d) + + # Create directory if it doesn't exist + local target_dir + target_dir=$(dirname "$target_file") + if [[ ! -d "$target_dir" ]]; then + if ! mkdir -p "$target_dir"; then + log_error "Failed to create directory: $target_dir" + return 1 + fi + fi + + if [[ ! -f "$target_file" ]]; then + # Create new file from template + local temp_file + temp_file=$(mktemp) || { + log_error "Failed to create temporary file" + return 1 + } + + if create_new_agent_file "$target_file" "$temp_file" "$project_name" "$current_date"; then + if mv "$temp_file" "$target_file"; then + log_success "Created new $agent_name context file" + else + log_error "Failed to move temporary file to $target_file" + rm -f "$temp_file" + return 1 + fi + else + log_error "Failed to create new agent file" + rm -f "$temp_file" + return 1 + fi + else + # Update existing file + if [[ ! -r "$target_file" ]]; then + log_error "Cannot read existing file: $target_file" + return 1 + fi + + if [[ ! -w "$target_file" ]]; then + log_error "Cannot write to existing file: $target_file" + return 1 + fi + + if update_existing_agent_file "$target_file" "$current_date"; then + log_success "Updated existing $agent_name context file" + else + log_error "Failed to update existing agent file" + return 1 + fi + fi + + return 0 +} + +#============================================================================== +# Agent Selection and Processing +#============================================================================== + +update_specific_agent() { + local agent_type="$1" + + case "$agent_type" in + claude) + update_agent_file "$CLAUDE_FILE" "Claude Code" || return 1 + ;; + gemini) + update_agent_file "$GEMINI_FILE" "Gemini CLI" || return 1 + ;; + copilot) + update_agent_file "$COPILOT_FILE" "GitHub Copilot" || return 1 + ;; + cursor-agent) + update_agent_file "$CURSOR_FILE" "Cursor IDE" || return 1 + ;; + qwen) + update_agent_file "$QWEN_FILE" "Qwen Code" || return 1 + ;; + opencode) + update_agent_file "$AGENTS_FILE" "opencode" || return 1 + ;; + codex) + update_agent_file "$AGENTS_FILE" "Codex CLI" || return 1 + ;; + windsurf) + update_agent_file "$WINDSURF_FILE" "Windsurf" || return 1 + ;; + kilocode) + update_agent_file "$KILOCODE_FILE" "Kilo Code" || return 1 + ;; + auggie) + update_agent_file "$AUGGIE_FILE" "Auggie CLI" || return 1 + ;; + roo) + update_agent_file "$ROO_FILE" "Roo Code" || return 1 + ;; + codebuddy) + update_agent_file "$CODEBUDDY_FILE" "CodeBuddy CLI" || return 1 + ;; + qodercli) + update_agent_file "$QODER_FILE" "Qoder CLI" || return 1 + ;; + amp) + update_agent_file "$AMP_FILE" "Amp" || return 1 + ;; + shai) + update_agent_file "$SHAI_FILE" "SHAI" || return 1 + ;; + tabnine) + update_agent_file "$TABNINE_FILE" "Tabnine CLI" || return 1 + ;; + kiro-cli) + update_agent_file "$KIRO_FILE" "Kiro CLI" || return 1 + ;; + agy) + update_agent_file "$AGY_FILE" "Antigravity" || return 1 + ;; + bob) + update_agent_file "$BOB_FILE" "IBM Bob" || return 1 + ;; + vibe) + update_agent_file "$VIBE_FILE" "Mistral Vibe" || return 1 + ;; + kimi) + update_agent_file "$KIMI_FILE" "Kimi Code" || return 1 + ;; + trae) + update_agent_file "$TRAE_FILE" "Trae" || return 1 + ;; + pi) + update_agent_file "$AGENTS_FILE" "Pi Coding Agent" || return 1 + ;; + iflow) + update_agent_file "$IFLOW_FILE" "iFlow CLI" || return 1 + ;; + generic) + log_info "Generic agent: no predefined context file. Use the agent-specific update script for your agent." + ;; + *) + log_error "Unknown agent type '$agent_type'" + log_error "Expected: claude|gemini|copilot|cursor-agent|qwen|opencode|codex|windsurf|kilocode|auggie|roo|codebuddy|amp|shai|tabnine|kiro-cli|agy|bob|vibe|qodercli|kimi|trae|pi|iflow|generic" + exit 1 + ;; + esac +} + +# Helper: skip non-existent files and files already updated (dedup by +# realpath so that variables pointing to the same file — e.g. AMP_FILE, +# KIRO_FILE, BOB_FILE all resolving to AGENTS_FILE — are only written once). +# Uses a linear array instead of associative array for bash 3.2 compatibility. +# Note: defined at top level because bash 3.2 does not support true +# nested/local functions. _updated_paths, _found_agent, and _all_ok are +# initialised exclusively inside update_all_existing_agents so that +# sourcing this script has no side effects on the caller's environment. + +_update_if_new() { + local file="$1" name="$2" + [[ -f "$file" ]] || return 0 + local real_path + real_path=$(realpath "$file" 2>/dev/null || echo "$file") + local p + if [[ ${#_updated_paths[@]} -gt 0 ]]; then + for p in "${_updated_paths[@]}"; do + [[ "$p" == "$real_path" ]] && return 0 + done + fi + # Record the file as seen before attempting the update so that: + # (a) aliases pointing to the same path are not retried on failure + # (b) _found_agent reflects file existence, not update success + _updated_paths+=("$real_path") + _found_agent=true + update_agent_file "$file" "$name" +} + +update_all_existing_agents() { + _found_agent=false + _updated_paths=() + local _all_ok=true + + _update_if_new "$CLAUDE_FILE" "Claude Code" || _all_ok=false + _update_if_new "$GEMINI_FILE" "Gemini CLI" || _all_ok=false + _update_if_new "$COPILOT_FILE" "GitHub Copilot" || _all_ok=false + _update_if_new "$CURSOR_FILE" "Cursor IDE" || _all_ok=false + _update_if_new "$QWEN_FILE" "Qwen Code" || _all_ok=false + _update_if_new "$AGENTS_FILE" "Codex/opencode" || _all_ok=false + _update_if_new "$AMP_FILE" "Amp" || _all_ok=false + _update_if_new "$KIRO_FILE" "Kiro CLI" || _all_ok=false + _update_if_new "$BOB_FILE" "IBM Bob" || _all_ok=false + _update_if_new "$WINDSURF_FILE" "Windsurf" || _all_ok=false + _update_if_new "$KILOCODE_FILE" "Kilo Code" || _all_ok=false + _update_if_new "$AUGGIE_FILE" "Auggie CLI" || _all_ok=false + _update_if_new "$ROO_FILE" "Roo Code" || _all_ok=false + _update_if_new "$CODEBUDDY_FILE" "CodeBuddy CLI" || _all_ok=false + _update_if_new "$SHAI_FILE" "SHAI" || _all_ok=false + _update_if_new "$TABNINE_FILE" "Tabnine CLI" || _all_ok=false + _update_if_new "$QODER_FILE" "Qoder CLI" || _all_ok=false + _update_if_new "$AGY_FILE" "Antigravity" || _all_ok=false + _update_if_new "$VIBE_FILE" "Mistral Vibe" || _all_ok=false + _update_if_new "$KIMI_FILE" "Kimi Code" || _all_ok=false + _update_if_new "$TRAE_FILE" "Trae" || _all_ok=false + _update_if_new "$IFLOW_FILE" "iFlow CLI" || _all_ok=false + + # If no agent files exist, create a default Claude file + if [[ "$_found_agent" == false ]]; then + log_info "No existing agent files found, creating default Claude file..." + update_agent_file "$CLAUDE_FILE" "Claude Code" || return 1 + fi + + [[ "$_all_ok" == true ]] +} +print_summary() { + echo + log_info "Summary of changes:" + + if [[ -n "$NEW_LANG" ]]; then + echo " - Added language: $NEW_LANG" + fi + + if [[ -n "$NEW_FRAMEWORK" ]]; then + echo " - Added framework: $NEW_FRAMEWORK" + fi + + if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]]; then + echo " - Added database: $NEW_DB" + fi + + echo + log_info "Usage: $0 [claude|gemini|copilot|cursor-agent|qwen|opencode|codex|windsurf|kilocode|auggie|roo|codebuddy|amp|shai|tabnine|kiro-cli|agy|bob|vibe|qodercli|kimi|trae|pi|iflow|generic]" +} + +#============================================================================== +# Main Execution +#============================================================================== + +main() { + # Validate environment before proceeding + validate_environment + + log_info "=== Updating agent context files for feature $CURRENT_BRANCH ===" + + # Parse the plan file to extract project information + if ! parse_plan_data "$NEW_PLAN"; then + log_error "Failed to parse plan data" + exit 1 + fi + + # Process based on agent type argument + local success=true + + if [[ -z "$AGENT_TYPE" ]]; then + # No specific agent provided - update all existing agent files + log_info "No agent specified, updating all existing agent files..." + if ! update_all_existing_agents; then + success=false + fi + else + # Specific agent provided - update only that agent + log_info "Updating specific agent: $AGENT_TYPE" + if ! update_specific_agent "$AGENT_TYPE"; then + success=false + fi + fi + + # Print summary + print_summary + + if [[ "$success" == true ]]; then + log_success "Agent context update completed successfully" + exit 0 + else + log_error "Agent context update completed with errors" + exit 1 + fi +} + +# Execute main function if script is run directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/.specify/templates/agent-file-template.md b/.specify/templates/agent-file-template.md new file mode 100644 index 00000000..4cc7fd66 --- /dev/null +++ b/.specify/templates/agent-file-template.md @@ -0,0 +1,28 @@ +# [PROJECT NAME] Development Guidelines + +Auto-generated from all feature plans. Last updated: [DATE] + +## Active Technologies + +[EXTRACTED FROM ALL PLAN.MD FILES] + +## Project Structure + +```text +[ACTUAL STRUCTURE FROM PLANS] +``` + +## Commands + +[ONLY COMMANDS FOR ACTIVE TECHNOLOGIES] + +## Code Style + +[LANGUAGE-SPECIFIC, ONLY FOR LANGUAGES IN USE] + +## Recent Changes + +[LAST 3 FEATURES AND WHAT THEY ADDED] + + + diff --git a/.specify/templates/checklist-template.md b/.specify/templates/checklist-template.md new file mode 100644 index 00000000..806657da --- /dev/null +++ b/.specify/templates/checklist-template.md @@ -0,0 +1,40 @@ +# [CHECKLIST TYPE] Checklist: [FEATURE NAME] + +**Purpose**: [Brief description of what this checklist covers] +**Created**: [DATE] +**Feature**: [Link to spec.md or relevant documentation] + +**Note**: This checklist is generated by the `/speckit.checklist` command based on feature context and requirements. + + + +## [Category 1] + +- [ ] CHK001 First checklist item with clear action +- [ ] CHK002 Second checklist item +- [ ] CHK003 Third checklist item + +## [Category 2] + +- [ ] CHK004 Another category item +- [ ] CHK005 Item with specific criteria +- [ ] CHK006 Final item in this category + +## Notes + +- Check items off as completed: `[x]` +- Add comments or findings inline +- Link to relevant resources or documentation +- Items are numbered sequentially for easy reference diff --git a/.specify/templates/constitution-template.md b/.specify/templates/constitution-template.md new file mode 100644 index 00000000..a4670ff4 --- /dev/null +++ b/.specify/templates/constitution-template.md @@ -0,0 +1,50 @@ +# [PROJECT_NAME] Constitution + + +## Core Principles + +### [PRINCIPLE_1_NAME] + +[PRINCIPLE_1_DESCRIPTION] + + +### [PRINCIPLE_2_NAME] + +[PRINCIPLE_2_DESCRIPTION] + + +### [PRINCIPLE_3_NAME] + +[PRINCIPLE_3_DESCRIPTION] + + +### [PRINCIPLE_4_NAME] + +[PRINCIPLE_4_DESCRIPTION] + + +### [PRINCIPLE_5_NAME] + +[PRINCIPLE_5_DESCRIPTION] + + +## [SECTION_2_NAME] + + +[SECTION_2_CONTENT] + + +## [SECTION_3_NAME] + + +[SECTION_3_CONTENT] + + +## Governance + + +[GOVERNANCE_RULES] + + +**Version**: [CONSTITUTION_VERSION] | **Ratified**: [RATIFICATION_DATE] | **Last Amended**: [LAST_AMENDED_DATE] + diff --git a/.specify/templates/plan-template.md b/.specify/templates/plan-template.md new file mode 100644 index 00000000..5a2fafeb --- /dev/null +++ b/.specify/templates/plan-template.md @@ -0,0 +1,104 @@ +# Implementation Plan: [FEATURE] + +**Branch**: `[###-feature-name]` | **Date**: [DATE] | **Spec**: [link] +**Input**: Feature specification from `/specs/[###-feature-name]/spec.md` + +**Note**: This template is filled in by the `/speckit.plan` command. See `.specify/templates/plan-template.md` for the execution workflow. + +## Summary + +[Extract from feature spec: primary requirement + technical approach from research] + +## Technical Context + + + +**Language/Version**: [e.g., Python 3.11, Swift 5.9, Rust 1.75 or NEEDS CLARIFICATION] +**Primary Dependencies**: [e.g., FastAPI, UIKit, LLVM or NEEDS CLARIFICATION] +**Storage**: [if applicable, e.g., PostgreSQL, CoreData, files or N/A] +**Testing**: [e.g., pytest, XCTest, cargo test or NEEDS CLARIFICATION] +**Target Platform**: [e.g., Linux server, iOS 15+, WASM or NEEDS CLARIFICATION] +**Project Type**: [e.g., library/cli/web-service/mobile-app/compiler/desktop-app or NEEDS CLARIFICATION] +**Performance Goals**: [domain-specific, e.g., 1000 req/s, 10k lines/sec, 60 fps or NEEDS CLARIFICATION] +**Constraints**: [domain-specific, e.g., <200ms p95, <100MB memory, offline-capable or NEEDS CLARIFICATION] +**Scale/Scope**: [domain-specific, e.g., 10k users, 1M LOC, 50 screens or NEEDS CLARIFICATION] + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +[Gates determined based on constitution file] + +## Project Structure + +### Documentation (this feature) + +```text +specs/[###-feature]/ +├── plan.md # This file (/speckit.plan command output) +├── research.md # Phase 0 output (/speckit.plan command) +├── data-model.md # Phase 1 output (/speckit.plan command) +├── quickstart.md # Phase 1 output (/speckit.plan command) +├── contracts/ # Phase 1 output (/speckit.plan command) +└── tasks.md # Phase 2 output (/speckit.tasks command - NOT created by /speckit.plan) +``` + +### Source Code (repository root) + + +```text +# [REMOVE IF UNUSED] Option 1: Single project (DEFAULT) +src/ +├── models/ +├── services/ +├── cli/ +└── lib/ + +tests/ +├── contract/ +├── integration/ +└── unit/ + +# [REMOVE IF UNUSED] Option 2: Web application (when "frontend" + "backend" detected) +backend/ +├── src/ +│ ├── models/ +│ ├── services/ +│ └── api/ +└── tests/ + +frontend/ +├── src/ +│ ├── components/ +│ ├── pages/ +│ └── services/ +└── tests/ + +# [REMOVE IF UNUSED] Option 3: Mobile + API (when "iOS/Android" detected) +api/ +└── [same as backend above] + +ios/ or android/ +└── [platform-specific structure: feature modules, UI flows, platform tests] +``` + +**Structure Decision**: [Document the selected structure and reference the real +directories captured above] + +## Complexity Tracking + +> **Fill ONLY if Constitution Check has violations that must be justified** + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| [e.g., 4th project] | [current need] | [why 3 projects insufficient] | +| [e.g., Repository pattern] | [specific problem] | [why direct DB access insufficient] | diff --git a/.specify/templates/spec-template.md b/.specify/templates/spec-template.md new file mode 100644 index 00000000..c67d9149 --- /dev/null +++ b/.specify/templates/spec-template.md @@ -0,0 +1,115 @@ +# Feature Specification: [FEATURE NAME] + +**Feature Branch**: `[###-feature-name]` +**Created**: [DATE] +**Status**: Draft +**Input**: User description: "$ARGUMENTS" + +## User Scenarios & Testing *(mandatory)* + + + +### User Story 1 - [Brief Title] (Priority: P1) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently - e.g., "Can be fully tested by [specific action] and delivers [specific value]"] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] +2. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 2 - [Brief Title] (Priority: P2) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 3 - [Brief Title] (Priority: P3) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +[Add more user stories as needed, each with an assigned priority] + +### Edge Cases + + + +- What happens when [boundary condition]? +- How does system handle [error scenario]? + +## Requirements *(mandatory)* + + + +### Functional Requirements + +- **FR-001**: System MUST [specific capability, e.g., "allow users to create accounts"] +- **FR-002**: System MUST [specific capability, e.g., "validate email addresses"] +- **FR-003**: Users MUST be able to [key interaction, e.g., "reset their password"] +- **FR-004**: System MUST [data requirement, e.g., "persist user preferences"] +- **FR-005**: System MUST [behavior, e.g., "log all security events"] + +*Example of marking unclear requirements:* + +- **FR-006**: System MUST authenticate users via [NEEDS CLARIFICATION: auth method not specified - email/password, SSO, OAuth?] +- **FR-007**: System MUST retain user data for [NEEDS CLARIFICATION: retention period not specified] + +### Key Entities *(include if feature involves data)* + +- **[Entity 1]**: [What it represents, key attributes without implementation] +- **[Entity 2]**: [What it represents, relationships to other entities] + +## Success Criteria *(mandatory)* + + + +### Measurable Outcomes + +- **SC-001**: [Measurable metric, e.g., "Users can complete account creation in under 2 minutes"] +- **SC-002**: [Measurable metric, e.g., "System handles 1000 concurrent users without degradation"] +- **SC-003**: [User satisfaction metric, e.g., "90% of users successfully complete primary task on first attempt"] +- **SC-004**: [Business metric, e.g., "Reduce support tickets related to [X] by 50%"] diff --git a/.specify/templates/tasks-template.md b/.specify/templates/tasks-template.md new file mode 100644 index 00000000..60f9be45 --- /dev/null +++ b/.specify/templates/tasks-template.md @@ -0,0 +1,251 @@ +--- + +description: "Task list template for feature implementation" +--- + +# Tasks: [FEATURE NAME] + +**Input**: Design documents from `/specs/[###-feature-name]/` +**Prerequisites**: plan.md (required), spec.md (required for user stories), research.md, data-model.md, contracts/ + +**Tests**: The examples below include test tasks. Tests are OPTIONAL - only include them if explicitly requested in the feature specification. + +**Organization**: Tasks are grouped by user story to enable independent implementation and testing of each story. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (e.g., US1, US2, US3) +- Include exact file paths in descriptions + +## Path Conventions + +- **Single project**: `src/`, `tests/` at repository root +- **Web app**: `backend/src/`, `frontend/src/` +- **Mobile**: `api/src/`, `ios/src/` or `android/src/` +- Paths shown below assume single project - adjust based on plan.md structure + + + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Project initialization and basic structure + +- [ ] T001 Create project structure per implementation plan +- [ ] T002 Initialize [language] project with [framework] dependencies +- [ ] T003 [P] Configure linting and formatting tools + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Core infrastructure that MUST be complete before ANY user story can be implemented + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete + +Examples of foundational tasks (adjust based on your project): + +- [ ] T004 Setup database schema and migrations framework +- [ ] T005 [P] Implement authentication/authorization framework +- [ ] T006 [P] Setup API routing and middleware structure +- [ ] T007 Create base models/entities that all stories depend on +- [ ] T008 Configure error handling and logging infrastructure +- [ ] T009 Setup environment configuration management + +**Checkpoint**: Foundation ready - user story implementation can now begin in parallel + +--- + +## Phase 3: User Story 1 - [Title] (Priority: P1) 🎯 MVP + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 1 (OPTIONAL - only if tests requested) ⚠️ + +> **NOTE: Write these tests FIRST, ensure they FAIL before implementation** + +- [ ] T010 [P] [US1] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T011 [P] [US1] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 1 + +- [ ] T012 [P] [US1] Create [Entity1] model in src/models/[entity1].py +- [ ] T013 [P] [US1] Create [Entity2] model in src/models/[entity2].py +- [ ] T014 [US1] Implement [Service] in src/services/[service].py (depends on T012, T013) +- [ ] T015 [US1] Implement [endpoint/feature] in src/[location]/[file].py +- [ ] T016 [US1] Add validation and error handling +- [ ] T017 [US1] Add logging for user story 1 operations + +**Checkpoint**: At this point, User Story 1 should be fully functional and testable independently + +--- + +## Phase 4: User Story 2 - [Title] (Priority: P2) + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 2 (OPTIONAL - only if tests requested) ⚠️ + +- [ ] T018 [P] [US2] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T019 [P] [US2] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 2 + +- [ ] T020 [P] [US2] Create [Entity] model in src/models/[entity].py +- [ ] T021 [US2] Implement [Service] in src/services/[service].py +- [ ] T022 [US2] Implement [endpoint/feature] in src/[location]/[file].py +- [ ] T023 [US2] Integrate with User Story 1 components (if needed) + +**Checkpoint**: At this point, User Stories 1 AND 2 should both work independently + +--- + +## Phase 5: User Story 3 - [Title] (Priority: P3) + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 3 (OPTIONAL - only if tests requested) ⚠️ + +- [ ] T024 [P] [US3] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T025 [P] [US3] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 3 + +- [ ] T026 [P] [US3] Create [Entity] model in src/models/[entity].py +- [ ] T027 [US3] Implement [Service] in src/services/[service].py +- [ ] T028 [US3] Implement [endpoint/feature] in src/[location]/[file].py + +**Checkpoint**: All user stories should now be independently functional + +--- + +[Add more user story phases as needed, following the same pattern] + +--- + +## Phase N: Polish & Cross-Cutting Concerns + +**Purpose**: Improvements that affect multiple user stories + +- [ ] TXXX [P] Documentation updates in docs/ +- [ ] TXXX Code cleanup and refactoring +- [ ] TXXX Performance optimization across all stories +- [ ] TXXX [P] Additional unit tests (if requested) in tests/unit/ +- [ ] TXXX Security hardening +- [ ] TXXX Run quickstart.md validation + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies - can start immediately +- **Foundational (Phase 2)**: Depends on Setup completion - BLOCKS all user stories +- **User Stories (Phase 3+)**: All depend on Foundational phase completion + - User stories can then proceed in parallel (if staffed) + - Or sequentially in priority order (P1 → P2 → P3) +- **Polish (Final Phase)**: Depends on all desired user stories being complete + +### User Story Dependencies + +- **User Story 1 (P1)**: Can start after Foundational (Phase 2) - No dependencies on other stories +- **User Story 2 (P2)**: Can start after Foundational (Phase 2) - May integrate with US1 but should be independently testable +- **User Story 3 (P3)**: Can start after Foundational (Phase 2) - May integrate with US1/US2 but should be independently testable + +### Within Each User Story + +- Tests (if included) MUST be written and FAIL before implementation +- Models before services +- Services before endpoints +- Core implementation before integration +- Story complete before moving to next priority + +### Parallel Opportunities + +- All Setup tasks marked [P] can run in parallel +- All Foundational tasks marked [P] can run in parallel (within Phase 2) +- Once Foundational phase completes, all user stories can start in parallel (if team capacity allows) +- All tests for a user story marked [P] can run in parallel +- Models within a story marked [P] can run in parallel +- Different user stories can be worked on in parallel by different team members + +--- + +## Parallel Example: User Story 1 + +```bash +# Launch all tests for User Story 1 together (if tests requested): +Task: "Contract test for [endpoint] in tests/contract/test_[name].py" +Task: "Integration test for [user journey] in tests/integration/test_[name].py" + +# Launch all models for User Story 1 together: +Task: "Create [Entity1] model in src/models/[entity1].py" +Task: "Create [Entity2] model in src/models/[entity2].py" +``` + +--- + +## Implementation Strategy + +### MVP First (User Story 1 Only) + +1. Complete Phase 1: Setup +2. Complete Phase 2: Foundational (CRITICAL - blocks all stories) +3. Complete Phase 3: User Story 1 +4. **STOP and VALIDATE**: Test User Story 1 independently +5. Deploy/demo if ready + +### Incremental Delivery + +1. Complete Setup + Foundational → Foundation ready +2. Add User Story 1 → Test independently → Deploy/Demo (MVP!) +3. Add User Story 2 → Test independently → Deploy/Demo +4. Add User Story 3 → Test independently → Deploy/Demo +5. Each story adds value without breaking previous stories + +### Parallel Team Strategy + +With multiple developers: + +1. Team completes Setup + Foundational together +2. Once Foundational is done: + - Developer A: User Story 1 + - Developer B: User Story 2 + - Developer C: User Story 3 +3. Stories complete and integrate independently + +--- + +## Notes + +- [P] tasks = different files, no dependencies +- [Story] label maps task to specific user story for traceability +- Each user story should be independently completable and testable +- Verify tests fail before implementing +- Commit after each task or logical group +- Stop at any checkpoint to validate story independently +- Avoid: vague tasks, same file conflicts, cross-story dependencies that break independence diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..30216ed6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,77 @@ +# CodeGraphContext Development Guidelines + +Auto-generated from all feature plans. Last updated: 2026-03-20 + +## Active Technologies +- Neo4j (production) / FalkorDB (default) — same shared instance as CGC core; (001-cgc-plugin-extension) + +- Python 3.10+ (constitutional constraint) (001-cgc-plugin-extension) + +## Project Structure + +```text +src/ + codegraphcontext/ + plugin_registry.py ← PluginRegistry (discovers cgc_cli_plugins + cgc_mcp_plugins entry points) + cli/main.py ← CLI app; loads plugin CLI commands at import time + server.py ← MCPServer; loads plugin MCP tools at init time +tests/ + unit/plugin/ ← Unit tests for plugin system (mocked entry points) + integration/plugin/ ← Integration tests (real stub plugin if installed) + e2e/plugin/ ← Full lifecycle E2E tests +plugins/ + cgc-plugin-stub/ ← Reference stub plugin (minimal test fixture) + cgc-plugin-otel/ ← OpenTelemetry span receiver plugin + cgc-plugin-xdebug/ ← Xdebug DBGp call-stack listener plugin +docs/ + plugins/ + authoring-guide.md ← How to write a CGC plugin + cross-layer-queries.md ← Canonical cross-layer Cypher queries +``` + +## Commands + +cd src [ONLY COMMANDS FOR ACTIVE TECHNOLOGIES][ONLY COMMANDS FOR ACTIVE TECHNOLOGIES] pytest [ONLY COMMANDS FOR ACTIVE TECHNOLOGIES][ONLY COMMANDS FOR ACTIVE TECHNOLOGIES] ruff check . + +## Code Style + +Python 3.10+ (constitutional constraint): Follow standard conventions + +## Recent Changes +- 001-cgc-plugin-extension: Added Python 3.10+ (constitutional constraint) + +- 001-cgc-plugin-extension: Added Python 3.10+ (constitutional constraint) + + +## Plugin System (001-cgc-plugin-extension) + +### Entry-point groups +- `cgc_cli_plugins` — plugins contribute a `(name, typer.Typer)` via `get_plugin_commands()` +- `cgc_mcp_plugins` — plugins contribute MCP tools via `get_mcp_tools()` and `get_mcp_handlers()` + +### Plugin layout convention +``` +plugins/cgc-plugin-/ +├── pyproject.toml ← entry-points in both cgc_cli_plugins + cgc_mcp_plugins +└── src/cgc_plugin_/ + ├── __init__.py ← PLUGIN_METADATA dict (required) + ├── cli.py ← get_plugin_commands() + └── mcp_tools.py ← get_mcp_tools() + get_mcp_handlers() +``` + +### MCP tool naming +Plugin tools must be prefixed with plugin name: `_` (e.g. `otel_query_spans`). + +### Install plugins for development +```bash +pip install -e plugins/cgc-plugin-stub # minimal test fixture +pip install -e plugins/cgc-plugin-otel +pip install -e plugins/cgc-plugin-xdebug +``` + +### Run plugin tests +```bash +PYTHONPATH=src pytest tests/unit/plugin/ tests/integration/plugin/ -v +PYTHONPATH=src pytest tests/e2e/plugin/ -v # e2e (needs plugins installed) +``` + diff --git a/Dockerfile.mcp b/Dockerfile.mcp new file mode 100644 index 00000000..3078ff01 --- /dev/null +++ b/Dockerfile.mcp @@ -0,0 +1,92 @@ +# Multi-stage build for CGC hosted MCP server (HTTP transport) +# +# Bundles CGC core + cgc-plugin-otel + cgc-plugin-xdebug into a single image +# that serves the Model Context Protocol over HTTP on port 8045. +# +# Build: +# docker build -f Dockerfile.mcp -t cgc-mcp:latest . +# +# Run (credentials supplied at runtime — never baked in): +# docker run -e DATABASE_TYPE=neo4j \ +# -e NEO4J_URI=bolt://neo4j:7687 \ +# -e NEO4J_USERNAME=neo4j \ +# -e NEO4J_PASSWORD= \ +# -p 8045:8045 cgc-mcp:latest + +# ── Builder stage ───────────────────────────────────────────────────────────── +FROM python:3.12-slim AS builder + +WORKDIR /app + +# System build dependencies +RUN apt-get update && apt-get install -y \ + gcc \ + g++ \ + make \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Install CGC core +COPY pyproject.toml README.md LICENSE MANIFEST.in ./ +COPY src/ ./src/ +RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ + pip install --no-cache-dir . + +# Install cgc-plugin-otel +COPY plugins/cgc-plugin-otel/ ./plugins/cgc-plugin-otel/ +RUN pip install --no-cache-dir ./plugins/cgc-plugin-otel + +# Install cgc-plugin-xdebug +COPY plugins/cgc-plugin-xdebug/ ./plugins/cgc-plugin-xdebug/ +RUN pip install --no-cache-dir ./plugins/cgc-plugin-xdebug + +# ── Production stage ────────────────────────────────────────────────────────── +FROM python:3.12-slim + +WORKDIR /app + +# Runtime system dependencies (curl required for HEALTHCHECK) +RUN apt-get update && apt-get install -y \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Non-root user +RUN groupadd --gid 1001 cgc && \ + useradd --uid 1001 --gid cgc --shell /bin/sh --create-home cgc + +# Copy installed Python packages from builder +COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages +COPY --from=builder /usr/local/bin/cgc /usr/local/bin/cgc +COPY --from=builder /usr/local/bin/codegraphcontext /usr/local/bin/codegraphcontext + +# Copy CGC core source +COPY --from=builder /app/src /app/src + +# Copy plugin sources (entry-point discovery requires the installed packages; +# source copies allow live plugin inspection if needed) +COPY --from=builder /app/plugins/cgc-plugin-otel /app/plugins/cgc-plugin-otel +COPY --from=builder /app/plugins/cgc-plugin-xdebug /app/plugins/cgc-plugin-xdebug + +# Directories owned by the non-root user +RUN mkdir -p /workspace /home/cgc/.codegraphcontext && \ + chown -R cgc:cgc /workspace /home/cgc/.codegraphcontext + +# Runtime environment — no secrets here +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV CGC_HOME=/home/cgc/.codegraphcontext + +# MCP HTTP server port +EXPOSE 8045 + +WORKDIR /workspace + +USER cgc + +# Health check via the /healthz HTTP endpoint exposed by the MCP server +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD curl -f http://localhost:8045/healthz || exit 1 + +# Start the MCP server over HTTP transport +CMD ["cgc", "mcp", "start", "--transport", "http"] diff --git a/cgc-extended-spec.md b/cgc-extended-spec.md new file mode 100644 index 00000000..26865ffd --- /dev/null +++ b/cgc-extended-spec.md @@ -0,0 +1,684 @@ +# CodeGraphContext-Extended (CGC-X) +## Requirements, Specification & Development Plan + +--- + +## 1. Project Overview + +**CodeGraphContext-Extended (CGC-X)** builds on top of the existing [CodeGraphContext](https://github.com/CodeGraphContext/CodeGraphContext) project, extending it with two additional data ingestion pipelines and bundling all components into a single, cohesive Docker Compose deployment. The result is a unified Neo4j knowledge graph that combines three complementary layers of understanding about a codebase. + +### The Three Layers + +| Layer | Source | What It Tells You | +|---|---|---| +| **Static** | CGC (existing) | Code structure — classes, methods, relationships as written | +| **Runtime** | OTEL + Xdebug (new) | Execution reality — what actually runs, how, across services | + +### Guiding Principles + +- **Same Neo4j instance** — both layers share one database, enabling cross-layer queries +- **Non-invasive** — no required changes to target applications beyond standard OTEL instrumentation +- **Composable** — each service is independently useful; the value multiplies when combined +- **Homelab-friendly** — runs behind a reverse proxy (Traefik), k8s-compatible, self-contained + +--- + +## 2. Repository Structure + +``` +cgc-extended/ +├── docker-compose.yml # Full stack +├── docker-compose.dev.yml # Dev overrides (Xdebug enabled) +├── .env.example +├── README.md +│ +├── services/ +│ ├── otel-processor/ # NEW: OTEL span → Neo4j ingestion +│ │ ├── Dockerfile +│ │ ├── src/ +│ │ │ ├── main.py +│ │ │ ├── span_processor.py +│ │ │ ├── neo4j_writer.py +│ │ │ └── schema.py +│ │ └── requirements.txt +│ │ +│ └── xdebug-listener/ # NEW: DBGp server → Neo4j ingestion +│ ├── Dockerfile +│ ├── src/ +│ │ ├── main.py +│ │ ├── dbgp_server.py +│ │ ├── neo4j_writer.py +│ │ └── schema.py +│ └── requirements.txt +│ +├── config/ +│ ├── otel-collector/ +│ │ └── config.yaml # OTel Collector pipeline config +│ └── neo4j/ +│ └── init.cypher # Schema constraints & indexes +│ +└── docs/ + ├── neo4j-schema.md + ├── laravel-setup.md + └── traefik-setup.md +``` + +--- + +## 3. Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ Target Applications │ +│ │ +│ Laravel App A Laravel App B │ +│ (OTEL SDK) (OTEL SDK + Xdebug) │ +└──────┬───────────────────────┬──────────────────────┘ + │ OTLP (gRPC/HTTP) │ OTLP + DBGp (9003) + ▼ │ +┌──────────────┐ │ +│ OTel │ │ +│ Collector │ │ +└──────┬───────┘ │ + │ OTLP (forwarded) │ + ▼ ▼ +┌────────────────────────────────────────────────────┐ +│ CGC-Extended Stack │ +│ │ +│ ┌─────────────────┐ ┌──────────────────────┐ │ +│ │ otel-processor │ │ xdebug-listener │ │ +│ │ (Python) │ │ (Python, port 9003) │ │ +│ └────────┬────────┘ └──────────┬───────────┘ │ +│ │ │ │ +│ ┌────────▼────────────────────────▼───────────┐ │ +│ │ Neo4j │ │ +│ │ (shared with CGC static nodes) │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ │ +│ ┌────────▼────────┐ │ +│ │ CodeGraphCtx │ │ +│ │ (CGC, static) │ │ +│ └─────────────────┘ │ +└────────────────────────────────────────────────────┘ + │ + ▼ + Traefik (reverse proxy) + → cgc-x.your-domain.com/mcp +``` + +--- + +## 4. Neo4j Unified Schema + +All nodes carry a `source` property that identifies their origin. This is the key to cross-layer querying. + +### Node Labels + +```cypher +// ── STATIC LAYER (CGC existing) ────────────────────────── +(:File { path, language, repo, indexed_at }) +(:Class { name, fqn, file_path, source: 'static' }) +(:Method { name, fqn, file_path, line, source: 'static' }) +(:Function { name, fqn, file_path, line, source: 'static' }) +(:Interface { name, fqn, source: 'static' }) + +// ── RUNTIME LAYER (OTEL) ───────────────────────────────── +(:Service { name, version, environment }) +(:Trace { trace_id, root_span_id, started_at, duration_ms }) +(:Span { + span_id, + trace_id, + name, + service, + kind, // SERVER, CLIENT, INTERNAL, PRODUCER, CONSUMER + class_name, // extracted from span attributes + method_name, // extracted from span attributes + http_method, // for HTTP spans + http_route, // for HTTP spans + db_statement, // for DB spans + duration_ms, + status, + source: 'runtime_otel' +}) + +// ── RUNTIME LAYER (Xdebug) ─────────────────────────────── +(:StackFrame { + class_name, + method_name, + fqn, + file_path, + line, + depth, + source: 'runtime_xdebug' +}) + +``` + +### Relationship Types + +```cypher +// Static +(Method)-[:BELONGS_TO]->(Class) +(Class)-[:IMPLEMENTS]->(Interface) +(Class)-[:EXTENDS]->(Class) +(Method)-[:CALLS]->(Method) +(File)-[:CONTAINS]->(Class) + +// Runtime — OTEL +(Span)-[:CHILD_OF]->(Span) +(Span)-[:PART_OF]->(Trace) +(Trace)-[:ORIGINATED_FROM]->(Service) +(Span)-[:CALLS_SERVICE]->(Service) // cross-service edges + +// Runtime — Xdebug +(StackFrame)-[:CALLED_BY]->(StackFrame) +(StackFrame)-[:RESOLVES_TO]->(Method) // ← links to static layer + +// Cross-layer correlation +(Span)-[:CORRELATES_TO]->(Method) // OTEL span → static method node +``` + +### Indexes & Constraints + +```cypher +-- init.cypher +CREATE CONSTRAINT class_fqn IF NOT EXISTS + FOR (c:Class) REQUIRE c.fqn IS UNIQUE; + +CREATE CONSTRAINT method_fqn IF NOT EXISTS + FOR (m:Method) REQUIRE m.fqn IS UNIQUE; + +CREATE CONSTRAINT span_id IF NOT EXISTS + FOR (s:Span) REQUIRE s.span_id IS UNIQUE; + +CREATE INDEX span_trace IF NOT EXISTS + FOR (s:Span) ON (s.trace_id); + +CREATE INDEX span_class IF NOT EXISTS + FOR (s:Span) ON (s.class_name); +``` + +--- + +## 5. Service Specifications + +### 5.1 OTEL Collector (config only, standard image) + +No custom code — use `otel/opentelemetry-collector-contrib`. + +```yaml +# config/otel-collector/config.yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 5s + send_batch_size: 512 + filter/drop_health: # drop noisy health check spans + spans: + exclude: + match_type: strict + attributes: + - key: http.route + value: /health + +exporters: + otlp/processor: # forward to your otel-processor + endpoint: otel-processor:5317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch, filter/drop_health] + exporters: [otlp/processor] +``` + +**Why a collector?** Decouples the app from your processor. Handles batching, retries, and filtering before spans hit Neo4j. Standard practice — apps just point `OTEL_EXPORTER_OTLP_ENDPOINT` at the collector. + +--- + +### 5.2 OTEL Processor Service + +**Language:** Python +**Base image:** `python:3.12-slim` +**Port:** 5317 (OTLP gRPC, internal only) + +#### Responsibilities + +1. Receive spans from the OTel Collector via OTLP +2. Extract structured data (service name, class, method, HTTP route, DB queries) +3. Upsert nodes and relationships into Neo4j +4. Attempt correlation with CGC static nodes by matching `fqn` + +#### Key Extraction Logic + +Laravel/PHP OTEL spans carry attributes you can parse: + +```python +# span_processor.py + +def extract_php_context(span) -> dict: + attrs = span.attributes or {} + + # Laravel auto-instrumentation sets these + code_namespace = attrs.get('code.namespace', '') # e.g. App\Http\Controllers\OrderController + code_function = attrs.get('code.function', '') # e.g. store + http_route = attrs.get('http.route', '') # e.g. /api/orders + db_statement = attrs.get('db.statement', '') + db_system = attrs.get('db.system', '') + + fqn = f"{code_namespace}::{code_function}" if code_namespace else None + + return { + 'class_name': code_namespace, + 'method_name': code_function, + 'fqn': fqn, + 'http_route': http_route, + 'db_statement': db_statement, + 'db_system': db_system, + } + +def correlate_to_static(tx, span_id: str, fqn: str): + """ + If CGC has already indexed a Method node with this fqn, + draw a CORRELATES_TO edge from the Span to that Method. + """ + tx.run(""" + MATCH (s:Span {span_id: $span_id}) + MATCH (m:Method {fqn: $fqn}) + MERGE (s)-[:CORRELATES_TO]->(m) + """, span_id=span_id, fqn=fqn) +``` + +#### Cross-Service Edge Detection + +When a span has `kind = CLIENT` and `http.url` or `peer.service` set, create a `CALLS_SERVICE` relationship — this is your cross-project graph edge. + +```python +def handle_cross_service(tx, span, context): + if span.kind == SpanKind.CLIENT: + peer = span.attributes.get('peer.service') or \ + extract_host(span.attributes.get('http.url', '')) + if peer: + tx.run(""" + MERGE (target:Service {name: $peer}) + WITH target + MATCH (s:Span {span_id: $span_id}) + MERGE (s)-[:CALLS_SERVICE]->(target) + """, peer=peer, span_id=span.span_id) +``` + +--- + +### 5.3 Xdebug Listener Service + +**Language:** Python +**Base image:** `python:3.12-slim` +**Port:** 9003 (DBGp, exposed — target dev apps connect to this) +**When to run:** Dev/staging only (excluded from production compose) + +#### Responsibilities + +1. Run a DBGp TCP server on port 9003 +2. Accept Xdebug connections from PHP applications +3. Walk stack frames on each breakpoint/trace event +4. Upsert `StackFrame` nodes and `CALLED_BY` edges +5. Attempt `RESOLVES_TO` correlation to CGC `Method` nodes + +#### DBGp Protocol Basics + +``` +PHP (Xdebug client) ──connects to──> DBGp Server (your listener) + +Key commands: + run → continue execution + stack_get → get current call stack (all frames) + context_get → get variables at a given depth +``` + +#### Recommended Library + +Use `python-dbgp` or implement a minimal DBGp server — the protocol is XML over TCP and straightforward: + +```python +# dbgp_server.py (simplified) +import socket, xml.etree.ElementTree as ET + +class DBGpServer: + def handle_connection(self, conn): + # 1. Receive init packet from Xdebug + init = self.recv_packet(conn) + + # 2. Send `run` to let execution proceed to next breakpoint + self.send_cmd(conn, 'run') + + # 3. On each stop, fetch the full stack + while True: + response = self.recv_packet(conn) + if response is None: + break + + self.send_cmd(conn, 'stack_get -i 1') + stack_xml = self.recv_packet(conn) + frames = self.parse_stack(stack_xml) + + self.write_to_neo4j(frames) + self.send_cmd(conn, 'run') + + def parse_stack(self, xml_str) -> list[dict]: + root = ET.fromstring(xml_str) + frames = [] + for stack in root.findall('stack'): + frames.append({ + 'class': stack.get('classname', ''), + 'method': stack.get('where', ''), + 'file': stack.get('filename', ''), + 'line': int(stack.get('lineno', 0)), + 'depth': int(stack.get('level', 0)), + }) + return frames +``` + +#### Deduplication Strategy + +The same call chain will repeat across thousands of requests. Use a hash of the call chain to deduplicate: + +```python +import hashlib + +def chain_hash(frames: list[dict]) -> str: + key = '|'.join(f"{f['class']}::{f['method']}" for f in frames) + return hashlib.sha256(key.encode()).hexdigest()[:16] + +# In neo4j_writer: only upsert if hash not seen recently +# Keep a local LRU cache of recent chain hashes to avoid Neo4j round-trips +``` + +--- + +## 6. Docker Compose + +```yaml +# docker-compose.yml +services: + + neo4j: + image: neo4j:5 + container_name: cgc-neo4j + restart: unless-stopped + environment: + NEO4J_AUTH: neo4j/${NEO4J_PASSWORD} + NEO4J_PLUGINS: '["apoc"]' + NEO4J_dbms_memory_heap_max__size: 2G + volumes: + - neo4j_data:/data + - ./config/neo4j/init.cypher:/var/lib/neo4j/import/init.cypher + ports: + - "7687:7687" # Bolt (internal use) + - "7474:7474" # Browser (optional, disable in prod) + healthcheck: + test: ["CMD", "neo4j", "status"] + interval: 30s + timeout: 10s + retries: 5 + + codegraphcontext: + image: codegraphcontext/codegraphcontext:latest # or build from source + container_name: cgc-static + restart: unless-stopped + environment: + NEO4J_URI: bolt://neo4j:7687 + NEO4J_USERNAME: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + depends_on: + neo4j: + condition: service_healthy + labels: + - "traefik.enable=true" + - "traefik.http.routers.cgc.rule=Host(`cgc.${DOMAIN}`)" + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: cgc-otel-collector + restart: unless-stopped + volumes: + - ./config/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - "4317:4317" # OTLP gRPC (apps send here) + - "4318:4318" # OTLP HTTP (apps send here) + depends_on: + - otel-processor + + otel-processor: + build: ./services/otel-processor + container_name: cgc-otel-processor + restart: unless-stopped + environment: + NEO4J_URI: bolt://neo4j:7687 + NEO4J_USERNAME: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + LISTEN_PORT: 5317 + LOG_LEVEL: INFO + depends_on: + neo4j: + condition: service_healthy + +volumes: + neo4j_data: +``` + +```yaml +# docker-compose.dev.yml (override for development) +services: + xdebug-listener: + build: ./services/xdebug-listener + container_name: cgc-xdebug + restart: unless-stopped + environment: + NEO4J_URI: bolt://neo4j:7687 + NEO4J_USERNAME: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + LISTEN_HOST: 0.0.0.0 + LISTEN_PORT: 9003 + DEDUP_CACHE_SIZE: 10000 + LOG_LEVEL: DEBUG + ports: + - "9003:9003" # DBGp — PHP apps connect to this + depends_on: + neo4j: + condition: service_healthy +``` + +--- + +## 7. Laravel Application Setup + +### OTEL Instrumentation (Production + Dev) + +```bash +composer require \ + open-telemetry/sdk \ + open-telemetry/exporter-otlp \ + open-telemetry/opentelemetry-auto-laravel \ + open-telemetry/opentelemetry-auto-psr18 +``` + +Add to `.env`: +```ini +OTEL_PHP_AUTOLOAD_ENABLED=true +OTEL_SERVICE_NAME=your-service-name +OTEL_TRACES_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +OTEL_EXPORTER_OTLP_ENDPOINT=http://cgc-otel-collector:4317 +OTEL_PROPAGATORS=tracecontext,baggage +OTEL_TRACES_SAMPLER=parentbased_traceidratio +OTEL_TRACES_SAMPLER_ARG=1.0 # 1.0 = 100% in dev, lower in prod +``` + +Add to `Dockerfile`: +```dockerfile +RUN pecl install opentelemetry +RUN echo "extension=opentelemetry.so" >> /usr/local/etc/php/conf.d/opentelemetry.ini +``` + +### Xdebug (Dev only) + +```dockerfile +# dev.Dockerfile or override +RUN pecl install xdebug +``` + +```ini +; xdebug.ini +xdebug.mode=debug,trace +xdebug.client_host=cgc-xdebug ; container name in same Docker network +xdebug.client_port=9003 +xdebug.start_with_request=trigger ; use XDEBUG_TRIGGER header/cookie +; or: xdebug.start_with_request=yes for all requests (noisy) +``` + +**Recommended:** use `trigger` mode. Set the `XDEBUG_TRIGGER` cookie in your browser to selectively capture traces rather than flooding Neo4j on every request. + +--- + +## 8. Development Phases + +### Phase 1 — Foundation (Week 1–2) + +Goal: Neo4j running, CGC indexing, schema in place. + +- [ ] Set up repository structure +- [ ] Write `config/neo4j/init.cypher` with constraints and indexes +- [ ] Wire up `docker-compose.yml` with Neo4j + CGC +- [ ] Verify CGC indexes a Laravel project into Neo4j +- [ ] Set up Traefik labels and confirm MCP endpoint is accessible +- [ ] Write `docs/neo4j-schema.md` as living document + +**Success criterion:** AI assistant can query static code nodes in Neo4j. + +--- + +### Phase 2 — OTEL Processor (Week 2–3) + +Goal: Laravel spans flowing into Neo4j, basic cross-layer correlation working. + +- [ ] Scaffold `services/otel-processor/` — Python OTLP receiver +- [ ] Implement span → Neo4j upsert for `Span`, `Trace`, `Service` nodes +- [ ] Implement `CHILD_OF` relationship from `parent_span_id` +- [ ] Implement PHP attribute extraction (`code.namespace`, `code.function`) +- [ ] Implement `CORRELATES_TO` correlation against existing CGC `Method` nodes +- [ ] Implement cross-service edge detection (`SpanKind.CLIENT`) +- [ ] Wire `otel-collector` → `otel-processor` in compose +- [ ] Test with a real Laravel app: instrument, send request, verify nodes appear +- [ ] Write `docs/laravel-setup.md` + +**Success criterion:** A single HTTP request to the Laravel app produces a complete span tree in Neo4j, with at least some spans connected to static Method nodes. + +--- + +### Phase 3 — Xdebug Listener (Week 3–4) + +Goal: Dev-time method-level traces captured and linked to static nodes. + +- [ ] Scaffold `services/xdebug-listener/` — Python DBGp server +- [ ] Implement TCP server, DBGp handshake, `stack_get` command +- [ ] Implement stack frame parsing and `StackFrame` node upsert +- [ ] Implement `CALLED_BY` chain from frame depth +- [ ] Implement call chain deduplication (hash + LRU cache) +- [ ] Implement `RESOLVES_TO` correlation to CGC `Method` nodes by `fqn` +- [ ] Wire into `docker-compose.dev.yml` +- [ ] Test: trigger Xdebug on a Laravel request, verify frame graph in Neo4j + +**Success criterion:** Xdebug trace for a request shows container-resolved classes (e.g., concrete repository implementation rather than interface) connected to the static graph. + +--- + +### Phase 4 — Cross-Layer Queries & MCP Tooling (Week 4–5) + +Goal: The unified graph is queryable in useful ways from an AI assistant. + +**Example queries to validate and document:** + +```cypher +-- "Show me everything that executes when POST /api/orders is called" +MATCH (s:Span {http_route: '/api/orders', http_method: 'POST'}) +MATCH (s)-[:CHILD_OF*1..10]->(child:Span) +OPTIONAL MATCH (child)-[:CORRELATES_TO]->(m:Method) +RETURN s, child, m + +-- "Show cross-service call chains" +MATCH (svc1:Service)-[:ORIGINATED_FROM]-(t:Trace)-[:PART_OF]-(s:Span) +MATCH (s)-[:CALLS_SERVICE]->(svc2:Service) +RETURN svc1.name, svc2.name, count(*) as call_count +ORDER BY call_count DESC + +-- "What static code is never observed at runtime?" +MATCH (m:Method) +WHERE NOT EXISTS { MATCH (m)<-[:CORRELATES_TO]-(:Span) } + AND NOT EXISTS { MATCH (m)<-[:RESOLVES_TO]-(:StackFrame) } +RETURN m.fqn, m.class_name +ORDER BY m.class_name +``` + +- [ ] Write and test the above canonical queries +- [ ] Document queries in `docs/neo4j-schema.md` +- [ ] Consider a thin MCP wrapper exposing these as named tools (optional) + +--- + +### Phase 5 — Polish & Release (Week 5–6) + +- [ ] Write comprehensive `README.md` with architecture diagram +- [ ] Create `.env.example` with all required variables documented +- [ ] Add `CONTRIBUTING.md` with credit to upstream CGC project +- [ ] Add health check endpoints to both custom services +- [ ] Test full stack teardown and restart (data persistence) +- [ ] Test k8s manifests (port from existing homelab patterns) +- [ ] Tag v0.1.0 + +--- + +## 9. Environment Variables Reference + +```ini +# .env.example + +# Neo4j +NEO4J_PASSWORD=changeme +DOMAIN=yourdomain.local + +# OTEL Processor +OTEL_PROCESSOR_LOG_LEVEL=INFO +OTEL_PROCESSOR_BATCH_SIZE=100 +OTEL_PROCESSOR_FLUSH_INTERVAL=5 + +# Xdebug Listener (dev only) +XDEBUG_DEDUP_CACHE_SIZE=10000 +XDEBUG_MAX_DEPTH=20 # max stack depth to capture + +``` + +--- + +## 10. Key Design Decisions + +**Why Python for otel-processor and xdebug-listener?** +The `opentelemetry-sdk` Python package has excellent OTLP receiver support and Neo4j's official `neo4j` Python driver is the most mature. Keeps both services consistent. + +**Why same Neo4j database (not separate databases)?** +Cross-layer queries require traversing between node types. If CGC static nodes and OTEL span nodes are in different databases, you cannot do `MATCH (s:Span)-[:CORRELATES_TO]->(m:Method)` in a single query. The unified schema with `source` property labels is sufficient to distinguish origins. + +**Why the OTel Collector in between?** +Direct OTLP from app → otel-processor works but is fragile. The collector handles batching, retry on failure, and gives you a place to add sampling rules or additional exporters (e.g., Jaeger for visual trace inspection) without touching application config. + +**Xdebug `trigger` mode rather than `yes` mode?** +`yes` mode captures every request, generating massive graph noise and degrading performance. `trigger` mode lets you selectively capture specific requests using the `XDEBUG_TRIGGER` cookie/header, giving you targeted, high-quality traces. diff --git a/cgc.spec b/cgc.spec index 8b2a8473..a83a93a3 100644 --- a/cgc.spec +++ b/cgc.spec @@ -5,7 +5,7 @@ import sys import os from pathlib import Path -from PyInstaller.utils.hooks import collect_data_files, collect_submodules, collect_all +from PyInstaller.utils.hooks import collect_data_files, collect_submodules, collect_all, collect_entry_point block_cipher = None @@ -138,14 +138,47 @@ hidden_imports = [ 'httpx', 'httpcore', 'importlib', + 'importlib.metadata', + 'importlib.metadata._meta', + 'importlib.metadata._adapters', + 'importlib.metadata._itertools', + 'importlib.metadata._functools', + 'importlib.metadata._text', 'asyncio', 'pkg_resources', + 'pkg_resources.extern', 'threading', 'subprocess', 'socket', 'atexit', + # plugin_registry.py discovers plugins via importlib.metadata.entry_points(); + # each installed plugin's distribution metadata must be bundled so that + # entry_points(group=...) resolves correctly in a frozen executable. + 'codegraphcontext.plugin_registry', ] +# ── Plugin entry-point metadata collection ──────────────────────────────── +# PyInstaller cannot discover entry points at freeze time unless the +# distribution METADATA / entry_points.txt files are explicitly copied into +# the bundle. collect_entry_point() returns (datas, hidden_imports) for +# every distribution that declares the requested group. +_plugin_ep_groups = ['cgc_cli_plugins', 'cgc_mcp_plugins'] +for _ep_group in _plugin_ep_groups: + try: + _ep_datas, _ep_hidden = collect_entry_point(_ep_group) + datas += _ep_datas + hidden_imports += _ep_hidden + except Exception as _ep_exc: + print(f"Warning: collect_entry_point('{_ep_group}') failed: {_ep_exc}") + +# Bundle the codegraphcontext distribution metadata so that +# importlib.metadata.version("codegraphcontext") resolves in the frozen binary +# and PluginRegistry._get_cgc_version() returns the correct version string. +try: + datas += collect_data_files('codegraphcontext', includes=['**/*.dist-info/**/*']) +except Exception as _cgc_meta_exc: + print(f"Warning: collect_data_files('codegraphcontext') failed: {_cgc_meta_exc}") + # Bin extensions by platform ext = '*.so' @@ -226,6 +259,44 @@ if not is_win: except Exception as e: print(f"Warning: collect_all failed for {pkg}: {e}") +# ── falkordblite: explicit auditwheel-vendored shared library collection ────── +# The Linux manylinux wheel ships shared libraries in a `falkordblite.libs/` +# directory (placed by auditwheel alongside the importable packages). This +# directory is NOT a Python package (no __init__.py), so collect_all/ +# collect_dynamic_libs operating on top-level package names ('dummy', +# 'redislite') will not find it. We must explicitly scan for it and register +# every .so* in it as a binary so the dynamic linker can resolve libcrypto, +# libssl, and libgomp at runtime inside the frozen one-file executable. +# +# On macOS the equivalent vendored dylibs live in redislite/.dylibs/ and are +# already captured by collect_all('falkordblite') above. On Windows the +# package is not installed (sys_platform != 'win32' marker), so this block +# is also guarded. +if not is_win: + for _sp in search_paths: + _fdb_libs = _sp / 'falkordblite.libs' + if _fdb_libs.exists(): + for _lib in _fdb_libs.iterdir(): + if _lib.is_file() and not _lib.suffix == '.py': + print(f"Bundling falkordblite.libs: {_lib}") + binaries.append((str(_lib), 'falkordblite.libs')) + break # found; no need to check further paths + +# falkordblite ships a top-level 'dummy' C extension (dummy.cpython-*.so). +# It is not an application import but must travel with the bundle as a +# sentinel that pip/auditwheel attaches native build metadata to. +# Collect it explicitly so PyInstaller does not silently drop it. +if not is_win: + # 'dummy' may resolve to the stdlib dummy module; guard by only picking up + # the .so file that lives directly in a site-packages root (where auditwheel + # places it) rather than in any sub-package directory. + for _sp in search_paths: + for _dummy_so in _sp.glob('dummy.cpython-*.so'): + if _dummy_so.is_file(): + print(f"Bundling falkordblite dummy extension: {_dummy_so}") + binaries.append((str(_dummy_so), '.')) + break + # stdlibs: dynamically imports py3.py, py312.py, etc. via importlib stdlibs_dir = find_pkg_dir('stdlibs') if stdlibs_dir: @@ -250,6 +321,10 @@ if tslp_dir: # Add redislite submodules to hidden imports hidden_imports += collect_submodules('redislite') hidden_imports += collect_submodules('falkordb') +# falkordblite's top_level.txt declares 'dummy' and 'redislite'. +# 'redislite' is covered above; add 'dummy' explicitly. +if not is_win: + hidden_imports.append('dummy') # Add platform-specific watchers if is_win: @@ -267,9 +342,9 @@ a = Analysis( binaries=binaries, datas=datas, hiddenimports=hidden_imports, - hookspath=[], + hookspath=['pyinstaller_hooks'], hooksconfig={}, - runtime_hooks=[], + runtime_hooks=['pyinstaller_hooks/rthook_importlib_metadata.py'], excludes=[ 'tkinter', '_tkinter', 'matplotlib', 'numpy', 'pandas', 'scipy', 'PIL', 'cv2', 'torch', 'tensorflow', 'jupyter', 'notebook', 'IPython', diff --git a/config/neo4j/init.cypher b/config/neo4j/init.cypher new file mode 100644 index 00000000..3908046b --- /dev/null +++ b/config/neo4j/init.cypher @@ -0,0 +1,31 @@ +// CGC Plugin Extension — Graph Schema Initialization +// Run this against Neo4j after startup to create all constraints and indexes. +// Idempotent: uses IF NOT EXISTS throughout. + +// ── OTEL Plugin: Service nodes ───────────────────────────────────────────── +CREATE CONSTRAINT service_name IF NOT EXISTS + FOR (s:Service) REQUIRE s.name IS UNIQUE; + +// ── OTEL Plugin: Trace nodes ─────────────────────────────────────────────── +CREATE CONSTRAINT trace_id IF NOT EXISTS + FOR (t:Trace) REQUIRE t.trace_id IS UNIQUE; + +// ── OTEL Plugin: Span nodes ──────────────────────────────────────────────── +CREATE CONSTRAINT span_id IF NOT EXISTS + FOR (s:Span) REQUIRE s.span_id IS UNIQUE; + +CREATE INDEX span_trace IF NOT EXISTS + FOR (s:Span) ON (s.trace_id); + +CREATE INDEX span_class IF NOT EXISTS + FOR (s:Span) ON (s.class_name); + +CREATE INDEX span_route IF NOT EXISTS + FOR (s:Span) ON (s.http_route); + +// ── Xdebug Plugin: StackFrame nodes ─────────────────────────────────────── +CREATE CONSTRAINT frame_id IF NOT EXISTS + FOR (sf:StackFrame) REQUIRE sf.frame_id IS UNIQUE; + +CREATE INDEX frame_fqn IF NOT EXISTS + FOR (sf:StackFrame) ON (sf.fqn); diff --git a/config/otel-collector/config.yaml b/config/otel-collector/config.yaml new file mode 100644 index 00000000..52dcdab2 --- /dev/null +++ b/config/otel-collector/config.yaml @@ -0,0 +1,33 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 5s + send_batch_size: 512 + + filter/drop_noise: + error_mode: ignore + traces: + span: + - 'attributes["http.route"] == "/health"' + - 'attributes["http.route"] == "/metrics"' + - 'attributes["http.route"] == "/ping"' + +exporters: + otlp/cgc_processor: + endpoint: cgc-otel-processor:5317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + processors: [filter/drop_noise, batch] + exporters: [otlp/cgc_processor] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..a8664f3e --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,38 @@ +# Development overlay — adds Xdebug DBGp listener service. +# +# Usage (with plugin stack — recommended): +# docker compose -f docker-compose.plugin-stack.yml -f docker-compose.dev.yml up -d +# +# Usage (with core template + neo4j profile): +# docker compose -f docker-compose.template.yml --profile neo4j -f docker-compose.dev.yml up -d +# +# IMPORTANT: The Xdebug listener only starts when CGC_PLUGIN_XDEBUG_ENABLED=true. +# IMPORTANT: Requires neo4j service with a healthcheck (provided by docker-compose.plugin-stack.yml). + +version: '3.8' + +services: + + # ── CGC Xdebug DBGp listener ────────────────────────────────────────────── + xdebug-listener: + build: + context: plugins/cgc-plugin-xdebug + dockerfile: Dockerfile + container_name: cgc-xdebug-listener + environment: + - NEO4J_URI=${NEO4J_URI:-bolt://neo4j:7687} + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - XDEBUG_LISTEN_HOST=${XDEBUG_LISTEN_HOST:-0.0.0.0} + - XDEBUG_LISTEN_PORT=${XDEBUG_LISTEN_PORT:-9003} + - XDEBUG_DEDUP_CACHE_SIZE=${XDEBUG_DEDUP_CACHE_SIZE:-10000} + - CGC_PLUGIN_XDEBUG_ENABLED=true + - LOG_LEVEL=${LOG_LEVEL:-DEBUG} + ports: + - "9003:9003" + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped diff --git a/docker-compose.plugin-stack.yml b/docker-compose.plugin-stack.yml new file mode 100644 index 00000000..226a45de --- /dev/null +++ b/docker-compose.plugin-stack.yml @@ -0,0 +1,158 @@ +# Full CGC plugin stack — self-contained for local development and manual testing. +# +# Includes: Neo4j + CGC core + OTEL collector + OTEL processor. +# Add Xdebug listener with: -f docker-compose.dev.yml +# +# Quick start: +# cp .env.example .env # edit NEO4J_PASSWORD at minimum +# docker compose -f docker-compose.plugin-stack.yml up -d +# docker compose -f docker-compose.plugin-stack.yml logs -f +# +# With Xdebug (dev): +# docker compose -f docker-compose.plugin-stack.yml -f docker-compose.dev.yml up -d +# +# Verify: +# docker compose -f docker-compose.plugin-stack.yml ps +# curl -s http://localhost:7474 # Neo4j Browser +# grpcurl -plaintext localhost:4317 list # OTEL gRPC endpoint (needs grpcurl) + +version: '3.8' + +services: + + # ── Neo4j graph database ─────────────────────────────────────────────────── + neo4j: + image: neo4j:2026.02.2 + container_name: cgc-neo4j + ports: + - "7474:7474" # Browser: http://localhost:7474 + - "7687:7687" # Bolt + environment: + - NEO4J_AUTH=${NEO4J_USERNAME:-neo4j}/${NEO4J_PASSWORD:-codegraph123} + - NEO4J_PLUGINS=["apoc"] + - NEO4J_server_memory_heap_max__size=2G + volumes: + - neo4j-data:/data + - neo4j-logs:/logs + - ./config/neo4j/init.cypher:/docker-entrypoint-initdb.d/init.cypher:ro + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:7474 || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 30s + networks: + - cgc-network + restart: unless-stopped + + # ── CGC core MCP server ──────────────────────────────────────────────────── + cgc-core: + build: + context: . + dockerfile: Dockerfile + container_name: cgc-core + environment: + - DATABASE_TYPE=neo4j + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - PYTHONUNBUFFERED=1 + volumes: + - ./:/workspace + - cgc-data:/root/.codegraphcontext + stdin_open: true + tty: true + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + + # ── CGC hosted MCP server (HTTP transport) ──────────────────────────────── + # Serves MCP over HTTP on port 8045 for remote clients (e.g. web IDEs, + # hosted agents). Distinct from cgc-core which uses stdio for local IDEs. + cgc-mcp: + build: + context: . + dockerfile: Dockerfile.mcp + container_name: cgc-mcp + environment: + - DATABASE_TYPE=${DATABASE_TYPE:-neo4j} + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - CGC_CORS_ORIGIN=${CGC_CORS_ORIGIN:-*} + - CGC_MCP_PORT=${CGC_MCP_PORT:-8045} + ports: + - "${CGC_MCP_PORT:-8045}:8045" + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8045/healthz"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 15s + + # ── OpenTelemetry Collector ──────────────────────────────────────────────── + # Receives spans from your application (ports 4317 gRPC, 4318 HTTP), + # filters noise, and forwards to cgc-otel-processor. + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: cgc-otel-collector + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./config/otel-collector/config.yaml:/etc/otelcol/config.yaml:ro + ports: + - "4317:4317" # OTLP gRPC — point your app here + - "4318:4318" # OTLP HTTP — alternative ingestion + depends_on: + - cgc-otel-processor + networks: + - cgc-network + restart: unless-stopped + + # ── CGC OTEL Processor ──────────────────────────────────────────────────── + # Receives filtered spans from collector, writes Service/Trace/Span nodes + # to Neo4j and correlates them to static Method nodes. + cgc-otel-processor: + build: + context: plugins/cgc-plugin-otel + dockerfile: Dockerfile + container_name: cgc-otel-processor + environment: + - DATABASE_TYPE=neo4j + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - OTEL_RECEIVER_PORT=${OTEL_RECEIVER_PORT:-5317} + - OTEL_FILTER_ROUTES=${OTEL_FILTER_ROUTES:-/health,/metrics,/ping} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + ports: + - "5317:5317" # Internal gRPC (collector → processor; not exposed to app) + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "python -c \"import grpc; print('ok')\" || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 15s + +volumes: + neo4j-data: + neo4j-logs: + cgc-data: + +networks: + cgc-network: + driver: bridge diff --git a/docker-compose.plugins.yml b/docker-compose.plugins.yml new file mode 100644 index 00000000..a9f40592 --- /dev/null +++ b/docker-compose.plugins.yml @@ -0,0 +1,60 @@ +# Plugin services overlay — OTEL collector/processor. +# +# NOTE: For local development, prefer docker-compose.plugin-stack.yml which is +# self-contained and includes Neo4j with a healthcheck. +# +# Usage (overlay on plugin-stack — recommended for adding to existing stack): +# # Already included in docker-compose.plugin-stack.yml +# +# Usage (overlay on core template — requires neo4j profile active): +# docker compose -f docker-compose.template.yml --profile neo4j -f docker-compose.plugins.yml up +# +# Prerequisites: +# - neo4j service running with healthcheck (docker-compose.plugin-stack.yml provides this) +# - NEO4J_URI, NEO4J_USERNAME, NEO4J_PASSWORD set in .env +# - DOMAIN set to your ingress domain (e.g. localhost) + +version: '3.8' + +services: + + # ── OpenTelemetry Collector ─────────────────────────────────────────────── + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: cgc-otel-collector + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./config/otel-collector/config.yaml:/etc/otelcol/config.yaml:ro + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + depends_on: + - cgc-otel-processor + networks: + - cgc-network + restart: unless-stopped + + # ── CGC OTEL Processor (receives from collector, writes to Neo4j) ───────── + cgc-otel-processor: + build: + context: plugins/cgc-plugin-otel + dockerfile: Dockerfile + container_name: cgc-otel-processor + environment: + - NEO4J_URI=${NEO4J_URI:-bolt://neo4j:7687} + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - OTEL_RECEIVER_PORT=${OTEL_RECEIVER_PORT:-5317} + - OTEL_FILTER_ROUTES=${OTEL_FILTER_ROUTES:-/health,/metrics,/ping} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + ports: + - "5317:5317" # internal gRPC (collector → processor) + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.otel.rule=Host(`otel.${DOMAIN:-localhost}`)" diff --git a/docker-compose.template.yml b/docker-compose.template.yml index 7c406e3d..a2d4af5a 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -41,24 +41,32 @@ services: - falkordb # Optional: Neo4j database (if you prefer Neo4j over FalkorDB) + # Required when using any CGC plugin (otel, xdebug). neo4j: - image: neo4j:5.15.0 + image: neo4j:2026.02.2 container_name: cgc-neo4j ports: - - "7474:7474" # HTTP + - "7474:7474" # HTTP Browser - "7687:7687" # Bolt environment: - - NEO4J_AUTH=neo4j/codegraph123 + - NEO4J_AUTH=${NEO4J_USERNAME:-neo4j}/${NEO4J_PASSWORD:-codegraph123} - NEO4J_PLUGINS=["apoc"] - NEO4J_dbms_security_procedures_unrestricted=apoc.* - NEO4J_dbms_memory_heap_max__size=2G volumes: - neo4j-data:/data - neo4j-logs:/logs + - ./config/neo4j/init.cypher:/docker-entrypoint-initdb.d/init.cypher:ro + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:7474 || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 30s networks: - cgc-network profiles: - - neo4j # Only start when explicitly requested + - neo4j # Start with: docker compose --profile neo4j up volumes: cgc-data: diff --git a/docs/deployment/MCP_SERVER_HOSTING.md b/docs/deployment/MCP_SERVER_HOSTING.md new file mode 100644 index 00000000..2bb77185 --- /dev/null +++ b/docs/deployment/MCP_SERVER_HOSTING.md @@ -0,0 +1,255 @@ +# Hosted MCP Server Deployment Guide + +The CGC hosted MCP server runs the Model Context Protocol over HTTP transport, +making it accessible to remote AI clients without requiring a local CGC +installation. Use it for shared team infrastructure (one server, many clients) +or when your AI client runs in a cloud environment and cannot use stdio. + +The standard CGC MCP mode (`cgc mcp start`) uses stdio — it is launched as a +child process by the IDE. The hosted server (`cgc mcp start --transport http`) +listens on a port and accepts JSON-RPC requests at `POST /mcp`. The same tools +are available; only the transport layer changes. + +--- + +## Quick Start (Docker Compose + Neo4j) + +The fastest path to a running server: + +```bash +# 1. Clone and enter the repo +git clone https://github.com/your-org/codegraphcontext.git +cd codegraphcontext + +# 2. Create an env file (minimum: set a real password) +cp .env.example .env +# Edit .env: set NEO4J_PASSWORD + +# 3. Start Neo4j + the hosted MCP server +docker compose -f docker-compose.plugin-stack.yml up -d neo4j cgc-mcp + +# 4. Verify the server is healthy +curl http://localhost:8045/healthz +# Expected: {"status":"ok","neo4j":"connected"} +``` + +The MCP endpoint is now available at `http://localhost:8045/mcp`. + +--- + +## Environment Variables + +| Variable | Default | Description | +|---|---|---| +| `DATABASE_TYPE` | `neo4j` | Backend database driver. Only `neo4j` is supported in the current release. | +| `NEO4J_URI` | `bolt://localhost:7687` | Bolt URI for the Neo4j instance. Use the container service name (e.g. `bolt://neo4j:7687`) when running in Docker. | +| `NEO4J_USERNAME` | `neo4j` | Neo4j username. | +| `NEO4J_PASSWORD` | *(required)* | Neo4j password. Always supply at runtime; never bake into an image. | +| `CGC_MCP_PORT` | `8045` | Port the HTTP server listens on. | +| `CGC_CORS_ORIGIN` | `*` | Value for the `Access-Control-Allow-Origin` response header. Set to your specific origin in production (e.g. `https://app.example.com`). | + +--- + +## Deployment Options + +### Docker Standalone + +```bash +docker build -f Dockerfile.mcp -t cgc-mcp:latest . + +docker run -d \ + --name cgc-mcp \ + -e DATABASE_TYPE=neo4j \ + -e NEO4J_URI=bolt://your-neo4j-host:7687 \ + -e NEO4J_USERNAME=neo4j \ + -e NEO4J_PASSWORD=your-password \ + -e CGC_CORS_ORIGIN=https://your-client-origin.com \ + -p 8045:8045 \ + cgc-mcp:latest +``` + +### Docker Compose + +The repository ships `docker-compose.plugin-stack.yml`, which includes the +`cgc-mcp` service wired to Neo4j on the `cgc-network` bridge. To start only +the hosted MCP server and its dependency: + +```bash +docker compose -f docker-compose.plugin-stack.yml up -d neo4j cgc-mcp +``` + +To start the full plugin stack (including the OTEL collector and processor): + +```bash +docker compose -f docker-compose.plugin-stack.yml up -d +``` + +The samples demo stack (`samples/docker-compose.yml`) also includes a +`cgc-mcp` service under the `mcp` profile — see +[Hosted MCP in the Sample Stack](#hosted-mcp-in-the-sample-stack) below. + +### Docker Swarm + +```bash +docker service create \ + --name cgc-mcp \ + --replicas 2 \ + --publish published=8045,target=8045 \ + --env DATABASE_TYPE=neo4j \ + --env NEO4J_URI=bolt://neo4j:7687 \ + --env NEO4J_USERNAME=neo4j \ + --secret cgc_neo4j_password \ + cgc-mcp:latest +``` + +Use Docker secrets (`--secret`) rather than `--env` for the password in Swarm +mode. + +### Kubernetes + +Manifests are in `k8s/cgc-mcp/`: + +```bash +# Apply the ConfigMap, Deployment, and Service +kubectl apply -f k8s/cgc-mcp/ + +# Verify rollout +kubectl rollout status deployment/cgc-mcp +kubectl get svc cgc-mcp +``` + +The Service exposes port 8045. Create an Ingress or use a LoadBalancer service +type to expose it externally. Supply `NEO4J_PASSWORD` via a Kubernetes Secret +rather than a plain ConfigMap value. + +--- + +## Client Configuration + +Any MCP client that supports HTTP transport can connect to `POST /mcp`. + +### Claude Desktop + +Add to `~/Library/Application Support/Claude/claude_desktop_config.json` +(macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows): + +```json +{ + "mcpServers": { + "codegraphcontext": { + "transport": "http", + "url": "http://your-server:8045/mcp" + } + } +} +``` + +Restart Claude Desktop after saving. + +### VS Code / Cursor (MCP Extension) + +In your workspace or user settings: + +```json +{ + "mcp.servers": { + "codegraphcontext": { + "transport": "http", + "url": "http://your-server:8045/mcp" + } + } +} +``` + +### Claude Code + +```bash +claude mcp add codegraphcontext --transport http --url http://your-server:8045/mcp +``` + +Verify it was added: + +```bash +claude mcp list +``` + +### Generic MCP Client + +Point the client at `http://your-server:8045/mcp` using HTTP transport. +Send JSON-RPC 2.0 requests with `Content-Type: application/json`. + +Example — list available tools: + +```bash +curl -s -X POST http://localhost:8045/mcp \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' \ + | jq '.result.tools[].name' +``` + +--- + +## Security + +The server has no application-level authentication. This is intentional: auth +is the responsibility of the reverse proxy in front of it. + +### Nginx — Bearer Token Auth + TLS + +```nginx +server { + listen 443 ssl; + server_name mcp.example.com; + + ssl_certificate /etc/ssl/certs/mcp.crt; + ssl_certificate_key /etc/ssl/private/mcp.key; + + location /mcp { + # Require a shared secret from the client + if ($http_authorization != "Bearer your-shared-secret") { + return 401; + } + proxy_pass http://127.0.0.1:8045; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + location /healthz { + proxy_pass http://127.0.0.1:8045; + } +} +``` + +### Traefik — Forward Auth Middleware + +```yaml +# docker-compose labels on the cgc-mcp service: +labels: + - "traefik.enable=true" + - "traefik.http.routers.cgc-mcp.rule=Host(`mcp.example.com`)" + - "traefik.http.routers.cgc-mcp.tls=true" + - "traefik.http.routers.cgc-mcp.middlewares=cgc-auth" + - "traefik.http.middlewares.cgc-auth.forwardauth.address=http://your-auth-service/verify" + - "traefik.http.services.cgc-mcp.loadbalancer.server.port=8045" +``` + +Set `CGC_CORS_ORIGIN` to the specific client origin rather than `*` whenever +the server is reachable from the public internet. + +--- + +## Health Checks + +`GET /healthz` returns the server's operational status. + +| Condition | HTTP Status | Response body | +|---|---|---| +| Server running, Neo4j reachable | `200 OK` | `{"status":"ok","neo4j":"connected"}` | +| Server running, Neo4j unreachable | `503 Service Unavailable` | `{"status":"degraded","neo4j":"unreachable"}` | + +The Docker image uses this endpoint for its `HEALTHCHECK` directive (every 30s, +3 retries before marking the container unhealthy). The Kubernetes liveness and +readiness probes in `k8s/cgc-mcp/deployment.yaml` use it for the same purpose. + +Monitor `/healthz` from your load balancer or uptime tool to detect Neo4j +connectivity issues early. diff --git a/docs/plugins/authoring-guide.md b/docs/plugins/authoring-guide.md new file mode 100644 index 00000000..f0bb910f --- /dev/null +++ b/docs/plugins/authoring-guide.md @@ -0,0 +1,226 @@ +# Plugin Authoring Guide + +This guide walks through creating a CGC plugin from scratch. +The `plugins/cgc-plugin-stub` directory is the canonical worked example — reference it +throughout. + +For the full contract specification see: +[`specs/001-cgc-plugin-extension/contracts/plugin-interface.md`](../../specs/001-cgc-plugin-extension/contracts/plugin-interface.md) + +--- + +## 1. Package Scaffold + +A CGC plugin is a standard Python package with two entry-point groups. + +``` +plugins/cgc-plugin-/ +├── pyproject.toml +└── src/ + └── cgc_plugin_/ + ├── __init__.py ← PLUGIN_METADATA + re-exports + ├── cli.py ← get_plugin_commands() + └── mcp_tools.py ← get_mcp_tools() + get_mcp_handlers() +``` + +Bootstrap it by copying the stub: + +```bash +cp -r plugins/cgc-plugin-stub plugins/cgc-plugin-myname +# then edit: pyproject.toml, __init__.py, cli.py, mcp_tools.py +``` + +--- + +## 2. `pyproject.toml` + +Minimum required configuration: + +```toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "cgc-plugin-myname" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["typer[all]>=0.9.0"] + +[project.entry-points.cgc_cli_plugins] +myname = "cgc_plugin_myname" + +[project.entry-points.cgc_mcp_plugins] +myname = "cgc_plugin_myname" +``` + +**Key points**: +- Entry point group: `cgc_cli_plugins` — for CLI commands +- Entry point group: `cgc_mcp_plugins` — for MCP tools +- Entry point name (`myname`) becomes the CLI command group name and the registry key +- Both groups must point to the same module for most plugins + +--- + +## 3. `__init__.py` — PLUGIN_METADATA + +```python +PLUGIN_METADATA = { + "name": "cgc-plugin-myname", # must match pyproject.toml name + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", # PEP 440 specifier + "description": "One-line description of what this plugin does", +} +``` + +**Required fields**: `name`, `version`, `cgc_version_constraint`, `description`. +Missing any field causes the plugin to be skipped at startup with a clear warning. + +The `cgc_version_constraint` is checked against the installed `codegraphcontext` version. +Use `">=0.1.0"` for maximum compatibility during early development. + +--- + +## 4. CLI Contract — `cli.py` + +```python +import typer + +myname_app = typer.Typer(help="My plugin commands.") + +@myname_app.command("hello") +def hello(name: str = typer.Option("World", help="Name to greet")): + """Say hello from myname plugin.""" + typer.echo(f"Hello from myname plugin, {name}!") + + +def get_plugin_commands(): + """Return (command_group_name, typer_app) to be registered with CGC.""" + return ("myname", myname_app) +``` + +**Contract**: +- `get_plugin_commands()` must return a `(str, typer.Typer)` tuple +- The string becomes the sub-command group: `cgc myname ` +- Raising an exception in `get_plugin_commands()` quarantines the plugin safely + +--- + +## 5. MCP Contract — `mcp_tools.py` + +```python +def get_mcp_tools(server_context: dict | None = None): + """Return dict of tool_name → MCP tool definition.""" + return { + "myname_hello": { + "name": "myname_hello", + "description": "Say hello from myname plugin", + "inputSchema": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "Name to greet"}, + }, + "required": ["name"], + }, + }, + } + + +def get_mcp_handlers(server_context: dict | None = None): + """Return dict of tool_name → callable handler.""" + db = (server_context or {}).get("db_manager") + + def handle_hello(name: str = "World"): + return {"greeting": f"Hello {name} from myname plugin!"} + + return {"myname_hello": handle_hello} +``` + +**Contract**: +- `get_mcp_tools()` returns `dict[str, ToolDefinition]` +- `get_mcp_handlers()` returns `dict[str, callable]` +- Tool names **must** be prefixed with the plugin name: `_` +- `server_context` carries `{"db_manager": }` when available +- Conflicting tool names: the first plugin to register a name wins + +--- + +## 6. Accessing Neo4j + +If your plugin needs graph access, use the `db_manager` from `server_context`: + +```python +def get_mcp_handlers(server_context=None): + db = (server_context or {}).get("db_manager") + + def handle_query(limit: int = 10): + if db is None: + return {"error": "No database connection available"} + results = db.execute_query( + "MATCH (n:Method) RETURN n.fqn LIMIT $limit", + {"limit": limit} + ) + return {"methods": [r["n.fqn"] for r in results]} + + return {"myname_query": handle_query} +``` + +--- + +## 7. Testing Your Plugin + +Write tests in `tests/unit/plugin/` and `tests/integration/plugin/`. + +```python +# tests/unit/plugin/test_myname_tools.py +from cgc_plugin_myname.mcp_tools import get_mcp_tools, get_mcp_handlers + +def test_tools_defined(): + tools = get_mcp_tools() + assert "myname_hello" in tools + +def test_hello_handler(): + handlers = get_mcp_handlers() + result = handlers["myname_hello"](name="Test") + assert result["greeting"] == "Hello Test from myname plugin!" +``` + +Run tests: +```bash +PYTHONPATH=src pytest tests/unit/plugin/ tests/integration/plugin/ -v +``` + +--- + +## 8. Install and Verify + +```bash +pip install -e plugins/cgc-plugin-myname + +# Verify CLI registration +cgc --help # should show 'myname' group +cgc plugin list # should show cgc-plugin-myname as loaded + +# Verify MCP registration (start MCP server and inspect tools/list) +cgc mcp start +# In MCP client: tools/list → should include myname_hello +``` + +--- + +## 9. Publishing to PyPI + +```bash +cd plugins/cgc-plugin-myname +pip install build +python -m build +pip install twine +twine upload dist/* +``` + +Users then install your plugin with: +```bash +pip install cgc-plugin-myname +``` + +CGC discovers it automatically at next startup — no configuration required. diff --git a/docs/plugins/cross-layer-queries.md b/docs/plugins/cross-layer-queries.md new file mode 100644 index 00000000..80e3f7a7 --- /dev/null +++ b/docs/plugins/cross-layer-queries.md @@ -0,0 +1,114 @@ +# Cross-Layer Cypher Queries + +These canonical queries validate **SC-005** (cross-layer intelligence) by joining +static code analysis nodes (Class, Method) with runtime nodes (Span, StackFrame). + +All queries assume: +- CGC has indexed a PHP/Laravel repository (Method, Class, File nodes exist) +- OTEL or Xdebug plugin has written at least some runtime data + +--- + +## 1. Execution Path for a Route + +Find every method observed at runtime for a given HTTP route, ordered by frequency. + +```cypher +MATCH (s:Span {http_route: "/api/orders"})-[:CORRELATES_TO]->(m:Method) +RETURN + m.fqn AS method, + m.class_name AS class, + count(s) AS executions, + avg(s.duration_ms) AS avg_duration_ms +ORDER BY executions DESC +LIMIT 20 +``` + +**Expected result schema**: + +| Column | Type | Description | +|--------|------|-------------| +| `method` | string | Fully-qualified method name, e.g. `App\Http\Controllers\OrderController::store` | +| `class` | string | Class name | +| `executions` | int | Number of spans that correlated to this method | +| `avg_duration_ms` | float | Average span duration in milliseconds | + +--- + +## 2. Cross-Service Call Chains + +Trace spans that exit the local service boundary (CLIENT kind with `peer.service` set), +showing the full service-to-service call path. + +```cypher +MATCH path = (caller:Span)-[:CALLS_SERVICE]->(callee:Service) +MATCH (caller)-[:PART_OF]->(t:Trace) +MATCH (caller)-[:ORIGINATED_FROM]->(src:Service) +RETURN + src.name AS from_service, + callee.name AS to_service, + caller.name AS span_name, + caller.duration_ms AS duration_ms, + t.trace_id AS trace_id +ORDER BY caller.start_time_ns DESC +LIMIT 25 +``` + +**Expected result schema**: + +| Column | Type | Description | +|--------|------|-------------| +| `from_service` | string | Originating service name | +| `to_service` | string | Called downstream service name | +| `span_name` | string | Name of the CLIENT span | +| `duration_ms` | float | Duration of the outbound call | +| `trace_id` | string | Trace identifier | + +--- + +## 3. Static Code Never Observed at Runtime + +Find Method nodes with no CORRELATES_TO span and no StackFrame. Surfaces dead code +candidates or code paths never triggered in the current environment. + +```cypher +MATCH (m:Method) +WHERE NOT EXISTS { MATCH (m)<-[:CORRELATES_TO]-(:Span) } + AND NOT EXISTS { MATCH (m)<-[:RESOLVES_TO]-(:StackFrame) } + AND m.fqn IS NOT NULL +RETURN + m.fqn AS method, + m.class_name AS class, + m.file_path AS file +ORDER BY m.class_name, m.fqn +LIMIT 50 +``` + +**Expected result schema**: + +| Column | Type | Description | +|--------|------|-------------| +| `method` | string | FQN of method with no observed execution | +| `class` | string | Owning class | +| `file` | string | Source file path | + +--- + +## Running These Queries + +Via CGC CLI: + +```bash +cgc query "MATCH (s:Span {http_route: '/api/orders'})-[:CORRELATES_TO]->(m:Method) RETURN m.fqn, count(s) AS executions ORDER BY executions DESC LIMIT 20" +``` + +Via MCP tool (`otel_cross_layer_query`): + +```json +{ + "tool": "otel_cross_layer_query", + "arguments": {"query_type": "never_observed"} +} +``` + +Via Neo4j Browser: connect to `bolt://localhost:7687` and paste any query above. diff --git a/docs/plugins/examples/send_test_span.py b/docs/plugins/examples/send_test_span.py new file mode 100644 index 00000000..c79fe412 --- /dev/null +++ b/docs/plugins/examples/send_test_span.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +""" +Send a synthetic OTLP span to the CGC OTEL Collector for manual testing. + +Usage: + pip install opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc + python docs/plugins/examples/send_test_span.py + + # Custom endpoint: + OTEL_ENDPOINT=localhost:4317 python docs/plugins/examples/send_test_span.py + +Verifying results in Neo4j Browser (http://localhost:7474): + MATCH (s:Span) RETURN s.name, s.http_route, s.duration_ms LIMIT 10 + MATCH (s:Service) RETURN s.name +""" +import os +import time + +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource + +ENDPOINT = os.environ.get("OTEL_ENDPOINT", "localhost:4317") +SERVICE_NAME = os.environ.get("OTEL_SERVICE_NAME", "cgc-test-service") + + +def send_test_spans(): + resource = Resource.create({"service.name": SERVICE_NAME}) + provider = TracerProvider(resource=resource) + + exporter = OTLPSpanExporter(endpoint=ENDPOINT, insecure=True) + provider.add_span_processor(BatchSpanProcessor(exporter)) + trace.set_tracer_provider(provider) + + tracer = trace.get_tracer("cgc.manual.test") + + print(f"Sending test spans to {ENDPOINT} (service: {SERVICE_NAME})...") + + # Simulate an HTTP request trace with a DB child span + with tracer.start_as_current_span("GET /api/orders") as root_span: + root_span.set_attribute("http.method", "GET") + root_span.set_attribute("http.route", "/api/orders") + root_span.set_attribute("http.status_code", 200) + root_span.set_attribute("code.namespace", "App\\Http\\Controllers") + root_span.set_attribute("code.function", "OrderController::index") + + time.sleep(0.01) # simulate work + + with tracer.start_as_current_span("DB: SELECT orders") as child_span: + child_span.set_attribute("db.system", "mysql") + child_span.set_attribute("db.statement", "SELECT * FROM orders LIMIT 10") + child_span.set_attribute("peer.service", "mysql") + time.sleep(0.005) + + # Simulate a second, different route + with tracer.start_as_current_span("POST /api/orders") as span2: + span2.set_attribute("http.method", "POST") + span2.set_attribute("http.route", "/api/orders") + span2.set_attribute("http.status_code", 201) + span2.set_attribute("code.namespace", "App\\Http\\Controllers") + span2.set_attribute("code.function", "OrderController::store") + time.sleep(0.02) + + # Flush + provider.force_flush() + print("Done. Check Neo4j: MATCH (s:Span) RETURN s.name, s.http_route LIMIT 10") + + +if __name__ == "__main__": + send_test_spans() diff --git a/docs/plugins/manual-testing.md b/docs/plugins/manual-testing.md new file mode 100644 index 00000000..99e342e6 --- /dev/null +++ b/docs/plugins/manual-testing.md @@ -0,0 +1,234 @@ +# Manual Testing Guide — CGC Plugin Stack + +Step-by-step instructions for spinning up the full plugin stack locally and verifying +each plugin works end-to-end. + +--- + +## Prerequisites + +- Docker + Docker Compose v2 (`docker compose version`) +- Python 3.10+ and pip (for CLI testing without Docker) +- `grpcurl` (optional, for OTEL gRPC smoke test — `brew install grpcurl`) +- A PHP application with OpenTelemetry SDK installed (for OTEL live test — optional) + +--- + +## Option A: Docker Stack (Recommended) + +### 1. Start the stack + +```bash +cp .env.example .env +# .env defaults work for local testing — change NEO4J_PASSWORD for anything non-local + +docker compose -f docker-compose.plugin-stack.yml up -d --build +``` + +Watch startup (Neo4j takes ~30s): +```bash +docker compose -f docker-compose.plugin-stack.yml logs -f +``` + +### 2. Verify all services are healthy + +```bash +docker compose -f docker-compose.plugin-stack.yml ps +``` + +Expected: all services show `healthy` or `running`. + +| Service | Port | Check | +|---|---|---| +| neo4j | 7474, 7687 | http://localhost:7474 → Neo4j Browser | +| cgc-otel-processor | 5317 | `docker logs cgc-otel-processor` → no errors | +| otel-collector | 4317, 4318 | `docker logs cgc-otel-collector` → "Everything is ready" | + +### 3. Verify graph schema initialized + +Open http://localhost:7474, login (neo4j / codegraph123), run: + +```cypher +SHOW CONSTRAINTS +``` + +Expected: `service_name`, `trace_id`, `span_id`, `frame_id` constraints present. + +```cypher +SHOW INDEXES +``` + +--- + +## Option B: Python (No Docker) + +Install everything editable in a venv: + +```bash +python -m venv .venv && source .venv/bin/activate +pip install -e . +pip install -e plugins/cgc-plugin-stub +pip install -e plugins/cgc-plugin-otel +pip install -e plugins/cgc-plugin-xdebug +``` + +Verify plugin discovery: +```bash +PYTHONPATH=src cgc plugin list +# Should show all four plugins as "loaded" + +PYTHONPATH=src cgc --help +# Should show: stub, otel, xdebug command groups +``` + +--- + +## Testing Each Plugin + +### Stub Plugin (smoke test — no DB needed) + +```bash +# CLI +PYTHONPATH=src cgc stub hello +# Expected: "Hello from stub plugin" + +PYTHONPATH=src cgc stub hello --name "Alice" +# Expected: "Hello Alice from stub plugin" +``` + +Via pytest (no install needed for mocked path): +```bash +PYTHONPATH=src pytest tests/unit/plugin/test_plugin_registry.py -v +``` + +--- + +### OTEL Plugin + +**Requires**: Neo4j + `cgc-otel-processor` + `otel-collector` running. + +#### Send a synthetic span + +Using `grpcurl` (easiest): +```bash +# Check collector is accepting connections +grpcurl -plaintext localhost:4317 list +# Expected: opentelemetry.proto.collector.trace.v1.TraceService +``` + +Using a Python script: +```bash +python docs/plugins/examples/send_test_span.py +# See docs/plugins/examples/ for this script +``` + +#### Configure a PHP/Laravel app + +Add to your app's `.env`: +```ini +OTEL_PHP_AUTOLOAD_ENABLED=true +OTEL_SERVICE_NAME=my-laravel-app +OTEL_TRACES_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +``` + +Send a request to your app, then query: +```bash +PYTHONPATH=src cgc otel query-spans --route "/api/orders" --limit 5 +PYTHONPATH=src cgc otel list-services +``` + +Verify in Neo4j Browser: +```cypher +MATCH (s:Service) RETURN s.name +MATCH (sp:Span) RETURN sp.name, sp.duration_ms, sp.http_route LIMIT 10 +MATCH (sp:Span)-[:CORRELATES_TO]->(m:Method) RETURN sp.name, m.fqn LIMIT 10 +``` + +--- + +### Xdebug Plugin + +**Requires**: Neo4j + PHP with Xdebug installed. + +Start the listener (Docker): +```bash +docker compose -f docker-compose.plugin-stack.yml -f docker-compose.dev.yml up -d xdebug-listener +docker logs cgc-xdebug-listener -f +# Expected: "DBGp server listening on 0.0.0.0:9003" +``` + +Start the listener (Python): +```bash +CGC_PLUGIN_XDEBUG_ENABLED=true PYTHONPATH=src cgc xdebug start +``` + +Configure PHP (`php.ini` or `.env`): +```ini +xdebug.mode=debug +xdebug.client_host=localhost +xdebug.client_port=9003 +xdebug.start_with_request=trigger +``` + +Trigger a trace by setting the `XDEBUG_TRIGGER` cookie in your browser, then: +```bash +PYTHONPATH=src cgc xdebug list-chains --limit 10 +PYTHONPATH=src cgc xdebug status +``` + +Verify in Neo4j Browser: +```cypher +MATCH (sf:StackFrame) RETURN sf.class_name, sf.method_name, sf.observation_count LIMIT 20 +MATCH (sf:StackFrame)-[:CALLED_BY]->(parent:StackFrame) RETURN sf.method_name, parent.method_name LIMIT 10 +MATCH (sf:StackFrame)-[:RESOLVES_TO]->(m:Method) RETURN sf.method_name, m.fqn LIMIT 10 +``` + +--- + +## Cross-Layer Validation + +After running all plugins with real data, validate the cross-layer queries: + +```bash +# Execution path for a route +PYTHONPATH=src cgc query " +MATCH (s:Span {http_route: '/api/orders'})-[:CORRELATES_TO]->(m:Method) +RETURN m.fqn, count(s) AS executions, avg(s.duration_ms) AS avg_duration_ms +ORDER BY executions DESC LIMIT 10 +" + +# Static code never observed at runtime +PYTHONPATH=src cgc query " +MATCH (m:Method) +WHERE NOT EXISTS { MATCH (m)<-[:CORRELATES_TO]-(:Span) } + AND NOT EXISTS { MATCH (m)<-[:RESOLVES_TO]-(:StackFrame) } +RETURN m.fqn, m.class_name LIMIT 10 +" +``` + +See `docs/plugins/cross-layer-queries.md` for canonical queries. + +--- + +## Teardown + +```bash +# Stop all services +docker compose -f docker-compose.plugin-stack.yml down + +# Remove volumes (clears Neo4j data) +docker compose -f docker-compose.plugin-stack.yml down -v +``` + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `service_healthy` wait times out | Neo4j slow to start | Increase `start_period` in healthcheck or wait longer | +| `cgc plugin list` shows plugin as failed | Plugin not installed | `pip install -e plugins/cgc-plugin-` | +| Spans sent but no graph nodes | Filter routes dropping them | Check `OTEL_FILTER_ROUTES`; default drops `/health` etc. | +| Xdebug not connecting | Wrong `client_host` | Use Docker host IP, not `localhost`, when PHP is in Docker | diff --git a/k8s/cgc-mcp/deployment.yaml b/k8s/cgc-mcp/deployment.yaml new file mode 100644 index 00000000..639c3824 --- /dev/null +++ b/k8s/cgc-mcp/deployment.yaml @@ -0,0 +1,71 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cgc-mcp + labels: + app: cgc-mcp + app.kubernetes.io/part-of: codegraphcontext +spec: + replicas: 1 + selector: + matchLabels: + app: cgc-mcp + template: + metadata: + labels: + app: cgc-mcp + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + containers: + - name: cgc-mcp + image: ghcr.io/codegraphcontext/cgc-mcp:latest + imagePullPolicy: IfNotPresent + ports: + - name: http-mcp + containerPort: 8045 + protocol: TCP + env: + - name: DATABASE_TYPE + valueFrom: + configMapKeyRef: + name: cgc-config + key: DATABASE_TYPE + - name: NEO4J_URI + valueFrom: + configMapKeyRef: + name: cgc-config + key: NEO4J_URI + - name: NEO4J_USERNAME + valueFrom: + configMapKeyRef: + name: cgc-config + key: NEO4J_USERNAME + - name: NEO4J_PASSWORD + valueFrom: + secretKeyRef: + name: cgc-secrets + key: NEO4J_PASSWORD + - name: CGC_MCP_PORT + value: "8045" + readinessProbe: + httpGet: + path: /healthz + port: http-mcp + initialDelaySeconds: 15 + periodSeconds: 15 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: http-mcp + initialDelaySeconds: 30 + periodSeconds: 30 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" diff --git a/k8s/cgc-mcp/service.yaml b/k8s/cgc-mcp/service.yaml new file mode 100644 index 00000000..711ecebf --- /dev/null +++ b/k8s/cgc-mcp/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: cgc-mcp + labels: + app: cgc-mcp + app.kubernetes.io/part-of: codegraphcontext +spec: + type: ClusterIP + selector: + app: cgc-mcp + ports: + - name: http-mcp + port: 8045 + targetPort: http-mcp + protocol: TCP diff --git a/k8s/cgc-plugin-otel/deployment.yaml b/k8s/cgc-plugin-otel/deployment.yaml new file mode 100644 index 00000000..bff4f2c8 --- /dev/null +++ b/k8s/cgc-plugin-otel/deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cgc-otel-processor + labels: + app: cgc-otel-processor + app.kubernetes.io/part-of: codegraphcontext +spec: + replicas: 1 + selector: + matchLabels: + app: cgc-otel-processor + template: + metadata: + labels: + app: cgc-otel-processor + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: cgc-otel-processor + image: ghcr.io/codegraphcontext/cgc-plugin-otel:latest + imagePullPolicy: IfNotPresent + ports: + - name: grpc-receiver + containerPort: 5317 + protocol: TCP + env: + - name: NEO4J_URI + valueFrom: + configMapKeyRef: + name: cgc-config + key: NEO4J_URI + - name: NEO4J_USERNAME + valueFrom: + configMapKeyRef: + name: cgc-config + key: NEO4J_USERNAME + - name: NEO4J_PASSWORD + valueFrom: + secretKeyRef: + name: cgc-secrets + key: NEO4J_PASSWORD + - name: OTEL_RECEIVER_PORT + value: "5317" + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["python", "-c", "import grpc; print('ok')"] + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + livenessProbe: + exec: + command: ["python", "-c", "import grpc; print('ok')"] + initialDelaySeconds: 20 + periodSeconds: 30 + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" diff --git a/k8s/cgc-plugin-otel/service.yaml b/k8s/cgc-plugin-otel/service.yaml new file mode 100644 index 00000000..4bf9d1bf --- /dev/null +++ b/k8s/cgc-plugin-otel/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: cgc-otel-processor + labels: + app: cgc-otel-processor + app.kubernetes.io/part-of: codegraphcontext +spec: + type: ClusterIP + selector: + app: cgc-otel-processor + ports: + - name: grpc-receiver + port: 5317 + targetPort: grpc-receiver + protocol: TCP + - name: http-otlp + port: 4318 + targetPort: 4318 + protocol: TCP diff --git a/k8s/neo4j-deployment.yaml b/k8s/neo4j-deployment.yaml index 873921b4..8f06d0b6 100644 --- a/k8s/neo4j-deployment.yaml +++ b/k8s/neo4j-deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: neo4j - image: neo4j:5.15.0 + image: neo4j:2026.02.2 ports: - containerPort: 7474 name: http diff --git a/plugins/cgc-plugin-otel/Dockerfile b/plugins/cgc-plugin-otel/Dockerfile new file mode 100644 index 00000000..e72d8549 --- /dev/null +++ b/plugins/cgc-plugin-otel/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12-slim + +# Security: run as non-root +RUN groupadd -r cgc && useradd -r -g cgc cgc + +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY src/ ./src/ + +RUN pip install --no-cache-dir -e ".[dev]" 2>/dev/null || pip install --no-cache-dir -e . && \ + pip install --no-cache-dir grpcio>=1.57.0 "opentelemetry-proto>=0.43b0" "opentelemetry-sdk>=1.20.0" \ + "typer[all]>=0.9.0" "neo4j>=5.15.0" || true + +USER cgc + +EXPOSE 5317 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD python -c "import grpc; print('ok')" || exit 1 + +CMD ["python", "-m", "cgc_plugin_otel.receiver"] diff --git a/plugins/cgc-plugin-otel/README.md b/plugins/cgc-plugin-otel/README.md new file mode 100644 index 00000000..b0b381a6 --- /dev/null +++ b/plugins/cgc-plugin-otel/README.md @@ -0,0 +1,45 @@ +# cgc-plugin-otel + +OpenTelemetry span processor plugin for [CodeGraphContext](https://github.com/CodeGraphContext/CodeGraphContext). + +## Overview + +This plugin ingests OpenTelemetry spans from PHP services (e.g. Laravel) via a gRPC +OTLP receiver and writes them into the CGC Neo4j graph. Spans are correlated with +code graph nodes (classes, methods) using `CORRELATES_TO` relationships, enabling +cross-layer queries that link runtime traces to static code structure. + +## Features + +- gRPC OTLP receiver listening on port 5317 +- Extracts PHP context from OTel span attributes (`code.namespace`, `code.function`, etc.) +- Writes `Service`, `Trace`, and `Span` nodes to Neo4j +- Creates `PART_OF`, `CHILD_OF`, `CORRELATES_TO`, and `CALLS_SERVICE` relationships +- Dead-letter queue (DLQ) for spans that fail to persist +- Exposes a `otel` CLI command group and MCP tools prefixed with `otel_` + +## Requirements + +- Python 3.10+ +- CodeGraphContext >= 0.3.0 +- Neo4j >= 5.15 +- grpcio >= 1.57 +- opentelemetry-sdk >= 1.20 + +## Installation + +```bash +pip install -e plugins/cgc-plugin-otel +``` + +## MCP tool naming + +All MCP tools contributed by this plugin are prefixed with `otel_` +(e.g. `otel_query_spans`). + +## Entry points + +| Group | Name | Target | +|---|---|---| +| `cgc_cli_plugins` | `otel` | `cgc_plugin_otel.cli:get_plugin_commands` | +| `cgc_mcp_plugins` | `otel` | `cgc_plugin_otel.mcp_tools:get_mcp_tools` | diff --git a/plugins/cgc-plugin-otel/pyproject.toml b/plugins/cgc-plugin-otel/pyproject.toml new file mode 100644 index 00000000..6adb46ac --- /dev/null +++ b/plugins/cgc-plugin-otel/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cgc-plugin-otel" +version = "0.1.0" +description = "OpenTelemetry span processor plugin for CodeGraphContext" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + { name = "CodeGraphContext Contributors" } +] +dependencies = [ + "codegraphcontext>=0.3.0", + "typer[all]>=0.9.0", + "neo4j>=5.15.0", + "grpcio>=1.57.0", + "grpcio-tools>=1.57.0", + "opentelemetry-proto>=0.43b0", + "opentelemetry-sdk>=1.20.0", + "opentelemetry-api>=1.20.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.11.0", +] + +[project.entry-points."cgc_cli_plugins"] +otel = "cgc_plugin_otel" + +[project.entry-points."cgc_mcp_plugins"] +otel = "cgc_plugin_otel" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["cgc_plugin_otel*"] diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py new file mode 100644 index 00000000..50bd2309 --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py @@ -0,0 +1,14 @@ +"""OTEL plugin for CodeGraphContext — receives OpenTelemetry spans and writes them to the graph.""" + +from cgc_plugin_otel.cli import get_plugin_commands +from cgc_plugin_otel.mcp_tools import get_mcp_handlers, get_mcp_tools + +PLUGIN_METADATA = { + "name": "cgc-plugin-otel", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": ( + "Receives OpenTelemetry traces via gRPC, writes Service/Trace/Span nodes to the " + "code graph, and correlates runtime spans to static Method nodes." + ), +} diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py new file mode 100644 index 00000000..16ae1cbe --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py @@ -0,0 +1,83 @@ +"""CLI command group contributed by the OTEL plugin.""" +from __future__ import annotations + +import os +import typer +from typing import Optional + +otel_app = typer.Typer(name="otel", help="OpenTelemetry span commands.") + + +@otel_app.command("query-spans") +def query_spans( + route: Optional[str] = typer.Option(None, "--route", help="Filter by HTTP route"), + service: Optional[str] = typer.Option(None, "--service", help="Filter by service name"), + limit: int = typer.Option(20, "--limit", help="Maximum results"), +): + """Query spans stored in the graph, optionally filtered by route or service.""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + except Exception as e: + typer.echo(f"Database unavailable: {e}", err=True) + raise typer.Exit(1) + + where_clauses = [] + params: dict = {"limit": limit} + if route: + where_clauses.append("sp.http_route = $route") + params["route"] = route + if service: + where_clauses.append("sp.service_name = $service") + params["service"] = service + + where = ("WHERE " + " AND ".join(where_clauses)) if where_clauses else "" + cypher = f"MATCH (sp:Span) {where} RETURN sp.span_id, sp.name, sp.service_name, sp.duration_ms ORDER BY sp.start_time_ns DESC LIMIT $limit" + + try: + driver = db.get_driver() + with driver.session() as session: + result = session.run(cypher, **params) + rows = result.data() + except Exception as e: + typer.echo(f"Query failed: {e}", err=True) + raise typer.Exit(1) + + if not rows: + typer.echo("No spans found.") + return + for row in rows: + typer.echo(f"[{row.get('sp.service_name')}] {row.get('sp.name')} — {row.get('sp.duration_ms', '?')}ms id={row.get('sp.span_id')}") + + +@otel_app.command("list-services") +def list_services(): + """List all services observed in the span graph.""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + driver = db.get_driver() + with driver.session() as session: + rows = session.run("MATCH (s:Service) RETURN s.name ORDER BY s.name").data() + except Exception as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) + + if not rows: + typer.echo("No services found.") + return + for row in rows: + typer.echo(row["s.name"]) + + +@otel_app.command("status") +def status(): + """Show whether the OTEL receiver process is configured.""" + port = os.environ.get("OTEL_RECEIVER_PORT", "5317") + typer.echo(f"OTEL receiver port: {port}") + typer.echo("Run 'python -m cgc_plugin_otel.receiver' to start the gRPC receiver.") + + +def get_plugin_commands() -> tuple[str, typer.Typer]: + """Entry point: return (command_name, typer_app).""" + return ("otel", otel_app) diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py new file mode 100644 index 00000000..5cb1bca4 --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py @@ -0,0 +1,130 @@ +"""MCP tools contributed by the OTEL plugin.""" +from __future__ import annotations + +from typing import Any + +_TOOLS: dict[str, dict] = { + "otel_query_spans": { + "name": "otel_query_spans", + "description": ( + "Query OpenTelemetry spans stored in the graph. " + "Filter by HTTP route and/or service name." + ), + "inputSchema": { + "type": "object", + "properties": { + "http_route": {"type": "string", "description": "Filter by HTTP route (e.g. /api/orders)"}, + "service": {"type": "string", "description": "Filter by service name"}, + "limit": {"type": "integer", "description": "Max results", "default": 20}, + }, + "required": [], + }, + }, + "otel_list_services": { + "name": "otel_list_services", + "description": "List all services observed in the runtime span graph.", + "inputSchema": {"type": "object", "properties": {}, "required": []}, + }, + "otel_cross_layer_query": { + "name": "otel_cross_layer_query", + "description": ( + "Run a pre-built cross-layer query combining static code structure with runtime spans. " + "query_type options: never_observed | cross_service_calls | recent_executions" + ), + "inputSchema": { + "type": "object", + "properties": { + "query_type": { + "type": "string", + "enum": ["never_observed", "cross_service_calls", "recent_executions"], + "description": "The cross-layer query to run", + }, + "limit": {"type": "integer", "default": 20}, + }, + "required": ["query_type"], + }, + }, +} + +_CROSS_LAYER_QUERIES: dict[str, str] = { + "never_observed": ( + "MATCH (m:Method) " + "WHERE NOT EXISTS { MATCH (m)<-[:CORRELATES_TO]-(:Span) } " + "AND NOT EXISTS { MATCH (m)<-[:RESOLVES_TO]-(:StackFrame) } " + "AND m.fqn IS NOT NULL " + "RETURN m.fqn AS fqn, m.class_name AS class_name " + "ORDER BY m.class_name, m.fqn LIMIT $limit" + ), + "cross_service_calls": ( + "MATCH (sp:Span)-[:CALLS_SERVICE]->(svc:Service) " + "RETURN sp.service_name AS caller, svc.name AS callee, sp.http_route AS route, count(*) AS calls " + "ORDER BY calls DESC LIMIT $limit" + ), + "recent_executions": ( + "MATCH (sp:Span)-[:CORRELATES_TO]->(m:Method) " + "RETURN sp.name AS span, m.fqn AS fqn, sp.duration_ms AS duration_ms " + "ORDER BY sp.start_time_ns DESC LIMIT $limit" + ), +} + + +def _make_query_spans_handler(db_manager: Any): + def handle(http_route: str | None = None, service: str | None = None, limit: int = 20) -> dict: + where_clauses = [] + params: dict = {"limit": limit} + if http_route: + where_clauses.append("sp.http_route = $http_route") + params["http_route"] = http_route + if service: + where_clauses.append("sp.service_name = $service") + params["service"] = service + where = ("WHERE " + " AND ".join(where_clauses)) if where_clauses else "" + cypher = ( + f"MATCH (sp:Span) {where} " + "RETURN sp.span_id AS span_id, sp.name AS name, sp.service_name AS service, " + "sp.duration_ms AS duration_ms, sp.http_route AS http_route " + "ORDER BY sp.start_time_ns DESC LIMIT $limit" + ) + driver = db_manager.get_driver() + with driver.session() as session: + return {"spans": session.run(cypher, **params).data()} + return handle + + +def _make_list_services_handler(db_manager: Any): + def handle(**_kwargs: Any) -> dict: + driver = db_manager.get_driver() + with driver.session() as session: + rows = session.run("MATCH (s:Service) RETURN s.name AS name ORDER BY s.name").data() + return {"services": [r["name"] for r in rows]} + return handle + + +def _make_cross_layer_handler(db_manager: Any): + def handle(query_type: str, limit: int = 20, **_kwargs: Any) -> dict: + cypher = _CROSS_LAYER_QUERIES.get(query_type) + if cypher is None: + return {"error": f"Unknown query_type '{query_type}'"} + driver = db_manager.get_driver() + with driver.session() as session: + return {"results": session.run(cypher, limit=limit).data()} + return handle + + +def get_mcp_tools(server_context: dict | None = None) -> dict[str, dict]: + """Entry point: return tool_name → ToolDefinition mapping.""" + return _TOOLS + + +def get_mcp_handlers(server_context: dict | None = None) -> dict[str, Any]: + """Entry point: return tool_name → callable mapping.""" + if server_context is None: + server_context = {} + db_manager = server_context.get("db_manager") + if db_manager is None: + return {} + return { + "otel_query_spans": _make_query_spans_handler(db_manager), + "otel_list_services": _make_list_services_handler(db_manager), + "otel_cross_layer_query": _make_cross_layer_handler(db_manager), + } diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py new file mode 100644 index 00000000..29762250 --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py @@ -0,0 +1,197 @@ +""" +Async Neo4j writer for the OTEL plugin. + +Batches incoming span dicts and flushes them periodically to Neo4j, +with a dead-letter queue for retries during database unavailability. +""" +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +logger = logging.getLogger(__name__) + +_BATCH_SIZE = 100 +_FLUSH_TIMEOUT_S = 5.0 +_QUEUE_MAXSIZE = 10_000 +_DLQ_MAXSIZE = 100_000 + +# --------------------------------------------------------------------------- +# Cypher templates +# --------------------------------------------------------------------------- + +_MERGE_SERVICE = """ +MERGE (s:Service {name: $service_name}) +ON CREATE SET s.first_seen = datetime() +ON MATCH SET s.last_seen = datetime() +""" + +_MERGE_TRACE = """ +MERGE (t:Trace {trace_id: $trace_id}) +ON CREATE SET t.first_seen = datetime() +""" + +_MERGE_SPAN = """ +MERGE (sp:Span {span_id: $span_id}) +ON CREATE SET + sp.trace_id = $trace_id, + sp.name = $name, + sp.span_kind = $span_kind, + sp.service_name = $service_name, + sp.http_route = $http_route, + sp.http_method = $http_method, + sp.class_name = $class_name, + sp.function_name = $function_name, + sp.fqn = $fqn, + sp.db_statement = $db_statement, + sp.db_system = $db_system, + sp.peer_service = $peer_service, + sp.duration_ms = $duration_ms, + sp.start_time_ns = $start_time_ns, + sp.end_time_ns = $end_time_ns, + sp.first_seen = datetime() +ON MATCH SET + sp.observation_count = coalesce(sp.observation_count, 0) + 1, + sp.last_seen = datetime() +""" + +_LINK_SPAN_TO_TRACE = """ +MATCH (sp:Span {span_id: $span_id}), (t:Trace {trace_id: $trace_id}) +MERGE (sp)-[:PART_OF]->(t) +""" + +_LINK_SPAN_TO_SERVICE = """ +MATCH (sp:Span {span_id: $span_id}), (s:Service {name: $service_name}) +MERGE (sp)-[:ORIGINATED_FROM]->(s) +""" + +_LINK_PARENT_SPAN = """ +MATCH (child:Span {span_id: $span_id}), (parent:Span {span_id: $parent_span_id}) +MERGE (child)-[:CHILD_OF]->(parent) +""" + +_LINK_CROSS_SERVICE = """ +MATCH (sp:Span {span_id: $span_id}), (svc:Service {name: $peer_service}) +MERGE (sp)-[:CALLS_SERVICE]->(svc) +""" + +_CORRELATE_TO_METHOD = """ +MATCH (sp:Span {span_id: $span_id}) +WHERE sp.fqn IS NOT NULL +MATCH (m:Method {fqn: sp.fqn}) +MERGE (sp)-[:CORRELATES_TO]->(m) +""" + + +class AsyncOtelWriter: + """ + Buffers spans in an asyncio queue and flushes them to Neo4j in batches. + + Usage:: + + writer = AsyncOtelWriter(db_manager) + asyncio.create_task(writer.run()) # start background flush loop + await writer.enqueue(span_dict) + """ + + def __init__(self, db_manager: Any) -> None: + self._db = db_manager + self._queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=_QUEUE_MAXSIZE) + self._dlq: asyncio.Queue[dict] = asyncio.Queue(maxsize=_DLQ_MAXSIZE) + self._running = False + + async def enqueue(self, span: dict) -> None: + """Add a span to the processing queue, dropping if full.""" + try: + self._queue.put_nowait(span) + except asyncio.QueueFull: + logger.warning("OTEL span queue full — dropping span %s", span.get("span_id")) + + async def run(self) -> None: + """Background task: collect batches and flush.""" + self._running = True + logger.info("AsyncOtelWriter started") + while self._running: + batch = await self._collect_batch() + if batch: + await self._flush_batch(batch) + await self._retry_dlq() + + async def stop(self) -> None: + self._running = False + # Drain remaining items + batch: list[dict] = [] + while not self._queue.empty(): + try: + batch.append(self._queue.get_nowait()) + except asyncio.QueueEmpty: + break + if batch: + await self._flush_batch(batch) + + async def write_batch(self, spans: list[dict]) -> None: + """Write a list of span dicts directly (used in tests and integration).""" + await self._flush_batch(spans) + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + async def _collect_batch(self) -> list[dict]: + batch: list[dict] = [] + try: + # Wait for first item + span = await asyncio.wait_for(self._queue.get(), timeout=_FLUSH_TIMEOUT_S) + batch.append(span) + except asyncio.TimeoutError: + return batch + + # Drain up to batch size + while len(batch) < _BATCH_SIZE: + try: + batch.append(self._queue.get_nowait()) + except asyncio.QueueEmpty: + break + return batch + + async def _flush_batch(self, spans: list[dict]) -> None: + try: + driver = self._db.get_driver() + with driver.session() as session: + for span in spans: + self._write_span_sync(session, span) + logger.debug("Flushed %d spans to Neo4j", len(spans)) + except Exception as exc: + logger.error("Neo4j flush failed (%s) — moving %d spans to DLQ", exc, len(spans)) + for span in spans: + try: + self._dlq.put_nowait(span) + except asyncio.QueueFull: + logger.warning("DLQ full — permanently dropping span %s", span.get("span_id")) + + def _write_span_sync(self, session: Any, span: dict) -> None: + session.run(_MERGE_SERVICE, service_name=span["service_name"]) + session.run(_MERGE_TRACE, trace_id=span["trace_id"]) + session.run(_MERGE_SPAN, **span) + session.run(_LINK_SPAN_TO_TRACE, span_id=span["span_id"], trace_id=span["trace_id"]) + session.run(_LINK_SPAN_TO_SERVICE, span_id=span["span_id"], service_name=span["service_name"]) + if span.get("parent_span_id"): + session.run(_LINK_PARENT_SPAN, span_id=span["span_id"], parent_span_id=span["parent_span_id"]) + if span.get("cross_service") and span.get("peer_service"): + session.run(_LINK_CROSS_SERVICE, span_id=span["span_id"], peer_service=span["peer_service"]) + if span.get("fqn"): + session.run(_CORRELATE_TO_METHOD, span_id=span["span_id"]) + + async def _retry_dlq(self) -> None: + if self._dlq.empty(): + return + retry_batch: list[dict] = [] + while len(retry_batch) < _BATCH_SIZE and not self._dlq.empty(): + try: + retry_batch.append(self._dlq.get_nowait()) + except asyncio.QueueEmpty: + break + if retry_batch: + logger.info("Retrying %d spans from DLQ", len(retry_batch)) + await self._flush_batch(retry_batch) diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py new file mode 100644 index 00000000..cbe6609b --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py @@ -0,0 +1,163 @@ +""" +OTLP gRPC receiver for the OTEL plugin. + +Listens for OpenTelemetry trace exports (ExportTraceServiceRequest) and +queues parsed spans for batch writing to Neo4j. + +Requires: + grpcio>=1.57.0 + opentelemetry-proto>=0.43b0 + +Start standalone:: + + python -m cgc_plugin_otel.receiver +""" +from __future__ import annotations + +import asyncio +import logging +import os +import signal +import sys +from typing import Any + +logger = logging.getLogger(__name__) + +_DEFAULT_PORT = int(os.environ.get("OTEL_RECEIVER_PORT", "5317")) +_FILTER_ROUTES = [r.strip() for r in os.environ.get("OTEL_FILTER_ROUTES", "/health,/metrics,/ping").split(",") if r.strip()] + + +def _span_kind_name(kind_int: int) -> str: + kinds = {0: "UNSPECIFIED", 1: "INTERNAL", 2: "SERVER", 3: "CLIENT", 4: "PRODUCER", 5: "CONSUMER"} + return kinds.get(kind_int, "UNSPECIFIED") + + +def _attrs_to_dict(attributes: Any) -> dict: + """Convert protobuf KeyValue list to a plain dict.""" + result: dict = {} + for kv in attributes: + val = kv.value + if val.HasField("string_value"): + result[kv.key] = val.string_value + elif val.HasField("int_value"): + result[kv.key] = val.int_value + elif val.HasField("double_value"): + result[kv.key] = val.double_value + elif val.HasField("bool_value"): + result[kv.key] = val.bool_value + return result + + +class OTLPSpanReceiver: + """ + gRPC servicer implementing the OpenTelemetry TraceService.Export RPC. + + Depends on generated protobuf stubs from ``opentelemetry-proto``. + Import failures are caught at startup; if gRPC is not installed the + plugin still loads but logs a warning. + """ + + def __init__(self, writer: Any, filter_routes: list[str] | None = None, loop: asyncio.AbstractEventLoop | None = None) -> None: + self._writer = writer + self._filter_routes = filter_routes or _FILTER_ROUTES + self._loop = loop + + def Export(self, request: Any, context: Any) -> Any: + """Handle ExportTraceServiceRequest — called by gRPC framework.""" + try: + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceResponse, + ) + except ImportError: + logger.error("opentelemetry-proto not installed — cannot process spans") + return None # type: ignore[return-value] + + from cgc_plugin_otel.span_processor import build_span_dict, should_filter_span + + for resource_spans in request.resource_spans: + service_name = "unknown" + for attr in resource_spans.resource.attributes: + if attr.key == "service.name": + service_name = attr.value.string_value + break + + for scope_spans in resource_spans.scope_spans: + for span in scope_spans.spans: + attrs = _attrs_to_dict(span.attributes) + if should_filter_span(attrs, self._filter_routes): + continue + + span_dict = build_span_dict( + span_id=span.span_id.hex(), + trace_id=span.trace_id.hex(), + parent_span_id=span.parent_span_id.hex() if span.parent_span_id else None, + name=span.name, + span_kind=_span_kind_name(span.kind), + start_time_ns=span.start_time_unix_nano, + end_time_ns=span.end_time_unix_nano, + attributes=attrs, + service_name=service_name, + ) + # Schedule on the main event loop — Export() runs in gRPC thread pool + if self._loop is not None: + asyncio.run_coroutine_threadsafe(self._writer.enqueue(span_dict), self._loop) + + return ExportTraceServiceResponse() + + +def main() -> None: + """Start the OTLP gRPC receiver and the async writer background task.""" + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + + try: + import grpc + from opentelemetry.proto.collector.trace.v1 import trace_service_pb2_grpc + except ImportError as exc: + logger.error("Cannot start OTEL receiver — missing dependency: %s", exc) + sys.exit(1) + + # Import db_manager from CGC core + try: + from codegraphcontext.core import get_database_manager + db_manager = get_database_manager() + except Exception as exc: + logger.error("Cannot connect to database: %s", exc) + sys.exit(1) + + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + writer = AsyncOtelWriter(db_manager) + servicer = OTLPSpanReceiver(writer, loop=loop) + + from concurrent.futures import ThreadPoolExecutor + + server = grpc.server(ThreadPoolExecutor(max_workers=4)) + trace_service_pb2_grpc.add_TraceServiceServicer_to_server(servicer, server) + server.add_insecure_port(f"[::]:{_DEFAULT_PORT}") + server.start() + logger.info("OTLP gRPC receiver listening on port %d", _DEFAULT_PORT) + + writer_task = loop.create_task(writer.run()) + + def _shutdown(signum: int, frame: Any) -> None: + logger.info("Shutting down OTEL receiver…") + server.stop(grace=5) + loop.call_soon_threadsafe(writer_task.cancel) + + signal.signal(signal.SIGTERM, _shutdown) + signal.signal(signal.SIGINT, _shutdown) + + try: + loop.run_forever() + except KeyboardInterrupt: + pass + finally: + loop.run_until_complete(writer.stop()) + loop.close() + + +if __name__ == "__main__": + main() diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py new file mode 100644 index 00000000..78194bcf --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py @@ -0,0 +1,103 @@ +""" +Pure-logic span processing for the OTEL plugin. + +No gRPC or database dependencies — these functions transform raw span attributes +into typed dicts that the writer can persist to the graph. +""" +from __future__ import annotations + + +def extract_php_context(span_attrs: dict) -> dict: + """ + Parse PHP-specific OpenTelemetry attributes from a span attribute dict. + + Returns a typed dict with all known PHP context keys. Missing keys are + returned as ``None`` rather than raising ``KeyError``. + """ + return { + "namespace": span_attrs.get("code.namespace"), + "function": span_attrs.get("code.function"), + "http_route": span_attrs.get("http.route"), + "http_method": span_attrs.get("http.method"), + "db_statement": span_attrs.get("db.statement"), + "db_system": span_attrs.get("db.system"), + "peer_service": span_attrs.get("peer.service"), + } + + +def build_fqn(namespace: str | None, function: str | None) -> str | None: + """ + Build a fully-qualified name from PHP code.namespace and code.function. + + Returns ``None`` if either component is missing. + """ + if namespace is None or function is None: + return None + return f"{namespace}::{function}" + + +def is_cross_service_span(span_kind: str, span_attrs: dict) -> bool: + """ + Return True when this span represents a call from one service to another. + + A span is cross-service when its kind is CLIENT and ``peer.service`` is set. + """ + return span_kind == "CLIENT" and bool(span_attrs.get("peer.service")) + + +def should_filter_span(span_attrs: dict, filter_routes: list[str]) -> bool: + """ + Return True when the span's HTTP route matches a configured noise filter. + + Spans without an ``http.route`` attribute are never filtered. + """ + if not filter_routes: + return False + route = span_attrs.get("http.route") + if route is None: + return False + return route in filter_routes + + +def build_span_dict( + *, + span_id: str, + trace_id: str, + parent_span_id: str | None, + name: str, + span_kind: str, + start_time_ns: int, + end_time_ns: int, + attributes: dict, + service_name: str, +) -> dict: + """ + Build a normalised span dict ready for Neo4j persistence. + + Duration is converted from nanoseconds to milliseconds. + """ + duration_ms = (end_time_ns - start_time_ns) / 1_000_000 + + php_ctx = extract_php_context(attributes) + fqn = build_fqn(php_ctx["namespace"], php_ctx["function"]) + + return { + "span_id": span_id, + "trace_id": trace_id, + "parent_span_id": parent_span_id, + "name": name, + "span_kind": span_kind, + "service_name": service_name, + "start_time_ns": start_time_ns, + "end_time_ns": end_time_ns, + "duration_ms": duration_ms, + "http_route": php_ctx["http_route"], + "http_method": php_ctx["http_method"], + "class_name": php_ctx["namespace"], + "function_name": php_ctx["function"], + "fqn": fqn, + "db_statement": php_ctx["db_statement"], + "db_system": php_ctx["db_system"], + "peer_service": php_ctx["peer_service"], + "cross_service": is_cross_service_span(span_kind, attributes), + } diff --git a/plugins/cgc-plugin-stub/README.md b/plugins/cgc-plugin-stub/README.md new file mode 100644 index 00000000..ccd2c448 --- /dev/null +++ b/plugins/cgc-plugin-stub/README.md @@ -0,0 +1,36 @@ +# cgc-plugin-stub + +Minimal stub plugin for testing the CGC plugin extension system. + +## Overview + +This package is a reference fixture used by the CGC test suite to exercise plugin +discovery, registration, and lifecycle without requiring any real infrastructure. +It implements the minimum required interface (`PLUGIN_METADATA`, `get_plugin_commands()`, +`get_mcp_tools()`, `get_mcp_handlers()`) and contributes a no-op `stub` CLI command +group and a single `stub_ping` MCP tool. + +## Usage + +Install for development to enable the plugin integration and E2E test suites: + +```bash +pip install -e plugins/cgc-plugin-stub +``` + +Then run: + +```bash +PYTHONPATH=src pytest tests/unit/plugin/ tests/integration/plugin/ tests/e2e/plugin/ -v +``` + +## Entry points + +| Group | Name | Target | +|---|---|---| +| `cgc_cli_plugins` | `stub` | `cgc_plugin_stub.cli:get_plugin_commands` | +| `cgc_mcp_plugins` | `stub` | `cgc_plugin_stub.mcp_tools:get_mcp_tools` | + +## Note + +This plugin is not intended for production use. It exists solely as a test fixture. diff --git a/plugins/cgc-plugin-stub/pyproject.toml b/plugins/cgc-plugin-stub/pyproject.toml new file mode 100644 index 00000000..c388f8d2 --- /dev/null +++ b/plugins/cgc-plugin-stub/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cgc-plugin-stub" +version = "0.1.0" +description = "Minimal stub plugin for testing the CGC plugin system" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + { name = "CodeGraphContext Contributors" } +] +dependencies = [ + "typer[all]>=0.9.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", +] + +[project.entry-points."cgc_cli_plugins"] +stub = "cgc_plugin_stub" + +[project.entry-points."cgc_mcp_plugins"] +stub = "cgc_plugin_stub" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["cgc_plugin_stub*"] diff --git a/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py new file mode 100644 index 00000000..7351c00d --- /dev/null +++ b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py @@ -0,0 +1,13 @@ +"""Stub plugin for testing the CGC plugin system.""" + +from cgc_plugin_stub.cli import get_plugin_commands +from cgc_plugin_stub.mcp_tools import get_mcp_handlers, get_mcp_tools + +PLUGIN_METADATA = { + "name": "cgc-plugin-stub", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": "Minimal stub plugin for testing CGC plugin discovery and loading.", +} + +__all__ = ["PLUGIN_METADATA", "get_plugin_commands", "get_mcp_tools", "get_mcp_handlers"] diff --git a/plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py new file mode 100644 index 00000000..085f9270 --- /dev/null +++ b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py @@ -0,0 +1,15 @@ +"""CLI command group contributed by the stub plugin.""" +import typer + +stub_app = typer.Typer(name="stub", help="Stub plugin commands (for testing).") + + +@stub_app.command() +def hello(): + """Echo a greeting from the stub plugin.""" + typer.echo("Hello from stub plugin") + + +def get_plugin_commands() -> tuple[str, typer.Typer]: + """Entry point: return (command_name, typer_app).""" + return ("stub", stub_app) diff --git a/plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py new file mode 100644 index 00000000..fff439e8 --- /dev/null +++ b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py @@ -0,0 +1,37 @@ +"""MCP tools contributed by the stub plugin.""" +from __future__ import annotations + +from typing import Any + + +_TOOLS: dict[str, dict] = { + "stub_hello": { + "name": "stub_hello", + "description": "Say hello — stub plugin smoke test tool.", + "inputSchema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name to greet", + "default": "World", + } + }, + "required": [], + }, + } +} + + +def _handle_stub_hello(name: str = "World", **_kwargs: Any) -> dict: + return {"greeting": f"Hello {name}"} + + +def get_mcp_tools(server_context: dict | None = None) -> dict[str, dict]: + """Entry point: return tool_name → ToolDefinition mapping.""" + return _TOOLS + + +def get_mcp_handlers(server_context: dict | None = None) -> dict[str, Any]: + """Entry point: return tool_name → callable mapping.""" + return {"stub_hello": _handle_stub_hello} diff --git a/plugins/cgc-plugin-xdebug/Dockerfile b/plugins/cgc-plugin-xdebug/Dockerfile new file mode 100644 index 00000000..6791d1ed --- /dev/null +++ b/plugins/cgc-plugin-xdebug/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12-slim + +RUN groupadd -r cgc && useradd -r -g cgc cgc + +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY src/ ./src/ + +RUN pip install --no-cache-dir -e . && \ + pip install --no-cache-dir "typer[all]>=0.9.0" "neo4j>=5.15.0" || true + +USER cgc + +EXPOSE 9003 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ + CMD python -c "import socket; socket.socket(socket.AF_INET, socket.SOCK_STREAM)" || exit 1 + +# CGC_PLUGIN_XDEBUG_ENABLED must be set to 'true' at runtime +CMD ["python", "-m", "cgc_plugin_xdebug.dbgp_server"] diff --git a/plugins/cgc-plugin-xdebug/README.md b/plugins/cgc-plugin-xdebug/README.md new file mode 100644 index 00000000..fb2f7c3d --- /dev/null +++ b/plugins/cgc-plugin-xdebug/README.md @@ -0,0 +1,50 @@ +# cgc-plugin-xdebug + +Xdebug DBGp call-stack listener plugin for [CodeGraphContext](https://github.com/CodeGraphContext/CodeGraphContext). + +**Intended for development and staging environments only — do not enable in production.** + +## Overview + +This plugin listens for Xdebug DBGp protocol connections on port 9003 and captures +PHP call-stack frames in real time. Parsed frames are written to the CGC Neo4j graph +as `CallChain` nodes, correlated with existing code graph nodes via their fully-qualified +names. This enables live execution path analysis alongside static code structure. + +## Features + +- TCP server accepting Xdebug DBGp connections on port 9003 +- Parses `stack_get` XML responses into structured frame dicts +- Computes deterministic `chain_hash` for deduplicating identical call chains +- Writes call-stack data to Neo4j as `CallChain` / `CallFrame` nodes +- Exposes an `xdebug` CLI command group and MCP tools prefixed with `xdebug_` + +## Requirements + +- Python 3.10+ +- CodeGraphContext >= 0.3.0 +- Neo4j >= 5.15 +- Xdebug configured with `xdebug.mode=debug` and `xdebug.client_host` pointing at the CGC host + +## Installation + +```bash +pip install -e plugins/cgc-plugin-xdebug +``` + +## Runtime activation + +Set the environment variable `CGC_PLUGIN_XDEBUG_ENABLED=true` before starting the +plugin server, otherwise the DBGp listener will not start. + +## MCP tool naming + +All MCP tools contributed by this plugin are prefixed with `xdebug_` +(e.g. `xdebug_query_callchain`). + +## Entry points + +| Group | Name | Target | +|---|---|---| +| `cgc_cli_plugins` | `xdebug` | `cgc_plugin_xdebug.cli:get_plugin_commands` | +| `cgc_mcp_plugins` | `xdebug` | `cgc_plugin_xdebug.mcp_tools:get_mcp_tools` | diff --git a/plugins/cgc-plugin-xdebug/pyproject.toml b/plugins/cgc-plugin-xdebug/pyproject.toml new file mode 100644 index 00000000..464cb093 --- /dev/null +++ b/plugins/cgc-plugin-xdebug/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cgc-plugin-xdebug" +version = "0.1.0" +description = "Xdebug DBGp listener plugin for CodeGraphContext (dev/staging only)" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + { name = "CodeGraphContext Contributors" } +] +dependencies = [ + "codegraphcontext>=0.3.0", + "typer[all]>=0.9.0", + "neo4j>=5.15.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.11.0", +] + +[project.entry-points."cgc_cli_plugins"] +xdebug = "cgc_plugin_xdebug" + +[project.entry-points."cgc_mcp_plugins"] +xdebug = "cgc_plugin_xdebug" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["cgc_plugin_xdebug*"] diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py new file mode 100644 index 00000000..6b4556ae --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py @@ -0,0 +1,19 @@ +"""Xdebug plugin for CodeGraphContext — captures PHP call stacks via DBGp and writes them to the graph. + +NOTE: This plugin is intended for development and staging environments only. +It must be explicitly enabled via CGC_PLUGIN_XDEBUG_ENABLED=true. +""" + +from cgc_plugin_xdebug.cli import get_plugin_commands +from cgc_plugin_xdebug.mcp_tools import get_mcp_handlers, get_mcp_tools + +PLUGIN_METADATA = { + "name": "cgc-plugin-xdebug", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": ( + "Runs a TCP DBGp listener, captures PHP call stacks from Xdebug, " + "deduplicates chains, and writes StackFrame nodes to the code graph. " + "Development/staging only — requires CGC_PLUGIN_XDEBUG_ENABLED=true." + ), +} diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py new file mode 100644 index 00000000..f2cb444f --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py @@ -0,0 +1,90 @@ +"""CLI command group contributed by the Xdebug plugin.""" +from __future__ import annotations + +import os +import threading +import typer +from typing import Optional + +xdebug_app = typer.Typer(name="xdebug", help="Xdebug DBGp call-stack capture commands.") + +_server_thread: threading.Thread | None = None + + +@xdebug_app.command("start") +def start( + host: str = typer.Option("0.0.0.0", "--host", help="Bind address"), + port: int = typer.Option(9003, "--port", help="DBGp listen port"), +): + """Start the Xdebug DBGp TCP listener (requires CGC_PLUGIN_XDEBUG_ENABLED=true).""" + global _server_thread + if os.environ.get("CGC_PLUGIN_XDEBUG_ENABLED", "").lower() != "true": + typer.echo("CGC_PLUGIN_XDEBUG_ENABLED is not set to 'true' — refusing to start.", err=True) + raise typer.Exit(1) + + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + except Exception as e: + typer.echo(f"Database unavailable: {e}", err=True) + raise typer.Exit(1) + + from cgc_plugin_xdebug.neo4j_writer import XdebugWriter + from cgc_plugin_xdebug.dbgp_server import DBGpServer + + writer = XdebugWriter(db) + server = DBGpServer(writer, host=host, port=port) + + _server_thread = threading.Thread(target=server.listen, daemon=True, name="xdebug-dbgp") + _server_thread.start() + typer.echo(f"Xdebug DBGp listener started on {host}:{port} (Ctrl-C to stop)") + try: + _server_thread.join() + except KeyboardInterrupt: + server.stop() + typer.echo("\nXdebug listener stopped.") + + +@xdebug_app.command("status") +def status(): + """Show Xdebug listener configuration.""" + enabled = os.environ.get("CGC_PLUGIN_XDEBUG_ENABLED", "false") + port = os.environ.get("XDEBUG_LISTEN_PORT", "9003") + typer.echo(f"CGC_PLUGIN_XDEBUG_ENABLED: {enabled}") + typer.echo(f"XDEBUG_LISTEN_PORT: {port}") + if enabled.lower() != "true": + typer.echo("Listener is NOT enabled.") + else: + typer.echo("Run 'cgc xdebug start' to start the listener.") + + +@xdebug_app.command("list-chains") +def list_chains( + limit: int = typer.Option(20, "--limit", help="Maximum chains to display"), +): + """List the most-observed call stack chains.""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + driver = db.get_driver() + with driver.session() as session: + rows = session.run( + "MATCH (sf:StackFrame) WHERE sf.observation_count > 0 " + "RETURN sf.fqn AS fqn, sf.observation_count AS count " + "ORDER BY count DESC LIMIT $limit", + limit=limit, + ).data() + except Exception as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) + + if not rows: + typer.echo("No chains recorded.") + return + for row in rows: + typer.echo(f"{row['count']:>6}x {row['fqn']}") + + +def get_plugin_commands() -> tuple[str, typer.Typer]: + """Entry point: return (command_name, typer_app).""" + return ("xdebug", xdebug_app) diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py new file mode 100644 index 00000000..93f4f3ed --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py @@ -0,0 +1,259 @@ +""" +DBGp TCP listener for the Xdebug plugin. + +Implements a minimal DBGp debug client that: + 1. Accepts inbound Xdebug connections on a configurable TCP port. + 2. Sends a ``stack_get`` command to retrieve the call stack. + 3. Parses the XML response into a list of frame dicts. + 4. Delegates persistence to XdebugWriter. + +The server only starts when CGC_PLUGIN_XDEBUG_ENABLED=true. +Uses only Python stdlib (socket, xml.etree.ElementTree, hashlib). +""" +from __future__ import annotations + +import hashlib +import logging +import os +import signal +import socket +import sys +import threading +import xml.etree.ElementTree as ET +from typing import Any + +logger = logging.getLogger(__name__) + +_DBGP_NS = "urn:debugger_protocol_v1" +_DEFAULT_HOST = os.environ.get("XDEBUG_LISTEN_HOST", "0.0.0.0") +_DEFAULT_PORT = int(os.environ.get("XDEBUG_LISTEN_PORT", "9003")) +_ENABLED_ENV = "CGC_PLUGIN_XDEBUG_ENABLED" + + +# --------------------------------------------------------------------------- +# Pure-logic helpers (no I/O — tested directly) +# --------------------------------------------------------------------------- + +def parse_stack_xml(xml_str: str) -> list[dict]: + """ + Parse a DBGp ``stack_get`` XML response into a list of frame dicts. + + Returns frames ordered by ``level`` (ascending, 0 = current frame). + The ``file://`` scheme prefix is stripped from filenames. + """ + try: + root = ET.fromstring(xml_str) + except ET.ParseError as exc: + logger.warning("Failed to parse DBGp XML: %s", exc) + return [] + + frames: list[dict] = [] + for stack_el in root.findall(f"{{{_DBGP_NS}}}stack") + root.findall("stack"): + filename = stack_el.get("filename", "") + if filename.startswith("file://"): + filename = filename[7:] + + frames.append({ + "where": stack_el.get("where", ""), + "level": int(stack_el.get("level", 0)), + "filename": filename, + "lineno": int(stack_el.get("lineno", 0)), + }) + + return sorted(frames, key=lambda f: f["level"]) + + +def compute_chain_hash(frames: list[dict]) -> str: + """ + Compute a deterministic SHA-256 hash for a call stack chain. + + Two identical chains (same where/filename/lineno sequence) produce the + same hash, enabling efficient deduplication. + """ + key = "|".join( + f"{f.get('where','')}:{f.get('filename','')}:{f.get('lineno',0)}" + for f in frames + ) + return hashlib.sha256(key.encode()).hexdigest() + + +def build_frame_id(class_name: str, method_name: str, file_path: str, lineno: int) -> str: + """ + Build a deterministic unique frame identifier string. + + The ID is a SHA-256 hex digest of the four components, ensuring + stability across restarts. + """ + key = f"{class_name}::{method_name}::{file_path}::{lineno}" + return hashlib.sha256(key.encode()).hexdigest() + + +def _parse_where(where: str) -> tuple[str | None, str | None]: + """Split a DBGp 'where' string (Class->method or Class::method) into (class, method).""" + for sep in ("->", "::"): + if sep in where: + parts = where.rsplit(sep, 1) + return parts[0], parts[1] + return None, where or None + + +# --------------------------------------------------------------------------- +# TCP Server +# --------------------------------------------------------------------------- + +class DBGpServer: + """ + Minimal TCP DBGp server that captures PHP call stacks. + + Only starts when ``CGC_PLUGIN_XDEBUG_ENABLED=true``. + """ + + def __init__(self, writer: Any, host: str = _DEFAULT_HOST, port: int = _DEFAULT_PORT) -> None: + self._writer = writer + self._host = host + self._port = port + self._running = False + self._sock: socket.socket | None = None + + def is_enabled(self) -> bool: + return os.environ.get(_ENABLED_ENV, "").lower() == "true" + + def listen(self) -> None: + """Start the TCP listener (blocking). Requires XDEBUG_ENABLED env var.""" + if not self.is_enabled(): + logger.warning( + "Xdebug DBGp server NOT started — set %s=true to enable", _ENABLED_ENV + ) + return + + self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._sock.bind((self._host, self._port)) + self._sock.listen(10) + self._running = True + logger.info("DBGp server listening on %s:%d", self._host, self._port) + + while self._running: + try: + conn, addr = self._sock.accept() + logger.debug("Xdebug connection from %s", addr) + t = threading.Thread(target=self._handle_connection, args=(conn,), daemon=True) + t.start() + except OSError: + break # socket closed + + def stop(self) -> None: + self._running = False + if self._sock: + self._sock.close() + + def _handle_connection(self, conn: socket.socket) -> None: + try: + self._process_session(conn) + except Exception as exc: + logger.debug("DBGp session error: %s", exc) + finally: + conn.close() + + def _process_session(self, conn: socket.socket) -> None: + # Read the init packet (Xdebug sends XML on connect) + _init_xml = self._recv_packet(conn) + + seq = 1 + # Send run to start execution + self._send_cmd(conn, f"run -i {seq}") + seq += 1 + + while True: + # Request the current call stack + self._send_cmd(conn, f"stack_get -i {seq}") + seq += 1 + + response = self._recv_packet(conn) + if not response: + break + + frames = parse_stack_xml(response) + if frames: + self._writer.write_chain(frames) + + # Send run to continue to next breakpoint / end of script + self._send_cmd(conn, f"run -i {seq}") + seq += 1 + + # Check if execution ended + ack = self._recv_packet(conn) + if not ack or "status=\"stopped\"" in ack or "status='stopped'" in ack: + break + + @staticmethod + def _send_cmd(conn: socket.socket, cmd: str) -> None: + data = (cmd + "\0").encode() + conn.sendall(data) + + @staticmethod + def _recv_packet(conn: socket.socket) -> str: + """Read a DBGp length-prefixed null-terminated packet.""" + chunks: list[bytes] = [] + # Read the length digits up to the first \0 + length_bytes = bytearray() + while True: + byte = conn.recv(1) + if not byte: + return "" + if byte == b"\0": + break + length_bytes.extend(byte) + + if not length_bytes: + return "" + + try: + length = int(length_bytes) + except ValueError: + return "" + + # Read the XML body (length bytes + trailing \0) + remaining = length + 1 + while remaining > 0: + chunk = conn.recv(min(remaining, 4096)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + + return b"".join(chunks).rstrip(b"\0").decode("utf-8", errors="replace") + + +def main() -> None: + """Start the DBGp server as a standalone process.""" + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(name)s %(message)s") + + if os.environ.get(_ENABLED_ENV, "").lower() != "true": + logger.warning("Xdebug DBGp server NOT started — set %s=true to enable", _ENABLED_ENV) + sys.exit(0) + + try: + from codegraphcontext.core import get_database_manager + db_manager = get_database_manager() + except Exception as exc: + logger.error("Cannot connect to database: %s", exc) + sys.exit(1) + + from cgc_plugin_xdebug.neo4j_writer import XdebugWriter + + writer = XdebugWriter(db_manager) + server = DBGpServer(writer) + + def _shutdown(signum: int, frame: Any) -> None: + logger.info("Shutting down Xdebug DBGp server…") + server.stop() + + signal.signal(signal.SIGTERM, _shutdown) + signal.signal(signal.SIGINT, _shutdown) + + server.listen() + + +if __name__ == "__main__": + main() diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py new file mode 100644 index 00000000..ce4ee345 --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py @@ -0,0 +1,101 @@ +"""MCP tools contributed by the Xdebug plugin.""" +from __future__ import annotations + +from typing import Any + +_TOOLS: dict[str, dict] = { + "xdebug_list_chains": { + "name": "xdebug_list_chains", + "description": ( + "List the most-observed PHP call stack chains captured by Xdebug. " + "Returns StackFrame nodes ordered by observation count." + ), + "inputSchema": { + "type": "object", + "properties": { + "limit": {"type": "integer", "default": 20, "description": "Max results"}, + "min_observations": { + "type": "integer", + "default": 1, + "description": "Minimum observation count to include", + }, + }, + "required": [], + }, + }, + "xdebug_query_chain": { + "name": "xdebug_query_chain", + "description": ( + "Query the call stack chains that include a specific class or method. " + "Returns StackFrame nodes with their CALLED_BY chain." + ), + "inputSchema": { + "type": "object", + "properties": { + "class_name": {"type": "string", "description": "PHP class name (partial match)"}, + "method_name": {"type": "string", "description": "PHP method name (partial match)"}, + "limit": {"type": "integer", "default": 10}, + }, + "required": [], + }, + }, +} + + +def _make_list_chains_handler(db_manager: Any): + def handle(limit: int = 20, min_observations: int = 1, **_: Any) -> dict: + driver = db_manager.get_driver() + with driver.session() as session: + rows = session.run( + "MATCH (sf:StackFrame) WHERE sf.observation_count >= $min_obs " + "RETURN sf.fqn AS fqn, sf.file_path AS file, sf.lineno AS lineno, " + "sf.observation_count AS observations " + "ORDER BY observations DESC LIMIT $limit", + min_obs=min_observations, + limit=limit, + ).data() + return {"chains": rows} + return handle + + +def _make_query_chain_handler(db_manager: Any): + def handle(class_name: str | None = None, method_name: str | None = None, limit: int = 10, **_: Any) -> dict: + where_parts = [] + params: dict = {"limit": limit} + if class_name: + where_parts.append("sf.class_name CONTAINS $class_name") + params["class_name"] = class_name + if method_name: + where_parts.append("sf.method_name CONTAINS $method_name") + params["method_name"] = method_name + + where = ("WHERE " + " AND ".join(where_parts)) if where_parts else "" + cypher = ( + f"MATCH (sf:StackFrame) {where} " + "OPTIONAL MATCH (sf)-[:CALLED_BY*1..5]->(caller:StackFrame) " + "RETURN sf.fqn AS root_fqn, collect(caller.fqn) AS call_chain, " + "sf.observation_count AS observations " + "ORDER BY observations DESC LIMIT $limit" + ) + driver = db_manager.get_driver() + with driver.session() as session: + return {"results": session.run(cypher, **params).data()} + return handle + + +def get_mcp_tools(server_context: dict | None = None) -> dict[str, dict]: + """Entry point: return tool_name → ToolDefinition mapping.""" + return _TOOLS + + +def get_mcp_handlers(server_context: dict | None = None) -> dict[str, Any]: + """Entry point: return tool_name → callable mapping.""" + if server_context is None: + server_context = {} + db_manager = server_context.get("db_manager") + if db_manager is None: + return {} + return { + "xdebug_list_chains": _make_list_chains_handler(db_manager), + "xdebug_query_chain": _make_query_chain_handler(db_manager), + } diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py new file mode 100644 index 00000000..e8d38854 --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py @@ -0,0 +1,142 @@ +""" +Neo4j writer for the Xdebug plugin. + +Persists PHP call stack chains as StackFrame nodes in the graph, +with LRU-based deduplication to avoid redundant writes. +""" +from __future__ import annotations + +import logging +import os +from typing import Any + +from cgc_plugin_xdebug.dbgp_server import ( + compute_chain_hash, + build_frame_id, + _parse_where, +) + +logger = logging.getLogger(__name__) + +_DEDUP_CACHE_SIZE = int(os.environ.get("XDEBUG_DEDUP_CACHE_SIZE", "10000")) + +# --------------------------------------------------------------------------- +# Cypher templates +# --------------------------------------------------------------------------- + +_MERGE_FRAME = """ +MERGE (sf:StackFrame {frame_id: $frame_id}) +ON CREATE SET + sf.fqn = $fqn, + sf.class_name = $class_name, + sf.method_name = $method_name, + sf.file_path = $file_path, + sf.lineno = $lineno, + sf.observation_count = 1, + sf.first_seen = datetime() +ON MATCH SET + sf.observation_count = coalesce(sf.observation_count, 0) + 1, + sf.last_seen = datetime() +""" + +_LINK_CALLED_BY = """ +MATCH (callee:StackFrame {frame_id: $callee_id}), (caller:StackFrame {frame_id: $caller_id}) +MERGE (callee)-[:CALLED_BY]->(caller) +""" + +_LINK_RESOLVES_TO = """ +MATCH (sf:StackFrame {frame_id: $frame_id}) +WHERE sf.fqn IS NOT NULL +MATCH (m:Method {fqn: sf.fqn}) +MERGE (sf)-[:RESOLVES_TO]->(m) +""" + +_INCREMENT_OBSERVATION = """ +MATCH (sf:StackFrame {frame_id: $frame_id}) +SET sf.observation_count = coalesce(sf.observation_count, 0) + 1, + sf.last_seen = datetime() +""" + + +class XdebugWriter: + """ + Writes Xdebug call stack chains to Neo4j with LRU deduplication. + + When the same chain hash is seen again the writer skips a full MERGE + and only increments the observation_count on the root frame. + """ + + def __init__(self, db_manager: Any, cache_size: int = _DEDUP_CACHE_SIZE) -> None: + self._db = db_manager + self._cache: dict[str, int] = {} # hash → root frame_id + self._cache_size = cache_size + + def write_chain(self, frames: list[dict]) -> None: + """ + Persist a call stack chain. + + If the chain was seen before, only increments the root frame's + observation_count; otherwise writes all StackFrame nodes and + CALLED_BY links, then attempts RESOLVES_TO for each frame. + """ + if not frames: + return + + chain_hash = compute_chain_hash(frames) + if chain_hash in self._cache: + root_frame_id = self._cache[chain_hash] + self._increment_observation(root_frame_id) + return + + driver = self._db.get_driver() + with driver.session() as session: + frame_ids: list[str] = [] + for frame in frames: + class_name, method_name = _parse_where(frame.get("where", "")) + fqn = f"{class_name}::{method_name}" if class_name and method_name else None + frame_id = build_frame_id( + class_name or "", + method_name or "", + frame.get("filename", ""), + frame.get("lineno", 0), + ) + session.run( + _MERGE_FRAME, + frame_id=frame_id, + fqn=fqn, + class_name=class_name, + method_name=method_name, + file_path=frame.get("filename"), + lineno=frame.get("lineno", 0), + ) + frame_ids.append(frame_id) + + # CALLED_BY links: frame[n] called by frame[n+1] + for i in range(len(frame_ids) - 1): + session.run( + _LINK_CALLED_BY, + callee_id=frame_ids[i], + caller_id=frame_ids[i + 1], + ) + + # Try to link each frame to a static Method node + for frame_id in frame_ids: + session.run(_LINK_RESOLVES_TO, frame_id=frame_id) + + root_frame_id = frame_ids[0] if frame_ids else "" + self._evict_if_needed() + self._cache[chain_hash] = root_frame_id + + def _increment_observation(self, frame_id: str) -> None: + try: + driver = self._db.get_driver() + with driver.session() as session: + session.run(_INCREMENT_OBSERVATION, frame_id=frame_id) + except Exception as exc: + logger.warning("Failed to increment observation for frame %s: %s", frame_id, exc) + + def _evict_if_needed(self) -> None: + if len(self._cache) >= self._cache_size: + # Evict oldest entry (first inserted key in CPython 3.7+) + oldest = next(iter(self._cache)) + del self._cache[oldest] diff --git a/pyinstaller_hooks/hook-codegraphcontext.plugin_registry.py b/pyinstaller_hooks/hook-codegraphcontext.plugin_registry.py new file mode 100644 index 00000000..e6963cd5 --- /dev/null +++ b/pyinstaller_hooks/hook-codegraphcontext.plugin_registry.py @@ -0,0 +1,51 @@ +# PyInstaller hook for codegraphcontext.plugin_registry +# +# plugin_registry.py uses importlib.metadata.entry_points(group=...) to +# discover installed CGC plugins at runtime. For this to work in a frozen +# binary the distribution METADATA (including entry_points.txt) for every +# relevant package must be bundled. +# +# This hook: +# 1. Collects the codegraphcontext distribution metadata so the core +# package's own entry points are resolvable in the frozen binary. +# 2. Declares importlib.metadata internals as hidden imports to ensure +# the metadata resolution machinery is included. + +from PyInstaller.utils.hooks import collect_data_files, collect_entry_point + +datas = [] +hiddenimports = [ + "importlib.metadata", + "importlib.metadata._meta", + "importlib.metadata._adapters", + "importlib.metadata._itertools", + "importlib.metadata._functools", + "importlib.metadata._text", + "importlib.metadata.compat.functools", + "importlib.metadata.compat.py39", + "pkg_resources", + "pkg_resources.extern", +] + +# Bundle the codegraphcontext package distribution metadata so that +# importlib.metadata.version("codegraphcontext") resolves inside the frozen +# binary and PluginRegistry._get_cgc_version() returns the correct version. +try: + datas += collect_data_files("codegraphcontext", includes=["**/*.dist-info/**/*"]) +except Exception: + pass + +# Collect distribution METADATA for both plugin entry-point groups so that +# entry_points(group="cgc_cli_plugins") and entry_points(group="cgc_mcp_plugins") +# resolve correctly for any plugin that is installed at freeze time. +for _group in ("cgc_cli_plugins", "cgc_mcp_plugins"): + try: + _ep_datas, _ep_hidden = collect_entry_point(_group) + datas += _ep_datas + hiddenimports += _ep_hidden + except Exception as exc: + import warnings + warnings.warn( + f"hook-codegraphcontext.plugin_registry: collect_entry_point('{_group}') " + f"failed: {exc}" + ) diff --git a/pyinstaller_hooks/rthook_importlib_metadata.py b/pyinstaller_hooks/rthook_importlib_metadata.py new file mode 100644 index 00000000..1a9f9253 --- /dev/null +++ b/pyinstaller_hooks/rthook_importlib_metadata.py @@ -0,0 +1,46 @@ +# Runtime hook: ensure importlib.metadata can resolve entry points in a +# PyInstaller one-file frozen executable. +# +# When the frozen binary unpacks into sys._MEIPASS, distribution METADATA +# directories land there. importlib.metadata uses PathDistribution finders +# that walk sys.path. PyInstaller already inserts _MEIPASS at sys.path[0], +# but the metadata sub-directories are nested under site-packages-style paths. +# This hook adds the _MEIPASS path explicitly so entry_points(group=...) works. +# +# It also registers a fallback using pkg_resources so that any code path that +# calls pkg_resources.iter_entry_points() also resolves correctly. + +import sys +import os + +_meipass = getattr(sys, "_MEIPASS", None) + +if _meipass: + # Ensure _MEIPASS is in sys.path for importlib.metadata path finders. + if _meipass not in sys.path: + sys.path.insert(0, _meipass) + + # Force pkg_resources to rescan working_set so entry points registered + # via .dist-info/entry_points.txt inside _MEIPASS are visible. + try: + import pkg_resources + pkg_resources._initialize_master_working_set() + except Exception: + pass + + # Patch importlib.metadata to also search _MEIPASS for distributions. + try: + from importlib.metadata import MetadataPathFinder + import importlib.metadata as _ilm + + _orig_search_paths = getattr(_ilm, "_search_paths", None) + + def _patched_search_paths(name): # type: ignore[override] + paths = [_meipass] + if _orig_search_paths is not None: + paths.extend(_orig_search_paths(name)) + return paths + + _ilm._search_paths = _patched_search_paths + except Exception: + pass diff --git a/pyproject.toml b/pyproject.toml index cadc0c08..ffc245bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Application Frameworks", ] dependencies = [ + "packaging>=23.0", "neo4j>=5.15.0", "watchdog>=3.0.0", "stdlibs>=2023.11.18", @@ -44,6 +45,18 @@ dev = [ "pytest>=7.4.0", "black>=23.11.0", "pytest-asyncio>=0.21.0", + "pytest-mock>=3.11.0", + "httpx>=0.27.0", +] +otel = [ + "cgc-plugin-otel>=0.1.0", +] +xdebug = [ + "cgc-plugin-xdebug>=0.1.0", +] +all = [ + "cgc-plugin-otel>=0.1.0", + "cgc-plugin-xdebug>=0.1.0", ] [project.urls] diff --git a/samples/KNOWN-LIMITATIONS.md b/samples/KNOWN-LIMITATIONS.md new file mode 100644 index 00000000..4f3c3cd5 --- /dev/null +++ b/samples/KNOWN-LIMITATIONS.md @@ -0,0 +1,70 @@ +# Known Limitations — CGC Sample Applications + +## FQN Correlation Gap + +**Status**: Known limitation — not a bug. Will be resolved in a future story. + +### Problem + +`CORRELATES_TO` edges (OTEL spans → static code nodes) and `RESOLVES_TO` edges (Xdebug +stack frames → static code nodes) **will never form** with the current codebase. + +### Root Cause + +The OTEL writer attempts to match spans to static nodes using: + +```cypher +MATCH (m:Method {fqn: $fqn}) +MERGE (sp)-[:CORRELATES_TO]->(m) +``` + +The Xdebug writer does the same for `RESOLVES_TO`: + +```cypher +MATCH (m:Method {fqn: $fqn}) +MERGE (sf)-[:RESOLVES_TO]->(m) +``` + +However, CGC's graph builder (`src/codegraphcontext/tools/graph_builder.py:379`) creates +**`Function` nodes** (not `Method` nodes) and does **not compute an `fqn` property**: + +```cypher +MERGE (n:Function {name: $name, path: $path, line_number: $line}) +``` + +This means: + +1. **Label mismatch**: Queries match `Method` but the graph contains `Function` +2. **Missing property**: Even if the label were correct, there is no `fqn` property to + match against + +### Impact on Sample Apps + +- OTEL spans will be ingested correctly (Service, Trace, Span nodes all form) +- Static code will be indexed correctly (Function, Class nodes all form) +- **Cross-layer correlation will not work** — the graph has both runtime and static nodes + but no edges connecting them +- The smoke script (`smoke-all.sh`) tests for this explicitly: the `correlates_to` + assertion expects a count of 0 and reports **WARN** (not FAIL) + +### What Each Sample App Would Produce (Once Fixed) + +| App | OTEL `code.namespace` | OTEL `code.function` | Expected FQN | +|---|---|---|---| +| PHP/Laravel | `App\Http\Controllers\OrderController` | `index` | `App\Http\Controllers\OrderController::index` | +| Python/FastAPI | `app.services.order_service.OrderService` | `list_orders` | `app.services.order_service.OrderService.list_orders` | +| TypeScript/Express | `DashboardService` | `getDashboard` | `DashboardService.getDashboard` | + +### Resolution Path + +A future story will: + +1. Add FQN computation to the graph builder (combining path, class, and function name + into a language-appropriate FQN) +2. Change `Function` nodes to `Method` nodes where appropriate (or add a `Method` label + alongside `Function`) +3. Add an `fqn` property to these nodes + +Once that story lands, the sample apps will serve as **regression fixtures** — re-running +`smoke-all.sh` should show the `correlates_to` assertion changing from WARN (count=0) to +PASS (count>0). diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 00000000..e81c9d30 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,177 @@ +# CGC Sample Applications + +Three sample apps demonstrating the full CGC plugin pipeline: **index code → run +instrumented app → generate OTEL spans → query cross-layer graph**. + +## Architecture + +``` + ┌──────────────────────────────────────────┐ + │ docker compose up │ + └──────────────────────────────────────────┘ + │ + ┌───────────────────────────┼───────────────────────────┐ + │ │ │ + ┌──────▼──────┐ ┌───────▼───────┐ ┌───────▼────────┐ + │ PHP/Laravel │ │ Python/FastAPI │ │ TS/Express │ + │ :8080 │ │ :8081 │ │ Gateway :8082 │ + │ OTEL+Xdebug │ │ OTEL │ │ OTEL │ + └──────┬──────┘ └───────┬───────┘ └───┬────┬───────┘ + │ spans │ spans spans │ │ HTTP + │ │ │ │ (cross-service) + └──────────┬───────────────┴──────────────────────┘ │ + │ │ + ┌──────▼──────┐ ┌────────▼────────┐ + │ OTEL │ │ PHP + Python │ + │ Collector │ │ backends │ + │ :4317/4318 │ │ (called by GW) │ + └──────┬──────┘ └─────────────────┘ + │ + ┌──────▼──────┐ + │ CGC OTEL │ + │ Processor │ + │ :5317 │ + └──────┬──────┘ + │ MERGE + ┌──────▼──────┐ + │ Neo4j │ + │ :7474/7687 │ + │ (graph) │ + └─────────────┘ +``` + +## Prerequisites + +- Docker and Docker Compose v2+ +- ~2 GB RAM available for all containers + +No local CGC install required — indexing runs inside a container. + +## Quick Start + +```bash +# From the repository root: +cd samples/ + +# 1. Build and start everything (Neo4j + OTEL stack + 3 sample apps) +docker compose up -d --build + +# 2. Index all three sample apps (one-shot container, no local install needed) +docker compose run --rm indexer + +# 3. Run the automated smoke test (generates traffic + validates graph) +bash smoke-all.sh + +# 4. Explore at http://localhost:7474 (neo4j / codegraph123) +``` + +That's it — no local CGC install, no manual container management. + +## What the Smoke Script Does + +| Phase | Action | +|-------|--------| +| 1. Wait | Polls `/health` on all services until ready (timeout: 120s) | +| 2. Index | Runs `cgc index` on each sample app directory | +| 3. Traffic | Sends GET/POST requests to all routes | +| 4. Ingest | Waits 15s for spans to flow through collector → processor → Neo4j | +| 5. Assert | Runs 7 Cypher assertions against the graph | +| 6. Summary | Reports PASS/WARN/FAIL counts | + +## Sample Apps + +### PHP/Laravel (`:8080`) + +| Route | Method | Purpose | +|-------|--------|---------| +| `/api/orders` | GET | List orders | +| `/api/orders` | POST | Create order (`{"name": "...", "quantity": N}`) | +| `/health` | GET | Health check | + +Exercises both OTEL and Xdebug plugins. PHP FQN format: +`App\Http\Controllers\OrderController::index`. + +### Python/FastAPI (`:8081`) + +| Route | Method | Purpose | +|-------|--------|---------| +| `/api/orders` | GET | List orders | +| `/api/orders` | POST | Create order (`{"name": "...", "quantity": N}`) | +| `/health` | GET | Health check | + +Exercises OTEL with Python conventions. Python FQN format: +`app.services.order_service.OrderService.list_orders`. + +### TypeScript/Express Gateway (`:8082`) + +| Route | Method | Purpose | +|-------|--------|---------| +| `/api/dashboard` | GET | Aggregates from PHP + Python backends | +| `/api/orders` | GET | Proxies to PHP backend | +| `/api/orders` | POST | Proxies to PHP backend | +| `/health` | GET | Health check | + +Exercises OTEL cross-service tracing. Gateway calls produce CLIENT spans with +`peer.service` attributes → `CALLS_SERVICE` edges in the graph. + +## Exploring the Graph + +After running the smoke script, open Neo4j Browser at http://localhost:7474 +(user: `neo4j`, password: `codegraph123`) and try: + +```cypher +-- All services +MATCH (s:Service) RETURN s; + +-- Spans for /api/orders +MATCH (sp:Span) WHERE sp.http_route CONTAINS '/api/orders' +RETURN sp.service_name, sp.http_route, sp.duration_ms +LIMIT 20; + +-- Cross-service call graph +MATCH (sp:Span)-[:CALLS_SERVICE]->(svc:Service) +RETURN sp.service_name AS caller, svc.name AS callee, count(*) AS calls; + +-- Full trace visualization +MATCH path = (sp:Span)-[:PART_OF]->(t:Trace) +RETURN path LIMIT 50; + +-- Static code indexed from samples +MATCH (f:Function) WHERE f.path CONTAINS 'samples/' +RETURN f.name, f.path LIMIT 20; +``` + +## Hosted MCP Server (Optional) + +The sample stack includes a `cgc-mcp` service that runs the CGC MCP server +over HTTP on port 8045. It is disabled by default so it does not interfere with +the plugin pipeline demo. To start it alongside the rest of the stack, use the +`mcp` Docker Compose profile: + +```bash +# Start the full sample stack plus the hosted MCP server +docker compose --profile mcp up -d --build + +# Or start only the MCP server after the stack is already running +docker compose --profile mcp up -d cgc-mcp +``` + +Once running, point any MCP-capable AI client (Claude Desktop, VS Code, Cursor, +Claude Code) at `http://localhost:8045/mcp`. The server exposes the same tools +as the local stdio mode plus the OTEL and Xdebug plugin tools bundled in the +`cgc-mcp` image. For reverse-proxy auth, TLS configuration, Kubernetes +manifests, and full client setup instructions see +[docs/docs/deployment/MCP_SERVER_HOSTING.md](../docs/docs/deployment/MCP_SERVER_HOSTING.md). + +## Known Limitations + +See [KNOWN-LIMITATIONS.md](KNOWN-LIMITATIONS.md) for documentation of the FQN +correlation gap — `CORRELATES_TO` edges between OTEL spans and static code nodes +will not form until FQN computation is added to the graph builder. + +## Cleanup + +```bash +cd samples/ +docker compose down -v # removes containers and volumes +``` diff --git a/samples/VALIDATION-REPORT.md b/samples/VALIDATION-REPORT.md new file mode 100644 index 00000000..25c8de1c --- /dev/null +++ b/samples/VALIDATION-REPORT.md @@ -0,0 +1,198 @@ +# Sample Apps Validation Report + +**Date**: 2026-03-20 +**Branch**: `001-cgc-plugin-extension` +**Neo4j**: 2026.02.2 +**Stack**: 3 sample apps + OTEL Collector + OTEL Processor + Xdebug Listener + Neo4j + +## Graph Summary + +| Node Type | Count | Source | +|-----------|-------|--------| +| Span | 362 | OTEL plugin (runtime) | +| Variable | 110 | CGC indexer (static) | +| Trace | 67 | OTEL plugin (runtime) | +| Function | 32 | CGC indexer (static) | +| File | 28 | CGC indexer (static) | +| Module | 28 | CGC indexer (static) | +| Directory | 17 | CGC indexer (static) | +| Parameter | 16 | CGC indexer (static) | +| Class | 14 | CGC indexer (static) | +| Service | 3 | OTEL plugin (runtime) | +| Repository | 3 | CGC indexer (static) | +| Interface | 2 | CGC indexer (static) | + +| Relationship | Count | Source | +|-------------|-------|--------| +| PART_OF | 362 | Span → Trace | +| ORIGINATED_FROM | 362 | Span → Service | +| CONTAINS | 232 | File/Class/Module containment | +| IMPORTS | 35 | Module imports | +| CALLS | 33 | Static function calls | +| CALLS_SERVICE | 20 | Cross-service CLIENT spans | +| HAS_PARAMETER | 17 | Function parameters | +| CHILD_OF | 7 | Distributed trace parent-child | +| CORRELATES_TO | 0 | Runtime → Static (broken) | + +--- + +## What Works + +### Static Analysis (CGC Indexer) + +The indexer correctly identifies the Controller → Service → Repository architecture +across all three apps. The full static call graph is visible: + +**PHP/Laravel** (13 functions, 6 classes): +``` +OrderController.index → OrderService.listOrders → OrderRepository.findAll +OrderController.store → OrderService.createOrder → OrderRepository.create +OrderRepository.__construct → OrderRepository.ensureTableExists +``` + +**Python/FastAPI** (11 functions, 4 classes): +``` +list_orders (router) → OrderService.list_orders → OrderRepository.find_all +create_order (router) → OrderService.create_order → OrderRepository.create +lifespan → OrderRepository.init_db +``` + +**TypeScript/Express** (8 functions, 4 classes): +``` +DashboardService.getDashboard → httpGet (×2, one per backend) +ProxyService.proxyGet → httpRequest +ProxyService.proxyPost → httpRequest +``` + +### Runtime Intelligence (OTEL Plugin) + +Three services discovered with accurate traffic attribution: + +| Service | Spans | Span Kinds | +|---------|-------|------------| +| sample-ts-gateway | 239 | 180 INTERNAL, 32 CLIENT, 27 SERVER | +| sample-python | 83 | 59 INTERNAL, 24 SERVER | +| sample-php | 40 | 40 SERVER | + +**Route-level performance**: + +| Service | Route | Avg Latency | Requests | +|---------|-------|-------------|----------| +| sample-python | /api/orders GET | 1.09 ms | 21 | +| sample-php | api/orders | 12.52 ms | 36 | +| sample-ts-gateway | /api/orders GET | 54.10 ms | 12 | +| sample-ts-gateway | /api/dashboard GET | 54.43 ms | 15 | + +The gateway routes are ~54ms because they proxy to backends — this is correctly +reflected in the latency data and explainable by the cross-service call graph. + +### Cross-Service Topology (CALLS_SERVICE) + +The OTEL plugin correctly identifies service-to-service dependencies: + +``` +sample-ts-gateway → sample-php (13 calls, 51ms avg) +sample-ts-gateway → sample-python (7 calls, 3ms avg) +``` + +This reveals that the gateway is a fan-out aggregator and that the PHP backend +is significantly slower than the Python backend (51ms vs 3ms for the same +GET operation). + +### Distributed Trace Linking (CHILD_OF) + +Parent-child span relationships work across services: + +``` +sample-ts-gateway: GET /api/dashboard + └─ sample-ts-gateway: GET (CLIENT → sample-php) + └─ sample-ts-gateway: GET (CLIENT → sample-python) + └─ sample-python: GET /api/orders (SERVER) +``` + +--- + +## What Doesn't Work + +### Cross-Layer Correlation (CORRELATES_TO): 0 edges + +**This is the documented FQN gap.** Runtime spans and static code nodes exist as +disconnected islands in the graph. No edges connect them. + +**Root cause (two-fold)**: + +1. **Graph builder stores `Function` nodes without `fqn` property.** + The OTEL writer attempts `MATCH (m:Method {fqn: sp.fqn})` but: + - Static nodes are labeled `Function`, not `Method` + - No `fqn` property exists on static nodes + - See `src/codegraphcontext/tools/graph_builder.py:379` + +2. **OTEL auto-instrumentation doesn't emit `code.namespace` / `code.function`.** + The standard auto-instrumentation libraries for PHP, Python, and Node.js + produce span names like `GET /api/orders` and `middleware - jsonParser` — + not function-level code attributes. All spans have `class_name: null`, + `function_name: null`, `fqn: null`. + +**Impact**: Queries that attempt to answer "which code paths are never executed +at runtime" report ALL functions as unobserved, which is misleading since the +services are clearly running and handling traffic. + +### Queries That Return Misleading Results + +**"Functions never observed at runtime"** — returns all 32 functions because +no correlation edges exist. This is the primary use case that cross-layer +queries are meant to serve, and it produces false negatives today. + +**"Class activity status"** — reports all 14 classes as DORMANT despite the +services actively processing requests. Same root cause. + +**"Gateway route → backend dependency map"** via SERVER spans → CALLS_SERVICE +join returns empty because the CALLS_SERVICE edges are on CLIENT spans, not +SERVER spans. This is a query design issue, not a data issue — querying +CLIENT spans directly works correctly. + +--- + +## Smoke Script Results + +``` +Phase 5: Running assertions... + PASS: service_count = 3 (>= 3) + PASS: span_orders = 16 (> 0) + PASS: static_functions = 27 (> 0) + PASS: static_classes = 12 (> 0) + PASS: cross_service > 0 + PASS: trace_links = 106 (> 0) + WARN: correlates_to = 0 (known FQN gap) +``` + +6 PASS, 1 WARN, 0 FAIL. + +--- + +## Recommendations for Future Work + +### To fix cross-layer correlation (separate story): + +1. Add FQN computation to graph builder — combine path, class name, and function + name into a language-appropriate FQN property on Function/Method nodes +2. Add custom OTEL instrumentation hooks to emit `code.namespace` and + `code.function` attributes (or compute FQN from span name + service context) +3. Change the CORRELATES_TO query to match `Function` nodes (not just `Method`) + +### To improve the sample apps: + +1. Add custom PHP OTEL hook to populate `code.namespace`/`code.function` from + Laravel route resolution +2. Add Python OTEL hook using `opentelemetry-instrumentation-fastapi` to emit + code attributes from route handlers +3. Consider adding `peer.service` to the OTEL Collector config via a processor + rather than hardcoding in the gateway instrumentation + +### To improve the OTEL plugin: + +1. Accept `net.peer.name` as a fallback for `peer.service` in cross-service + detection (the HTTP instrumentation sets `net.peer.name` even without + `peer.service`) +2. Add a span attribute for the target URL so that cross-service calls can be + correlated even without explicit `peer.service` diff --git a/samples/docker-compose.yml b/samples/docker-compose.yml new file mode 100644 index 00000000..3d8d8339 --- /dev/null +++ b/samples/docker-compose.yml @@ -0,0 +1,141 @@ +# CGC Sample Applications — full pipeline demo. +# +# Extends the plugin stack (Neo4j + OTEL Collector + OTEL Processor) and adds +# three sample apps: PHP/Laravel, Python/FastAPI, TypeScript/Express gateway. +# +# Usage: +# cd samples/ +# docker compose up -d --build +# docker compose run --rm indexer # one-shot: indexes all sample apps +# bash smoke-all.sh +# +# Neo4j Browser: http://localhost:7474 +# PHP app: http://localhost:8080/api/orders +# Python app: http://localhost:8081/api/orders +# TS gateway: http://localhost:8082/api/dashboard + +include: + - path: ../docker-compose.plugin-stack.yml + - path: ../docker-compose.dev.yml + +services: + + # ── PHP/Laravel sample (OTEL + Xdebug) ──────────────────────────────────── + sample-php: + build: + context: ./php-laravel + dockerfile: Dockerfile + container_name: cgc-sample-php + environment: + - OTEL_PHP_AUTOLOAD_ENABLED=true + - OTEL_SERVICE_NAME=sample-php + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - APP_KEY=${APP_KEY:-base64:dGhpc2lzYXRlc3RrZXlmb3JzYW1wbGVhcHBzMTIzNA==} + ports: + - "8080:8080" + depends_on: + otel-collector: + condition: service_started + networks: + - cgc-network + restart: unless-stopped + + # ── Python/FastAPI sample (OTEL) ─────────────────────────────────────────── + sample-python: + build: + context: ./python-fastapi + dockerfile: Dockerfile + container_name: cgc-sample-python + environment: + - OTEL_SERVICE_NAME=sample-python + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + ports: + - "8081:8081" + depends_on: + otel-collector: + condition: service_started + networks: + - cgc-network + restart: unless-stopped + + # ── TypeScript/Express gateway (OTEL cross-service) ──────────────────────── + sample-ts-gateway: + build: + context: ./ts-express-gateway + dockerfile: Dockerfile + container_name: cgc-sample-ts-gateway + environment: + - OTEL_SERVICE_NAME=sample-ts-gateway + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - PHP_BACKEND_URL=http://sample-php:8080 + - PYTHON_BACKEND_URL=http://sample-python:8081 + ports: + - "8082:8082" + depends_on: + sample-php: + condition: service_started + sample-python: + condition: service_started + otel-collector: + condition: service_started + networks: + - cgc-network + restart: unless-stopped + + # ── Hosted MCP server (HTTP transport, optional) ────────────────────────── + # Serves the Model Context Protocol over HTTP on port 8045. + # This is an alternative to the stdio-based local MCP mode used by IDEs. + # It is NOT started by default — activate with the "mcp" profile: + # + # docker compose --profile mcp up -d cgc-mcp + # + # Connect AI clients to http://localhost:8045/mcp + # See docs/docs/deployment/MCP_SERVER_HOSTING.md for full details. + cgc-mcp: + build: + context: .. + dockerfile: ../Dockerfile.mcp + container_name: cgc-sample-mcp + environment: + - DATABASE_TYPE=neo4j + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - CGC_CORS_ORIGIN=${CGC_CORS_ORIGIN:-*} + - CGC_MCP_PORT=${CGC_MCP_PORT:-8045} + ports: + - "${CGC_MCP_PORT:-8045}:8045" + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + profiles: + - mcp + + # ── One-shot indexer (indexes all sample apps then exits) ────────────────── + # Usage: docker compose run --rm indexer + indexer: + build: + context: .. + dockerfile: Dockerfile + environment: + - DATABASE_TYPE=neo4j + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + volumes: + - ../:/workspace + entrypoint: ["/bin/sh", "-c"] + command: ["echo 'Indexing PHP/Laravel...' && cgc index /workspace/samples/php-laravel && echo 'Indexing Python/FastAPI...' && cgc index /workspace/samples/python-fastapi && echo 'Indexing TypeScript/Express...' && cgc index /workspace/samples/ts-express-gateway && echo 'Done — all sample apps indexed.'"] + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + profiles: + - indexer diff --git a/samples/php-laravel/Dockerfile b/samples/php-laravel/Dockerfile new file mode 100644 index 00000000..2b0f654e --- /dev/null +++ b/samples/php-laravel/Dockerfile @@ -0,0 +1,38 @@ +FROM php:8.3-cli + +# System deps +RUN apt-get update && apt-get install -y \ + libzip-dev libsqlite3-dev unzip git \ + && docker-php-ext-install zip pdo_sqlite \ + && rm -rf /var/lib/apt/lists/* + +# Install Composer +COPY --from=composer:2 /usr/bin/composer /usr/bin/composer + +# Install OTEL + Xdebug extensions +RUN pecl install opentelemetry xdebug \ + && docker-php-ext-enable opentelemetry xdebug + +# Configure Xdebug for remote debugging +RUN echo "xdebug.mode=debug,trace" >> /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini \ + && echo "xdebug.start_with_request=trigger" >> /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini \ + && echo "xdebug.client_host=xdebug-listener" >> /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini \ + && echo "xdebug.client_port=9003" >> /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini + +WORKDIR /app + +COPY . . +RUN composer install --no-dev --no-interaction --prefer-dist + +# Ensure SQLite database exists +RUN mkdir -p database bootstrap/cache storage/framework/{cache,sessions,views} storage/logs \ + && touch database/database.sqlite + +ENV OTEL_PHP_AUTOLOAD_ENABLED=true +ENV OTEL_SERVICE_NAME=sample-php +ENV OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 +ENV OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + +EXPOSE 8080 + +CMD ["php", "artisan", "serve", "--host=0.0.0.0", "--port=8080"] diff --git a/samples/php-laravel/README.md b/samples/php-laravel/README.md new file mode 100644 index 00000000..2699607b --- /dev/null +++ b/samples/php-laravel/README.md @@ -0,0 +1,72 @@ +# CGC Sample: PHP / Laravel + +A minimal Laravel 11 application that exercises both the **OTEL** and **Xdebug** +CGC plugins. The app provides a Controller -> Service -> Repository call +hierarchy so that traces and call stacks capture meaningful multi-layer spans. + +## Routes + +| Method | Path | Handler | +|--------|-----------------|--------------------------------------------------| +| GET | `/health` | `App\Http\Controllers\HealthController::index` | +| GET | `/api/orders` | `App\Http\Controllers\OrderController::index` | +| POST | `/api/orders` | `App\Http\Controllers\OrderController::store` | + +## Call Hierarchy + +``` +OrderController::index + -> OrderService::listOrders + -> OrderRepository::findAll + +OrderController::store + -> OrderService::createOrder + -> OrderRepository::create +``` + +## FQN Format + +PHP uses `\` as the namespace separator and `::` as the method separator: + +``` +App\Http\Controllers\OrderController::index +App\Services\OrderService::listOrders +App\Repositories\OrderRepository::findAll +``` + +## OTEL Span Attributes + +The OpenTelemetry auto-instrumentation for Laravel emits spans with: + +- `code.namespace` = `App\Http\Controllers\OrderController` +- `code.function` = `index` + +These attributes let CGC correlate runtime spans back to the static code graph. + +## Triggering Xdebug Traces + +Xdebug is configured with `start_with_request=trigger`. To activate a debug +session, include the trigger in your request: + +```bash +# Via cookie +curl -b "XDEBUG_TRIGGER=1" http://localhost:8080/api/orders + +# Via query parameter +curl "http://localhost:8080/api/orders?XDEBUG_TRIGGER=1" +``` + +The Xdebug client host is set to `xdebug-listener` (the CGC Xdebug plugin +container) on port 9003. + +## Running + +This app is intended to run as part of the CGC Docker Compose stack. +See `samples/README.md` for the full walkthrough. + +To run standalone for development: + +```bash +composer install +php artisan serve --port=8080 +``` diff --git a/samples/php-laravel/app/Http/Controllers/Controller.php b/samples/php-laravel/app/Http/Controllers/Controller.php new file mode 100644 index 00000000..0fe932d2 --- /dev/null +++ b/samples/php-laravel/app/Http/Controllers/Controller.php @@ -0,0 +1,9 @@ +json(['status' => 'ok']); + } +} diff --git a/samples/php-laravel/app/Http/Controllers/OrderController.php b/samples/php-laravel/app/Http/Controllers/OrderController.php new file mode 100644 index 00000000..244baa5e --- /dev/null +++ b/samples/php-laravel/app/Http/Controllers/OrderController.php @@ -0,0 +1,44 @@ + Service -> Repository + * call hierarchy that produces OTEL spans and Xdebug call stacks. + */ + public function index(): JsonResponse + { + $orders = $this->orderService->listOrders(); + + return response()->json($orders); + } + + /** + * POST /api/orders + * + * Creates a new order. Expects JSON body with "product" and "quantity". + */ + public function store(Request $request): JsonResponse + { + $data = $request->validate([ + 'product' => 'required|string|max:255', + 'quantity' => 'required|integer|min:1', + ]); + + $order = $this->orderService->createOrder($data); + + return response()->json($order, 201); + } +} diff --git a/samples/php-laravel/app/Providers/AppServiceProvider.php b/samples/php-laravel/app/Providers/AppServiceProvider.php new file mode 100644 index 00000000..82c69a3e --- /dev/null +++ b/samples/php-laravel/app/Providers/AppServiceProvider.php @@ -0,0 +1,27 @@ +app->singleton(OrderRepository::class); + $this->app->singleton(OrderService::class); + } + + /** + * Bootstrap application services. + */ + public function boot(): void + { + // + } +} diff --git a/samples/php-laravel/app/Repositories/OrderRepository.php b/samples/php-laravel/app/Repositories/OrderRepository.php new file mode 100644 index 00000000..dda23aac --- /dev/null +++ b/samples/php-laravel/app/Repositories/OrderRepository.php @@ -0,0 +1,73 @@ +pdo = new PDO("sqlite:{$dbPath}"); + $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + $this->pdo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC); + + $this->ensureTableExists(); + } + + /** + * Create the orders table if it does not exist. + */ + private function ensureTableExists(): void + { + $this->pdo->exec(' + CREATE TABLE IF NOT EXISTS orders ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + product TEXT NOT NULL, + quantity INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime(\'now\')) + ) + '); + } + + /** + * Fetch all orders. + * + * @return array> + */ + public function findAll(): array + { + $stmt = $this->pdo->query('SELECT * FROM orders ORDER BY id DESC'); + + return $stmt->fetchAll(); + } + + /** + * Insert a new order and return it. + * + * @param array{product: string, quantity: int} $data + * @return array + */ + public function create(array $data): array + { + $stmt = $this->pdo->prepare( + 'INSERT INTO orders (product, quantity) VALUES (:product, :quantity)' + ); + + $stmt->execute([ + ':product' => $data['product'], + ':quantity' => $data['quantity'], + ]); + + $id = $this->pdo->lastInsertId(); + + $stmt = $this->pdo->prepare('SELECT * FROM orders WHERE id = :id'); + $stmt->execute([':id' => $id]); + + return $stmt->fetch(); + } +} diff --git a/samples/php-laravel/app/Services/OrderService.php b/samples/php-laravel/app/Services/OrderService.php new file mode 100644 index 00000000..bda08dcd --- /dev/null +++ b/samples/php-laravel/app/Services/OrderService.php @@ -0,0 +1,41 @@ +> + */ + public function listOrders(): array + { + return $this->orderRepository->findAll(); + } + + /** + * Create a new order after validation. + * + * @param array{product: string, quantity: int} $data + * @return array + */ + public function createOrder(array $data): array + { + if (empty($data['product'])) { + throw new \InvalidArgumentException('Product name is required'); + } + + if (($data['quantity'] ?? 0) < 1) { + throw new \InvalidArgumentException('Quantity must be at least 1'); + } + + return $this->orderRepository->create($data); + } +} diff --git a/samples/php-laravel/artisan b/samples/php-laravel/artisan new file mode 100755 index 00000000..482fdedd --- /dev/null +++ b/samples/php-laravel/artisan @@ -0,0 +1,23 @@ +#!/usr/bin/env php +make(Illuminate\Contracts\Console\Kernel::class); + +$status = $kernel->handle( + $input = new ArgvInput, + new Symfony\Component\Console\Output\ConsoleOutput +); + +$kernel->terminate($input, $status); + +exit($status); diff --git a/samples/php-laravel/bootstrap/app.php b/samples/php-laravel/bootstrap/app.php new file mode 100644 index 00000000..e59f1389 --- /dev/null +++ b/samples/php-laravel/bootstrap/app.php @@ -0,0 +1,18 @@ +withRouting( + api: __DIR__.'/../routes/api.php', + apiPrefix: '', + ) + ->withMiddleware(function (Middleware $middleware) { + // No additional middleware needed for this sample + }) + ->withExceptions(function (Exceptions $exceptions) { + // Default exception handling + }) + ->create(); diff --git a/samples/php-laravel/bootstrap/providers.php b/samples/php-laravel/bootstrap/providers.php new file mode 100644 index 00000000..38b258d1 --- /dev/null +++ b/samples/php-laravel/bootstrap/providers.php @@ -0,0 +1,5 @@ + env('APP_NAME', 'CGC Sample PHP'), + + /* + |-------------------------------------------------------------------------- + | Application Environment + |-------------------------------------------------------------------------- + */ + 'env' => env('APP_ENV', 'local'), + + /* + |-------------------------------------------------------------------------- + | Application Debug Mode + |-------------------------------------------------------------------------- + */ + 'debug' => (bool) env('APP_DEBUG', true), + + /* + |-------------------------------------------------------------------------- + | Application URL + |-------------------------------------------------------------------------- + */ + 'url' => env('APP_URL', 'http://localhost:8080'), + + /* + |-------------------------------------------------------------------------- + | Application Timezone + |-------------------------------------------------------------------------- + */ + 'timezone' => 'UTC', + + /* + |-------------------------------------------------------------------------- + | Application Locale + |-------------------------------------------------------------------------- + */ + 'locale' => 'en', + 'fallback_locale' => 'en', + 'faker_locale' => 'en_US', + + /* + |-------------------------------------------------------------------------- + | Encryption Key + |-------------------------------------------------------------------------- + */ + 'cipher' => 'AES-256-CBC', + 'key' => env('APP_KEY', 'base64:'.base64_encode(str_repeat('0', 32))), + + /* + |-------------------------------------------------------------------------- + | Maintenance Mode Driver + |-------------------------------------------------------------------------- + */ + 'maintenance' => [ + 'driver' => 'file', + ], + +]; diff --git a/samples/php-laravel/database/database.sqlite b/samples/php-laravel/database/database.sqlite new file mode 100644 index 00000000..e69de29b diff --git a/samples/php-laravel/public/index.php b/samples/php-laravel/public/index.php new file mode 100644 index 00000000..8f2c25b7 --- /dev/null +++ b/samples/php-laravel/public/index.php @@ -0,0 +1,24 @@ +make(Illuminate\Contracts\Http\Kernel::class); + +$response = $kernel->handle( + $request = Request::capture() +)->send(); + +$kernel->terminate($request, $response); diff --git a/samples/php-laravel/routes/api.php b/samples/php-laravel/routes/api.php new file mode 100644 index 00000000..a3d1fd09 --- /dev/null +++ b/samples/php-laravel/routes/api.php @@ -0,0 +1,10 @@ + Service (app.services.order_service.OrderService) + -> Repository (app.repositories.order_repository.OrderRepository) +``` + +## FQN Format + +Python uses dotted module paths throughout, which differs from PHP conventions: + +| Attribute | Python example | PHP equivalent | +|--------------------|------------------------------------------------------------|---------------------------------------------------| +| `code.namespace` | `app.services.order_service.OrderService` | `App\Services\OrderService` | +| `code.function` | `list_orders` | `listOrders` | +| Full FQN | `app.services.order_service.OrderService.list_orders` | `App\Services\OrderService::listOrders` | + +Key differences from PHP: +- Python uses `.` as the separator throughout (module path, class, method). +- PHP uses `\` for namespaces and `::` for methods. + +## Part of CGC Sample Apps + +This sample is part of the CodeGraphContext sample application suite. See `samples/README.md` for the full walkthrough including Docker Compose setup and OTEL collector configuration. diff --git a/samples/python-fastapi/app/__init__.py b/samples/python-fastapi/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/samples/python-fastapi/app/main.py b/samples/python-fastapi/app/main.py new file mode 100644 index 00000000..91e14294 --- /dev/null +++ b/samples/python-fastapi/app/main.py @@ -0,0 +1,19 @@ +from contextlib import asynccontextmanager +from typing import AsyncIterator + +from fastapi import FastAPI + +from app.routers.orders import order_router, _repository +from app.routers.health import health_router + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + """Initialize the SQLite database on startup.""" + await _repository.init_db() + yield + + +app = FastAPI(title="CGC Sample — Python/FastAPI", lifespan=lifespan) +app.include_router(order_router) +app.include_router(health_router) diff --git a/samples/python-fastapi/app/models.py b/samples/python-fastapi/app/models.py new file mode 100644 index 00000000..2b6dedda --- /dev/null +++ b/samples/python-fastapi/app/models.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel + + +class OrderCreate(BaseModel): + name: str + quantity: int + + +class Order(BaseModel): + id: int + name: str + quantity: int + created_at: str diff --git a/samples/python-fastapi/app/repositories/__init__.py b/samples/python-fastapi/app/repositories/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/samples/python-fastapi/app/repositories/order_repository.py b/samples/python-fastapi/app/repositories/order_repository.py new file mode 100644 index 00000000..e8c01674 --- /dev/null +++ b/samples/python-fastapi/app/repositories/order_repository.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import aiosqlite + + +class OrderRepository: + """Thin persistence layer over an aiosqlite database.""" + + def __init__(self, db_path: str) -> None: + self._db_path = db_path + + async def init_db(self) -> None: + """Create the orders table if it does not already exist.""" + async with aiosqlite.connect(self._db_path) as db: + await db.execute( + """ + CREATE TABLE IF NOT EXISTS orders ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + quantity INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + ) + """ + ) + await db.commit() + + async def find_all(self) -> list[dict]: + """Return every order row as a dict.""" + async with aiosqlite.connect(self._db_path) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT id, name, quantity, created_at FROM orders") + rows = await cursor.fetchall() + return [dict(row) for row in rows] + + async def create(self, data: dict) -> dict: + """Insert a new order and return it (with generated id and created_at).""" + async with aiosqlite.connect(self._db_path) as db: + cursor = await db.execute( + "INSERT INTO orders (name, quantity) VALUES (?, ?)", + (data["name"], data["quantity"]), + ) + await db.commit() + order_id = cursor.lastrowid + + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT id, name, quantity, created_at FROM orders WHERE id = ?", + (order_id,), + ) + row = await cursor.fetchone() + return dict(row) diff --git a/samples/python-fastapi/app/routers/__init__.py b/samples/python-fastapi/app/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/samples/python-fastapi/app/routers/health.py b/samples/python-fastapi/app/routers/health.py new file mode 100644 index 00000000..69c4f3ee --- /dev/null +++ b/samples/python-fastapi/app/routers/health.py @@ -0,0 +1,8 @@ +from fastapi import APIRouter + +health_router = APIRouter(prefix="/health", tags=["health"]) + + +@health_router.get("") +async def health_check() -> dict: + return {"status": "ok"} diff --git a/samples/python-fastapi/app/routers/orders.py b/samples/python-fastapi/app/routers/orders.py new file mode 100644 index 00000000..d1a36a90 --- /dev/null +++ b/samples/python-fastapi/app/routers/orders.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter, HTTPException + +from app.models import Order, OrderCreate +from app.services.order_service import OrderService +from app.repositories.order_repository import OrderRepository + +order_router = APIRouter(prefix="/api/orders", tags=["orders"]) + +_DB_PATH = "orders.db" +_repository = OrderRepository(db_path=_DB_PATH) +_service = OrderService(repository=_repository) + + +@order_router.get("", response_model=list[Order]) +async def list_orders() -> list[dict]: + return await _service.list_orders() + + +@order_router.post("", response_model=Order, status_code=201) +async def create_order(data: OrderCreate) -> dict: + try: + return await _service.create_order(data) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) diff --git a/samples/python-fastapi/app/services/__init__.py b/samples/python-fastapi/app/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/samples/python-fastapi/app/services/order_service.py b/samples/python-fastapi/app/services/order_service.py new file mode 100644 index 00000000..bde6e021 --- /dev/null +++ b/samples/python-fastapi/app/services/order_service.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from app.models import OrderCreate +from app.repositories.order_repository import OrderRepository + + +class OrderService: + """Business-logic layer sitting between routers and the repository.""" + + def __init__(self, repository: OrderRepository) -> None: + self._repository = repository + + async def list_orders(self) -> list[dict]: + """Retrieve all orders from the repository.""" + return await self._repository.find_all() + + async def create_order(self, data: OrderCreate) -> dict: + """Validate and persist a new order.""" + if data.quantity <= 0: + raise ValueError("quantity must be a positive integer") + return await self._repository.create(data.model_dump()) diff --git a/samples/python-fastapi/requirements.txt b/samples/python-fastapi/requirements.txt new file mode 100644 index 00000000..88035628 --- /dev/null +++ b/samples/python-fastapi/requirements.txt @@ -0,0 +1,6 @@ +fastapi>=0.110.0 +uvicorn[standard]>=0.27.0 +aiosqlite>=0.19.0 +opentelemetry-distro>=0.44b0 +opentelemetry-exporter-otlp>=1.23.0 +opentelemetry-instrumentation-fastapi>=0.44b0 diff --git a/samples/smoke-all.sh b/samples/smoke-all.sh new file mode 100755 index 00000000..e2bef817 --- /dev/null +++ b/samples/smoke-all.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# smoke-all.sh — Automated end-to-end validation for CGC sample applications. +# +# Runs 6 phases: +# 1. Wait for services to be healthy +# 2. Index sample code via cgc +# 3. Generate HTTP traffic to all sample apps +# 4. Wait for span ingestion +# 5. Assert graph state via Cypher queries +# 6. Print summary +# +# Usage: +# cd samples/ +# docker compose up -d +# bash smoke-all.sh +# +# Exit codes: +# 0 — all assertions passed (WARNs are OK) +# 1 — at least one assertion FAILed + +set -euo pipefail + +# ── Configuration ──────────────────────────────────────────────────────────── + +NEO4J_URI="${NEO4J_URI:-bolt://localhost:7687}" +NEO4J_USERNAME="${NEO4J_USERNAME:-neo4j}" +NEO4J_PASSWORD="${NEO4J_PASSWORD:-codegraph123}" + +PHP_URL="${PHP_URL:-http://localhost:8080}" +PYTHON_URL="${PYTHON_URL:-http://localhost:8081}" +GATEWAY_URL="${GATEWAY_URL:-http://localhost:8082}" + +WAIT_TIMEOUT="${WAIT_TIMEOUT:-120}" # seconds to wait for services +INGEST_WAIT="${INGEST_WAIT:-15}" # seconds to wait for span ingestion + +# ── Helpers ────────────────────────────────────────────────────────────────── + +PASS_COUNT=0 +WARN_COUNT=0 +FAIL_COUNT=0 + +pass() { echo " ✓ PASS: $1"; PASS_COUNT=$((PASS_COUNT + 1)); } +warn() { echo " ⚠ WARN: $1"; WARN_COUNT=$((WARN_COUNT + 1)); } +fail() { echo " ✗ FAIL: $1"; FAIL_COUNT=$((FAIL_COUNT + 1)); } + +cypher_count() { + local query="$1" + # Use cypher-shell if available, otherwise fall back to Neo4j HTTP API + if command -v cypher-shell &>/dev/null; then + cypher-shell -u "$NEO4J_USERNAME" -p "$NEO4J_PASSWORD" -a "$NEO4J_URI" \ + --format plain "$query" 2>/dev/null | tail -1 | tr -d '[:space:]' + else + # Use Neo4j HTTP API (available at port 7474) + local http_url="${NEO4J_HTTP_URL:-http://localhost:7474}" + local result + result=$(curl -s -X POST "$http_url/db/neo4j/tx/commit" \ + -H "Content-Type: application/json" \ + -u "$NEO4J_USERNAME:$NEO4J_PASSWORD" \ + -d "{\"statements\":[{\"statement\":\"$query\"}]}" 2>/dev/null) + echo "$result" | python3 -c " +import sys, json +data = json.load(sys.stdin) +rows = data.get('results', [{}])[0].get('data', []) +if rows: + print(rows[0]['row'][0]) +else: + print(0) +" 2>/dev/null || echo "0" + fi +} + +wait_for_url() { + local url="$1" + local name="$2" + local elapsed=0 + while [ $elapsed -lt "$WAIT_TIMEOUT" ]; do + if curl -sf "$url" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo " Timed out waiting for $name ($url)" + return 1 +} + +# ── Phase 1: Wait for services ────────────────────────────────────────────── + +echo "Phase 1: Waiting for services to be healthy..." + +wait_for_url "$PHP_URL/health" "PHP/Laravel" || { fail "PHP app not reachable"; } +wait_for_url "$PYTHON_URL/health" "Python/FastAPI" || { fail "Python app not reachable"; } +wait_for_url "$GATEWAY_URL/health" "TS/Express gateway" || { fail "Gateway not reachable"; } +wait_for_url "http://localhost:7474" "Neo4j" || { fail "Neo4j not reachable"; } + +echo " All services responding." +echo + +# ── Phase 2: Index sample code ────────────────────────────────────────────── + +echo "Phase 2: Indexing sample application code..." + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +echo " Running indexer service..." +docker compose -f "$SCRIPT_DIR/docker-compose.yml" run --rm indexer +echo " Indexing complete." +echo + +# ── Phase 3: Generate traffic ──────────────────────────────────────────────── + +echo "Phase 3: Generating HTTP traffic to sample apps..." + +# PHP app +curl -sf "$PHP_URL/api/orders" >/dev/null 2>&1 || true +curl -sf -X POST "$PHP_URL/api/orders" \ + -H "Content-Type: application/json" \ + -d '{"name":"test-order","quantity":1}' >/dev/null 2>&1 || true +curl -sf "$PHP_URL/api/orders" >/dev/null 2>&1 || true + +# Python app +curl -sf "$PYTHON_URL/api/orders" >/dev/null 2>&1 || true +curl -sf -X POST "$PYTHON_URL/api/orders" \ + -H "Content-Type: application/json" \ + -d '{"name":"test-order","quantity":2}' >/dev/null 2>&1 || true +curl -sf "$PYTHON_URL/api/orders" >/dev/null 2>&1 || true + +# TS gateway (triggers cross-service calls) +curl -sf "$GATEWAY_URL/api/orders" >/dev/null 2>&1 || true +curl -sf "$GATEWAY_URL/api/dashboard" >/dev/null 2>&1 || true +curl -sf "$GATEWAY_URL/api/dashboard" >/dev/null 2>&1 || true + +echo " Traffic generated (3 requests per app + gateway aggregation)." +echo + +# ── Phase 4: Wait for span ingestion ──────────────────────────────────────── + +echo "Phase 4: Waiting ${INGEST_WAIT}s for span ingestion..." +sleep "$INGEST_WAIT" +echo " Done." +echo + +# ── Phase 5: Assert graph state ───────────────────────────────────────────── + +echo "Phase 5: Running assertions..." + +# Assertion 1: service_count >= 3 +count=$(cypher_count "MATCH (s:Service) RETURN count(s)") +if [ "$count" -ge 3 ] 2>/dev/null; then + pass "service_count = $count (>= 3)" +else + fail "service_count = $count (expected >= 3)" +fi + +# Assertion 2: span_orders > 0 +count=$(cypher_count "MATCH (sp:Span) WHERE sp.http_route CONTAINS '/api/orders' RETURN count(sp)") +if [ "$count" -gt 0 ] 2>/dev/null; then + pass "span_orders = $count (> 0)" +else + fail "span_orders = $count (expected > 0)" +fi + +# Assertion 3: static_functions > 0 +count=$(cypher_count "MATCH (f:Function) WHERE f.path CONTAINS 'samples/' RETURN count(f)") +if [ "$count" -gt 0 ] 2>/dev/null; then + pass "static_functions = $count (> 0)" +else + fail "static_functions = $count (expected > 0 — was cgc index run?)" +fi + +# Assertion 4: static_classes > 0 +count=$(cypher_count "MATCH (c:Class) WHERE c.path CONTAINS 'samples/' RETURN count(c)") +if [ "$count" -gt 0 ] 2>/dev/null; then + pass "static_classes = $count (> 0)" +else + fail "static_classes = $count (expected > 0 — was cgc index run?)" +fi + +# Assertion 5: cross_service > 0 +count=$(cypher_count "MATCH (sp:Span)-[:CALLS_SERVICE]->(svc:Service) RETURN count(sp)") +if [ "$count" -gt 0 ] 2>/dev/null; then + pass "cross_service = $count (> 0)" +else + fail "cross_service = $count (expected > 0)" +fi + +# Assertion 6: trace_links > 0 +count=$(cypher_count "MATCH (sp:Span)-[:PART_OF]->(t:Trace) RETURN count(sp)") +if [ "$count" -gt 0 ] 2>/dev/null; then + pass "trace_links = $count (> 0)" +else + fail "trace_links = $count (expected > 0)" +fi + +# Assertion 7: correlates_to == 0 (known gap — WARN, not FAIL) +count=$(cypher_count "MATCH (sp:Span)-[:CORRELATES_TO]->(m) RETURN count(sp)") +if [ "$count" -eq 0 ] 2>/dev/null; then + warn "correlates_to = 0 (known FQN gap — see KNOWN-LIMITATIONS.md)" +else + pass "correlates_to = $count (> 0 — FQN gap may be resolved!)" +fi + +echo + +# ── Phase 6: Summary ──────────────────────────────────────────────────────── + +echo "════════════════════════════════════════════════════════════" +echo " Smoke Test Summary" +echo "════════════════════════════════════════════════════════════" +echo " PASS: $PASS_COUNT" +echo " WARN: $WARN_COUNT" +echo " FAIL: $FAIL_COUNT" +echo "════════════════════════════════════════════════════════════" + +if [ "$FAIL_COUNT" -gt 0 ]; then + echo " Result: FAILED" + exit 1 +else + echo " Result: PASSED" + exit 0 +fi diff --git a/samples/ts-express-gateway/Dockerfile b/samples/ts-express-gateway/Dockerfile new file mode 100644 index 00000000..0efac37e --- /dev/null +++ b/samples/ts-express-gateway/Dockerfile @@ -0,0 +1,25 @@ +# Build stage +FROM node:20-slim AS builder +WORKDIR /app +COPY package.json package-lock.json* ./ +RUN npm install +COPY tsconfig.json ./ +COPY src/ ./src/ +RUN npm run build + +# Run stage +FROM node:20-slim +WORKDIR /app +COPY package.json package-lock.json* ./ +RUN npm install --omit=dev +COPY --from=builder /app/dist ./dist + +ENV OTEL_SERVICE_NAME=sample-ts-gateway +ENV OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 +ENV OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +ENV PHP_BACKEND_URL=http://sample-php:8080 +ENV PYTHON_BACKEND_URL=http://sample-python:8081 + +EXPOSE 8082 + +CMD ["node", "dist/index.js"] diff --git a/samples/ts-express-gateway/README.md b/samples/ts-express-gateway/README.md new file mode 100644 index 00000000..208afe57 --- /dev/null +++ b/samples/ts-express-gateway/README.md @@ -0,0 +1,39 @@ +# CGC Sample: TypeScript/Express Gateway + +An API gateway that proxies and aggregates requests to the PHP and Python +sample backends, exercising OTEL cross-service tracing. + +## Routes + +| Method | Path | Description | +|--------|-------------------|--------------------------------------------------| +| GET | `/api/dashboard` | Aggregates data from both PHP and Python backends | +| GET | `/api/orders` | Proxies to PHP backend | +| POST | `/api/orders` | Proxies to PHP backend | +| GET | `/health` | Liveness check | + +## Cross-service tracing + +The gateway makes outbound HTTP calls to the PHP backend +(`http://sample-php:8080`) and the Python backend (`http://sample-python:8081`). +OTEL auto-instrumentation wraps these calls with **CLIENT spans** that carry +`peer.service` attributes. The CGC OTEL receiver converts these into +`CALLS_SERVICE` edges in the code graph. + +**W3C trace context propagation** ensures that distributed traces are linked +end-to-end across all three services (gateway, PHP, Python). + +## Running + +This application is designed to run as part of the CGC sample Docker Compose +stack. See `samples/README.md` for the full walkthrough. + +```bash +# Standalone (development) +npm install +npm run dev + +# Docker +docker build -t cgc-sample-ts-gateway . +docker run -p 8082:8082 cgc-sample-ts-gateway +``` diff --git a/samples/ts-express-gateway/package.json b/samples/ts-express-gateway/package.json new file mode 100644 index 00000000..6189f919 --- /dev/null +++ b/samples/ts-express-gateway/package.json @@ -0,0 +1,27 @@ +{ + "name": "cgc-sample-ts-gateway", + "version": "0.1.0", + "description": "CGC sample: Express gateway with OTEL cross-service tracing", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "dev": "ts-node src/index.ts" + }, + "dependencies": { + "express": "^4.18.0", + "@opentelemetry/api": "^1.7.0", + "@opentelemetry/sdk-node": "^0.48.0", + "@opentelemetry/auto-instrumentations-node": "^0.43.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.48.0", + "@opentelemetry/instrumentation-http": "^0.48.0", + "@opentelemetry/instrumentation-express": "^0.35.0", + "@opentelemetry/resources": "^1.22.0", + "@opentelemetry/semantic-conventions": "^1.22.0" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.11.0", + "typescript": "^5.3.0" + } +} diff --git a/samples/ts-express-gateway/src/index.ts b/samples/ts-express-gateway/src/index.ts new file mode 100644 index 00000000..9f787b60 --- /dev/null +++ b/samples/ts-express-gateway/src/index.ts @@ -0,0 +1,20 @@ +// Side-effect import: initialises OTEL SDK before anything else. +import "./instrumentation"; + +import express from "express"; +import { dashboardRouter } from "./routes/dashboard"; +import { ordersRouter } from "./routes/orders"; +import { healthRouter } from "./routes/health"; + +const app = express(); +const PORT = parseInt(process.env.PORT ?? "8082", 10); + +app.use(express.json()); + +app.use("/api/dashboard", dashboardRouter); +app.use("/api/orders", ordersRouter); +app.use("/health", healthRouter); + +app.listen(PORT, () => { + console.log(`sample-ts-gateway listening on :${PORT}`); +}); diff --git a/samples/ts-express-gateway/src/instrumentation.ts b/samples/ts-express-gateway/src/instrumentation.ts new file mode 100644 index 00000000..ae9aa0ec --- /dev/null +++ b/samples/ts-express-gateway/src/instrumentation.ts @@ -0,0 +1,61 @@ +/** + * OTEL SDK setup — must be imported before any other modules so that + * auto-instrumentations can monkey-patch http, express, etc. + */ + +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; +import { HttpInstrumentation } from "@opentelemetry/instrumentation-http"; +import { ExpressInstrumentation } from "@opentelemetry/instrumentation-express"; +import { Resource } from "@opentelemetry/resources"; +import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import type { Span } from "@opentelemetry/api"; +import type { ClientRequest } from "http"; + +const endpoint = + process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://otel-collector:4318"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [SEMRESATTRS_SERVICE_NAME]: "sample-ts-gateway", + }), + traceExporter: new OTLPTraceExporter({ + url: `${endpoint}/v1/traces`, + }), + instrumentations: [ + getNodeAutoInstrumentations({ + // Disable fs instrumentation to reduce noise + "@opentelemetry/instrumentation-fs": { enabled: false }, + }), + new HttpInstrumentation({ + requestHook: (span, request) => { + // Map target hostname to service name for CALLS_SERVICE edges + const req = request as ClientRequest; + const host = (req.getHeader?.("host") ?? "").toString(); + const peerMap: Record = { + "sample-php:8080": "sample-php", + "sample-python:8081": "sample-python", + }; + const peer = peerMap[host]; + if (peer) { + span.setAttribute("peer.service", peer); + } + }, + }), + new ExpressInstrumentation(), + ], +}); + +sdk.start(); + +process.on("SIGTERM", () => { + sdk.shutdown().then( + () => console.log("OTEL SDK shut down"), + (err) => console.error("Error shutting down OTEL SDK", err) + ); +}); + +console.log( + `OTEL instrumentation initialized — exporting to ${endpoint}/v1/traces` +); diff --git a/samples/ts-express-gateway/src/routes/dashboard.ts b/samples/ts-express-gateway/src/routes/dashboard.ts new file mode 100644 index 00000000..c4a2bc59 --- /dev/null +++ b/samples/ts-express-gateway/src/routes/dashboard.ts @@ -0,0 +1,23 @@ +import { Router, Request, Response } from "express"; +import { DashboardService } from "../services/dashboard-service"; + +export const dashboardRouter = Router(); +const service = new DashboardService(); + +/** + * GET /api/dashboard + * + * Aggregates data from both the PHP and Python backends into a single + * response. The outgoing HTTP calls produce CLIENT spans with + * `peer.service` attributes, which generate `CALLS_SERVICE` edges in + * the CGC graph. + */ +dashboardRouter.get("/", async (_req: Request, res: Response) => { + try { + const result = await service.getDashboard(); + res.json(result); + } catch (err) { + console.error("Dashboard aggregation failed:", err); + res.status(502).json({ error: "Failed to aggregate dashboard data" }); + } +}); diff --git a/samples/ts-express-gateway/src/routes/health.ts b/samples/ts-express-gateway/src/routes/health.ts new file mode 100644 index 00000000..2e5b327b --- /dev/null +++ b/samples/ts-express-gateway/src/routes/health.ts @@ -0,0 +1,10 @@ +import { Router, Request, Response } from "express"; + +export const healthRouter = Router(); + +/** + * GET /health — simple liveness check. + */ +healthRouter.get("/", (_req: Request, res: Response) => { + res.json({ status: "ok" }); +}); diff --git a/samples/ts-express-gateway/src/routes/orders.ts b/samples/ts-express-gateway/src/routes/orders.ts new file mode 100644 index 00000000..9eb58a4b --- /dev/null +++ b/samples/ts-express-gateway/src/routes/orders.ts @@ -0,0 +1,37 @@ +import { Router, Request, Response } from "express"; +import { ProxyService } from "../services/proxy-service"; + +export const ordersRouter = Router(); +const proxy = new ProxyService(); + +const PHP_BACKEND = + process.env.PHP_BACKEND_URL ?? "http://sample-php:8080"; + +/** + * GET /api/orders — proxies to PHP backend. + */ +ordersRouter.get("/", async (_req: Request, res: Response) => { + try { + const data = await proxy.proxyGet(`${PHP_BACKEND}/api/orders`); + res.json(data); + } catch (err) { + console.error("Proxy GET /api/orders failed:", err); + res.status(502).json({ error: "Failed to fetch orders from backend" }); + } +}); + +/** + * POST /api/orders — proxies to PHP backend. + */ +ordersRouter.post("/", async (req: Request, res: Response) => { + try { + const data = await proxy.proxyPost( + `${PHP_BACKEND}/api/orders`, + req.body + ); + res.status(201).json(data); + } catch (err) { + console.error("Proxy POST /api/orders failed:", err); + res.status(502).json({ error: "Failed to create order via backend" }); + } +}); diff --git a/samples/ts-express-gateway/src/services/dashboard-service.ts b/samples/ts-express-gateway/src/services/dashboard-service.ts new file mode 100644 index 00000000..79c006be --- /dev/null +++ b/samples/ts-express-gateway/src/services/dashboard-service.ts @@ -0,0 +1,51 @@ +/** + * Aggregates data from PHP and Python backend services. + * + * Uses Node's built-in `http` module so that OTEL HttpInstrumentation + * wraps these calls with CLIENT spans and injects W3C traceparent headers, + * producing `CALLS_SERVICE` edges in the CGC graph. + */ +import http from "http"; + +const PHP_BACKEND = + process.env.PHP_BACKEND_URL ?? "http://sample-php:8080"; +const PYTHON_BACKEND = + process.env.PYTHON_BACKEND_URL ?? "http://sample-python:8081"; + +export interface DashboardResult { + orders: unknown[]; + stats: { + php_count: number; + python_count: number; + }; +} + +function httpGet(url: string): Promise { + return new Promise((resolve, reject) => { + http.get(url, (res) => { + let data = ""; + res.on("data", (chunk) => (data += chunk)); + res.on("end", () => resolve(data)); + }).on("error", reject); + }); +} + +export class DashboardService { + async getDashboard(): Promise { + const [phpData, pythonData] = await Promise.all([ + httpGet(`${PHP_BACKEND}/api/orders`), + httpGet(`${PYTHON_BACKEND}/api/orders`), + ]); + + const phpOrders = JSON.parse(phpData) as unknown[]; + const pythonOrders = JSON.parse(pythonData) as unknown[]; + + return { + orders: [...phpOrders, ...pythonOrders], + stats: { + php_count: phpOrders.length, + python_count: pythonOrders.length, + }, + }; + } +} diff --git a/samples/ts-express-gateway/src/services/proxy-service.ts b/samples/ts-express-gateway/src/services/proxy-service.ts new file mode 100644 index 00000000..1f835f5d --- /dev/null +++ b/samples/ts-express-gateway/src/services/proxy-service.ts @@ -0,0 +1,55 @@ +/** + * Lightweight proxy that forwards requests to backend services. + * + * Uses Node's built-in `http` module so that OTEL HttpInstrumentation + * wraps these calls with CLIENT spans and injects W3C traceparent headers. + */ +import http from "http"; + +function httpRequest( + url: string, + options: http.RequestOptions = {}, + body?: string +): Promise<{ status: number; data: string }> { + return new Promise((resolve, reject) => { + const req = http.request(url, options, (res) => { + let data = ""; + res.on("data", (chunk) => (data += chunk)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, data }) + ); + }); + req.on("error", reject); + if (body) req.write(body); + req.end(); + }); +} + +export class ProxyService { + async proxyGet(url: string): Promise { + const res = await httpRequest(url); + if (res.status < 200 || res.status >= 300) { + throw new Error(`Upstream GET ${url} returned ${res.status}`); + } + return JSON.parse(res.data); + } + + async proxyPost(url: string, body: unknown): Promise { + const payload = JSON.stringify(body); + const res = await httpRequest( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload).toString(), + }, + }, + payload + ); + if (res.status < 200 || res.status >= 300) { + throw new Error(`Upstream POST ${url} returned ${res.status}`); + } + return JSON.parse(res.data); + } +} diff --git a/samples/ts-express-gateway/tsconfig.json b/samples/ts-express-gateway/tsconfig.json new file mode 100644 index 00000000..0a6e9e4f --- /dev/null +++ b/samples/ts-express-gateway/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "lib": ["ES2022"], + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/specs/001-cgc-plugin-extension/checklists/requirements.md b/specs/001-cgc-plugin-extension/checklists/requirements.md new file mode 100644 index 00000000..45d719dd --- /dev/null +++ b/specs/001-cgc-plugin-extension/checklists/requirements.md @@ -0,0 +1,48 @@ +# Specification Quality Checklist: CGC Plugin Extension System + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-03-14 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) — requirements are + user/outcome-focused; technical protocol references (OTEL, DBGp) are domain- + inherent, not avoidable implementation choices; specifics confined to Assumptions +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders (with domain-specific protocol names + explained by context) +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (except SC-010 which names K8s primitives + — acceptable since K8s compatibility is the explicit stated goal of the feature) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded (5 user stories with explicit in/out of scope via + Assumptions section) +- [x] Dependencies and assumptions identified (Assumptions section present) + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows (plugin lifecycle, each of the three plugin + types, and CI/CD pipeline) +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification (technical protocols are domain + vocabulary, not implementation choices; language/tooling confined to Assumptions) + +## Notes + +- All items passed on first validation iteration. No spec updates required before + `/speckit.plan` or `/speckit.clarify`. +- SC-010 intentionally references Kubernetes primitives because K8s compatibility is + the explicit stated requirement from the feature description; this is not an + implementation leak. +- Protocol names (OTEL/OpenTelemetry, DBGp/Xdebug, MCP) are treated as domain + vocabulary equivalent to naming "REST API" or "OAuth" — they identify the integration + standard, not the implementation approach. diff --git a/specs/001-cgc-plugin-extension/contracts/cicd-pipeline.md b/specs/001-cgc-plugin-extension/contracts/cicd-pipeline.md new file mode 100644 index 00000000..2665b6e2 --- /dev/null +++ b/specs/001-cgc-plugin-extension/contracts/cicd-pipeline.md @@ -0,0 +1,143 @@ +# Contract: CI/CD Pipeline for Plugin Service Images + +**Version**: 1.0.0 +**Feature**: 001-cgc-plugin-extension +**Audience**: CGC maintainers and plugin authors contributing container services + +--- + +## 1. Pipeline Triggers + +The shared Docker build pipeline (`docker-publish.yml`) runs on: + +| Trigger | Behavior | +|---|---| +| Push to `main` branch | Build all service images; push with `latest` tag | +| Push of a semver tag (`v*`) | Build all images; push with version tags + `latest` | +| Pull request to `main` | Build all images; smoke test; do NOT push | +| Manual dispatch | Build all images; push with `latest` | + +--- + +## 2. Service Registry + +All container services are declared in `.github/services.json`. This is the only file +that MUST be edited to add or remove a service from the pipeline. + +**Schema**: +```json +[ + { + "name": "cgc-core", + "path": ".", + "dockerfile": "Dockerfile", + "health_check": "version" + }, + { + "name": "cgc-plugin-otel", + "path": "plugins/cgc-plugin-otel", + "dockerfile": "plugins/cgc-plugin-otel/Dockerfile", + "health_check": "grpc_ping" + }, +] +``` + +| Field | Type | Description | +|---|---|---| +| `name` | string | Image name (used as registry path segment) | +| `path` | string | Docker build context path (relative to repo root) | +| `dockerfile` | string | Path to Dockerfile (relative to repo root) | +| `health_check` | string | Smoke test type: `"version"`, `"grpc_ping"`, `"http_health"` | + +--- + +## 3. Image Tagging Convention + +All images are published to the configured registry under: +`//:` + +Tags produced per build: + +| Event | Tags | +|---|---| +| Tag `v1.2.3` pushed | `1.2.3`, `1.2`, `1`, `latest` | +| Push to `main` | `latest`, `main-` | +| Push to other branch | `-` | +| Pull request | `pr-` (not pushed) | + +--- + +## 4. Smoke Test Types + +Each service MUST declare a `health_check` type. The pipeline runs the corresponding +test against the locally-built image before pushing. + +| Type | Test command | Pass condition | +|---|---|---| +| `version` | `docker run --rm --version` | Exit code 0 | +| `grpc_ping` | `docker run --rm python -c "import grpc; print('ok')"` | Exit code 0 | +| `http_health` | Start container, `curl -f http://localhost:/health` | HTTP 200 | + +A build that fails its smoke test MUST NOT be pushed to the registry. +Other services' builds continue regardless (`fail-fast: false`). + +--- + +## 5. Dockerfile Requirements + +Every service Dockerfile MUST: + +1. Use a minimal base image (e.g. `python:3.12-slim`, NOT `python:3.12`) +2. Run as a non-root user (final `USER` instruction MUST NOT be root) +3. Include a `HEALTHCHECK` instruction that CGC's health_check type can exercise +4. Accept all configuration via environment variables (no credentials in `ENV`) +5. Produce a reproducible build (pin dependency versions) + +**Example `HEALTHCHECK`** for a Python service: +```dockerfile +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD python -c "import sys; sys.exit(0)" || exit 1 +``` + +--- + +## 6. Kubernetes Compatibility Requirements + +Published images MUST be deployable via standard Kubernetes `Deployment` + `Service` +manifests with no special configuration: + +- No `hostNetwork: true` required +- No `privileged: true` required +- All config via environment variables (compatible with `ConfigMap` + `Secret`) +- Readiness and liveness probes derivable from `HEALTHCHECK` +- No persistent volume required for stateless plugin services + (Neo4j connection details passed via env vars) + +Reference K8s manifests are provided in `k8s//` for each service. + +--- + +## 7. Adding a New Service + +To add a new plugin service to the pipeline: + +1. Add the service entry to `.github/services.json` +2. Ensure the plugin directory has a `Dockerfile` satisfying §5 +3. The pipeline automatically picks up the new service on the next run + +No other workflow changes are required. + +--- + +## 8. Registry Configuration + +The target registry is configured via repository secrets/variables: + +| Secret/Variable | Description | Example | +|---|---|---| +| `REGISTRY` | Registry hostname | `ghcr.io` | +| `REGISTRY_USERNAME` | Login username (or use `${{ github.actor }}`) | `myorg` | +| `REGISTRY_PASSWORD` | Login password / token | (GitHub token for GHCR) | + +For GHCR (GitHub Container Registry), `REGISTRY_PASSWORD` is `${{ secrets.GITHUB_TOKEN }}` +and no additional secret configuration is required. diff --git a/specs/001-cgc-plugin-extension/contracts/plugin-interface.md b/specs/001-cgc-plugin-extension/contracts/plugin-interface.md new file mode 100644 index 00000000..0950a485 --- /dev/null +++ b/specs/001-cgc-plugin-extension/contracts/plugin-interface.md @@ -0,0 +1,241 @@ +# Contract: CGC Plugin Interface + +**Version**: 1.0.0 +**Feature**: 001-cgc-plugin-extension +**Audience**: Plugin authors + +This document is the authoritative contract for building CGC-compatible plugins. +Plugins that satisfy this contract will be auto-discovered and loaded by CGC core. + +--- + +## 1. Package Structure + +A CGC plugin is a standard Python package installable via pip. It MUST follow this +structure: + +``` +cgc-plugin-/ +├── pyproject.toml # Entry point declarations (required) +└── src/ + └── cgc_plugin_/ + ├── __init__.py # PLUGIN_METADATA declaration (required) + ├── cli.py # CLI contract (required if contributing CLI commands) + └── mcp_tools.py # MCP contract (required if contributing MCP tools) +``` + +--- + +## 2. Plugin Metadata (REQUIRED) + +Every plugin MUST declare `PLUGIN_METADATA` in its package `__init__.py`: + +```python +PLUGIN_METADATA = { + "name": "my-plugin", # str, kebab-case, globally unique + "version": "0.1.0", # str, PEP-440 + "cgc_version_constraint": ">=0.3.0,<1.0", # str, PEP-440 specifier + "description": "One-line description", # str + "author": "Your Name", # str, optional +} +``` + +**Rules**: +- `name` MUST be unique across all installed plugins. Conflicts are resolved by + skipping the second plugin with a warning. +- `cgc_version_constraint` MUST be a valid PEP-440 specifier. Plugins whose constraint + does not match the installed CGC version are skipped at startup. +- All required fields MUST be present. A plugin with missing required fields is skipped. + +--- + +## 3. Entry Point Declarations + +In the plugin's `pyproject.toml`, declare entry points under one or both groups: + +```toml +[project.entry-points."cgc_cli_plugins"] +my-plugin = "cgc_plugin_myname.cli:get_plugin_commands" + +[project.entry-points."cgc_mcp_plugins"] +my-plugin = "cgc_plugin_myname.mcp_tools:get_mcp_tools" +``` + +- A plugin MAY declare CLI entry points only, MCP entry points only, or both. +- The entry point name (left of `=`) MUST match the plugin's `name` in `PLUGIN_METADATA`. + +--- + +## 4. CLI Contract + +If the plugin declares a `cgc_cli_plugins` entry point, the target function MUST have +this signature: + +```python +def get_plugin_commands() -> tuple[str, typer.Typer]: + """ + Returns a (command_group_name, typer_app) tuple. + + - command_group_name: str, kebab-case, globally unique across plugins + - typer_app: typer.Typer instance with commands registered on it + + MUST NOT: + - Have side effects (no database access, no file writes, no network calls) + - Raise unhandled exceptions (caught and logged by PluginRegistry) + - Import CGC internals at module level (use lazy imports inside handlers) + """ +``` + +**Example**: +```python +# cgc_plugin_myname/cli.py +import typer + +my_app = typer.Typer(help="My plugin commands") + +@my_app.command("hello") +def hello(): + """Say hello.""" + typer.echo("Hello from my-plugin!") + +def get_plugin_commands() -> tuple[str, typer.Typer]: + return ("my-plugin", my_app) +``` + +After installation, the user sees: `cgc my-plugin hello` + +--- + +## 5. MCP Contract + +If the plugin declares a `cgc_mcp_plugins` entry point, the target module MUST expose +two functions: + +### 5.1 get_mcp_tools() + +```python +def get_mcp_tools(server_context: dict) -> dict[str, dict]: + """ + Returns tool definitions for registration in CGC's MCP tool manifest. + + Args: + server_context: { + "db_manager": DatabaseManager, # shared graph DB connection + "version": str, # installed CGC version + } + + Returns: + dict mapping tool_name (str) → ToolDefinition (dict) + + MUST NOT: + - Register tools whose names conflict with built-in CGC tools + (conflicts are silently skipped with a warning) + - Raise unhandled exceptions + """ +``` + +### 5.2 get_mcp_handlers() + +```python +def get_mcp_handlers(server_context: dict) -> dict[str, callable]: + """ + Returns handler callables for each tool registered in get_mcp_tools(). + + Args: + server_context: same as get_mcp_tools() + + Returns: + dict mapping tool_name (str) → handler callable + + Handler callable signature: + def handler(**kwargs) -> dict: + # kwargs match the tool's inputSchema properties + # Returns a JSON-serialisable dict + """ +``` + +### 5.3 ToolDefinition Schema + +Each value in the `get_mcp_tools()` return dict MUST conform to this schema: + +```python +{ + "name": str, # MUST match the dict key + "description": str, # Human-readable description (shown in AI tool listings) + "inputSchema": { # JSON Schema draft-07 object + "type": "object", + "properties": { + "": { + "type": "string" | "integer" | "boolean" | "array" | "object", + "description": str, + # ... other JSON Schema keywords + } + }, + "required": [str, ...] # list of required property names + } +} +``` + +**Example**: +```python +# cgc_plugin_myname/mcp_tools.py + +def get_mcp_tools(server_context): + db = server_context["db_manager"] + return { + "myplugin_greet": { + "name": "myplugin_greet", + "description": "Greet by name", + "inputSchema": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "Name to greet"} + }, + "required": ["name"] + } + } + } + +def get_mcp_handlers(server_context): + db = server_context["db_manager"] + def greet_handler(name: str) -> dict: + return {"greeting": f"Hello, {name}!"} + return {"myplugin_greet": greet_handler} +``` + +--- + +## 6. Naming Conventions + +To prevent conflicts in a shared namespace, plugin-registered names MUST be prefixed: + +| Artifact | Naming Rule | Example | +|---|---|---| +| CLI command group | plugin name (kebab-case) | `cgc otel ...` | +| MCP tool names | `_` | `otel_query_spans` | +| Graph node labels | PascalCase, no prefix needed | `Span`, `StackFrame` | +| Graph `source` values | `"runtime_"` | `"runtime_otel"` | + +--- + +## 7. Error Handling Expectations + +CGC wraps all plugin calls. Plugins SHOULD still implement defensive error handling: + +- Handlers SHOULD catch database exceptions and return an `{"error": "..."}` dict + rather than raising exceptions, to produce clean error messages for AI agents. +- Handlers MUST be idempotent for write operations (use MERGE, not CREATE). +- Handlers MUST NOT retain state across calls beyond what the `db_manager` persists. + +--- + +## 8. Testing Requirements + +Plugin packages MUST include: + +- `tests/unit/` — unit tests for extraction/parsing logic (mocked database) +- `tests/integration/` — tests verifying the plugin registers correctly with a real + CGC server instance + +Plugin tests MUST pass with `pytest tests/unit tests/integration`. +Plugin tests SHOULD be runnable independently without CGC core installed (via mocks). diff --git a/specs/001-cgc-plugin-extension/data-model.md b/specs/001-cgc-plugin-extension/data-model.md new file mode 100644 index 00000000..2726b8f4 --- /dev/null +++ b/specs/001-cgc-plugin-extension/data-model.md @@ -0,0 +1,272 @@ +# Data Model: CGC Plugin Extension System + +**Feature**: 001-cgc-plugin-extension +**Date**: 2026-03-14 + +This document describes both the in-memory runtime data model for the plugin system +and the new graph nodes/relationships added to the CGC graph schema by the plugins. + +--- + +## Part 1: Plugin System Runtime Model + +These entities exist at Python runtime only (not persisted to the graph). + +--- + +### PluginMetadata + +Declared by each plugin in `__init__.py::PLUGIN_METADATA`. Validated by `PluginRegistry` +at startup. + +| Field | Type | Required | Description | +|---|---|---|---| +| `name` | str | ✅ | Unique plugin identifier (kebab-case) | +| `version` | str | ✅ | Plugin version (PEP-440, e.g. `"0.1.0"`) | +| `cgc_version_constraint` | str | ✅ | PEP-440 specifier for compatible CGC versions (e.g. `">=0.3.0,<1.0"`) | +| `description` | str | ✅ | One-line human description | +| `author` | str | ❌ | Author name or team | + +**Validation rules**: +- `name` MUST be unique across all installed plugins +- `cgc_version_constraint` MUST be a valid PEP-440 specifier string +- Plugin is rejected if `cgc_version_constraint` does not match installed CGC version + +--- + +### PluginRegistration + +Runtime state of a successfully loaded plugin, held in `PluginRegistry.loaded_plugins`. + +| Field | Type | Description | +|---|---|---| +| `name` | str | Plugin name (from metadata) | +| `metadata` | PluginMetadata | Validated metadata dict | +| `cli_commands` | `list[Tuple[str, typer.Typer]]` | Registered command groups | +| `mcp_tools` | `dict[str, ToolDefinition]` | Registered MCP tool schemas | +| `mcp_handlers` | `dict[str, Callable]` | Tool name → handler function | +| `status` | `"loaded" \| "failed" \| "skipped"` | Load outcome | +| `failure_reason` | `str \| None` | Set when status is failed or skipped | + +--- + +### PluginRegistry + +Singleton held by the CGC process. Manages discovery, validation, and lifecycle. + +| Field | Type | Description | +|---|---|---| +| `loaded_plugins` | `dict[str, PluginRegistration]` | Name → registration for successfully loaded plugins | +| `failed_plugins` | `dict[str, str]` | Name → failure reason for failed/skipped plugins | + +**State transitions**: +``` +discovered → compatibility_check → [compatible] → loaded + → [incompatible] → skipped + → [import error] → failed + → [call error] → failed +``` + +--- + +### CLIPluginContract + +The callable contract each CLI plugin entry point MUST satisfy. + +```python +def get_plugin_commands() -> tuple[str, typer.Typer]: + """ + Returns: + (command_group_name, typer_app_instance) + + Raises: + Any exception → caught and logged by PluginRegistry; plugin skipped + """ +``` + +**Invariants**: +- `command_group_name` MUST be unique (CGC rejects duplicates with a warning) +- `typer_app` MUST be a `typer.Typer` instance +- Function MUST NOT have side effects beyond creating the Typer app + +--- + +### MCPPluginContract + +The callable contract each MCP plugin entry point MUST satisfy. + +```python +def get_mcp_tools(server_context: dict) -> dict[str, ToolDefinition]: + """ + Args: + server_context: { + "db_manager": DatabaseManager, + "version": str, + } + + Returns: + dict of tool_name → ToolDefinition + + Raises: + Any exception → caught and logged by PluginRegistry; plugin skipped + """ + +def get_mcp_handlers(server_context: dict) -> dict[str, Callable]: + """ + Returns: + dict of tool_name → handler_callable(**args) -> dict + """ +``` + +**ToolDefinition schema** (matches existing `tool_definitions.py` pattern): +```python +{ + "name": str, # MUST match dict key + "description": str, # Human description + "inputSchema": { # JSON Schema object + "type": "object", + "properties": { ... }, + "required": [ ... ] + } +} +``` + +--- + +## Part 2: Graph Schema Extensions + +New node labels and relationship types added by each plugin to the existing CGC graph. +All new nodes carry a `source` property identifying their origin layer. + +--- + +### OTEL Plugin Nodes + +#### Service + +Represents a named microservice observed in telemetry data. + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `name` | string | ✅ | UNIQUE | Service name from OTEL resource attributes | +| `version` | string | ❌ | — | Service version if reported | +| `environment` | string | ❌ | — | Environment tag (prod, staging, dev) | +| `source` | string | ✅ | — | Always `"runtime_otel"` | + +**Constraint**: `UNIQUE (s.name)` — service names are globally unique identifiers. + +--- + +#### Trace + +Represents a single distributed trace (root span + all children). + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `trace_id` | string | ✅ | UNIQUE | 128-bit trace ID as hex string | +| `root_span_id` | string | ✅ | — | Span ID of the root span | +| `started_at` | long | ✅ | — | Start time in Unix milliseconds | +| `duration_ms` | long | ✅ | — | Total trace duration in milliseconds | +| `source` | string | ✅ | — | Always `"runtime_otel"` | + +**Constraint**: `UNIQUE (t.trace_id)`. + +--- + +#### Span + +Represents a single operation within a trace. + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `span_id` | string | ✅ | UNIQUE | 64-bit span ID as hex string | +| `trace_id` | string | ✅ | INDEX | Parent trace ID (for batch queries) | +| `name` | string | ✅ | — | Span name | +| `service` | string | ✅ | — | Source service name | +| `kind` | string | ✅ | — | `SERVER`, `CLIENT`, `INTERNAL`, `PRODUCER`, `CONSUMER` | +| `class_name` | string | ❌ | INDEX | PHP: `code.namespace` attribute | +| `method_name` | string | ❌ | — | PHP: `code.function` attribute | +| `http_method` | string | ❌ | — | HTTP verb for SERVER/CLIENT spans | +| `http_route` | string | ❌ | INDEX | Route template (e.g. `/api/orders`) | +| `db_statement` | string | ❌ | — | SQL/query statement for DB spans | +| `duration_ms` | long | ✅ | — | Span duration in milliseconds | +| `status` | string | ✅ | — | `OK`, `ERROR`, `UNSET` | +| `source` | string | ✅ | — | Always `"runtime_otel"` | + +**Constraints**: `UNIQUE (s.span_id)`. +**Indexes**: `(s.trace_id)`, `(s.class_name)`, `(s.http_route)`. + +--- + +### Xdebug Plugin Nodes + +#### StackFrame + +Represents a single frame in a PHP execution call stack captured via DBGp. + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `frame_id` | string | ✅ | UNIQUE | Hash of `class_name::method_name:file_path:line` | +| `class_name` | string | ✅ | INDEX | PHP class name (fully qualified) | +| `method_name` | string | ✅ | — | PHP method name | +| `fqn` | string | ✅ | INDEX | `ClassName::methodName` for correlation | +| `file_path` | string | ✅ | — | Absolute file path from DBGp | +| `line` | int | ✅ | — | Line number | +| `depth` | int | ✅ | — | Call stack depth (0 = top) | +| `chain_hash` | string | ✅ | INDEX | Deduplication hash of the full call chain | +| `observation_count` | int | ✅ | — | Number of times this chain was observed | +| `source` | string | ✅ | — | Always `"runtime_xdebug"` | + +**Constraint**: `UNIQUE (sf.frame_id)`. +**Index**: `(sf.fqn)` for `RESOLVES_TO` correlation lookups. + +--- + +## Part 3: Graph Relationship Extensions + +New relationships added by the plugins. Existing CGC relationships are not modified. + +--- + +### OTEL Relationships + +| Relationship | From → To | Properties | Description | +|---|---|---|---| +| `CHILD_OF` | Span → Span | — | Parent-child span hierarchy | +| `PART_OF` | Span → Trace | — | Span belongs to trace | +| `ORIGINATED_FROM` | Trace → Service | — | Trace started in service | +| `CALLS_SERVICE` | Span → Service | — | Cross-service call (CLIENT spans only) | +| `CORRELATES_TO` | Span → Method | `confidence: "fqn_match"` | Runtime → static correlation | + +--- + +### Xdebug Relationships + +| Relationship | From → To | Properties | Description | +|---|---|---|---| +| `CALLED_BY` | StackFrame → StackFrame | `depth_diff: int` | Call chain (child called by parent) | +| `RESOLVES_TO` | StackFrame → Method | `match_type: "fqn_exact"` | Frame → static method node | + +--- + +## Part 4: Schema Migration + +All new node labels and relationship types are additive — they do not modify existing +CGC node labels (`File`, `Class`, `Method`, `Function`) or existing relationships +(`CALLS`, `IMPORTS`, `INHERITS`, `DEFINES`). + +Required Cypher initialization statements (added to `config/neo4j/init.cypher`): + +```cypher +-- OTEL constraints & indexes +CREATE CONSTRAINT service_name IF NOT EXISTS FOR (s:Service) REQUIRE s.name IS UNIQUE; +CREATE CONSTRAINT trace_id IF NOT EXISTS FOR (t:Trace) REQUIRE t.trace_id IS UNIQUE; +CREATE CONSTRAINT span_id IF NOT EXISTS FOR (s:Span) REQUIRE s.span_id IS UNIQUE; +CREATE INDEX span_trace IF NOT EXISTS FOR (s:Span) ON (s.trace_id); +CREATE INDEX span_class IF NOT EXISTS FOR (s:Span) ON (s.class_name); +CREATE INDEX span_route IF NOT EXISTS FOR (s:Span) ON (s.http_route); + +-- Xdebug constraints & indexes +CREATE CONSTRAINT frame_id IF NOT EXISTS FOR (sf:StackFrame) REQUIRE sf.frame_id IS UNIQUE; +CREATE INDEX frame_fqn IF NOT EXISTS FOR (sf:StackFrame) ON (sf.fqn); +``` diff --git a/specs/001-cgc-plugin-extension/plan.md b/specs/001-cgc-plugin-extension/plan.md new file mode 100644 index 00000000..1cbcfc5f --- /dev/null +++ b/specs/001-cgc-plugin-extension/plan.md @@ -0,0 +1,203 @@ +# Implementation Plan: CGC Plugin Extension System + +**Branch**: `001-cgc-plugin-extension` | **Date**: 2026-03-14 | **Spec**: [spec.md](spec.md) +**Input**: Feature specification from `specs/001-cgc-plugin-extension/spec.md` + +## Summary + +Extend CodeGraphContext with a Python entry-points plugin system that allows independently +installable packages to contribute CLI commands (Typer) and MCP tools without modifying +CGC core. Two first-party plugins ship with the extension: an OTEL span processor (runtime +intelligence) and an Xdebug DBGp listener (dev-time stack traces). A shared GitHub Actions +matrix CI/CD pipeline builds and publishes versioned Docker images for each plugin service. +A hosted MCP server container image exposes a plain JSON-RPC HTTP endpoint for remote AI +clients without requiring stdio transport. All plugin data flows into the existing +Neo4j/FalkorDB graph, enabling cross-layer queries across static code and runtime execution. + +## Technical Context + +**Language/Version**: Python 3.10+ (constitutional constraint) +**Primary Dependencies**: +- Plugin system: `importlib.metadata` (stdlib), `packaging>=23.0` (version constraint checking) +- OTEL plugin: `grpcio>=1.57.0`, `opentelemetry-proto>=0.43b0`, `opentelemetry-sdk>=1.20.0` +- Xdebug plugin: stdlib only (`socket`, `xml.etree.ElementTree`, `hashlib`) +- HTTP transport: `uvicorn>=0.27.0`, `starlette>=0.36.0` (already dependencies of core) +- All plugins: `typer[all]>=0.9.0`, `neo4j>=5.15.0` (shared with core) + +**Storage**: Neo4j (production) / FalkorDB (default) — same shared instance as CGC core; +new additive node labels and relationships per `data-model.md` + +**Testing**: pytest + pytest-asyncio; existing `tests/run_tests.sh` extended with +`tests/unit/plugin/`, `tests/integration/plugin/`, `tests/e2e/plugin/` + +**Target Platform**: Linux server (Docker containers); Kubernetes compatible (no host +networking, env-var-only config) + +**Project Type**: Python library + CLI extensions + containerised microservices + +**Performance Goals**: +- CGC startup with all plugins: ≤ 15 seconds +- Span data queryable within 10 seconds of request completion under normal load +- Plugin load failure: ≤ 5-second timeout per plugin (SIGALRM) +- MCP HTTP server: `/healthz` passes within 5 seconds of readiness + +**Constraints**: +- Plugin failures MUST NOT crash CGC core (strict isolation) +- No credentials baked into container images +- `./tests/run_tests.sh fast` MUST pass after each phase +- Xdebug plugin MUST default to disabled (security: TCP listener) +- HTTP transport: plain JSON-RPC request/response (no SSE/streaming) +- HTTP transport: single-process async (uvicorn default asyncio event loop) +- HTTP transport: no application-level auth — defer to reverse proxy/network controls +- `/healthz` returns 503 with `{"status":"unhealthy"}` when Neo4j unreachable + +**Scale/Scope**: 2 plugin packages, 1 shared CI/CD pipeline, 5 container services +(including hosted MCP server), 3 sample applications (PHP/Laravel, Python/FastAPI, +TypeScript/Express) + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +| Principle | Status | Evidence | +|---|---|---| +| **I. Graph-First Architecture** | ✅ PASS | All plugin output (spans, stack frames) writes to the graph as typed nodes + relationships per `data-model.md`. No flat data structures. Graph schema is the output target for both plugins. | +| **II. Dual Interface — CLI + MCP** | ✅ PASS | Each plugin MUST contribute both CLI commands AND MCP tools (per plugin interface contract). The plugin contract enforces parity by design. US6 adds HTTP transport for MCP, extending accessibility without changing the interface. | +| **III. Testing Pyramid** | ✅ PASS | Plugin packages include `tests/unit/` and `tests/integration/`. `./tests/run_tests.sh fast` is extended to cover plugin directories. E2E tests cover the full plugin lifecycle. Tests written and observed to FAIL before implementation (Red-Green-Refactor). | +| **IV. Multi-Language Parser Parity** | ✅ PASS | No new language parsers introduced. Runtime nodes carry `source` property (`"runtime_otel"`, `"runtime_xdebug"`) that distinguish origin layers without breaking existing cross-language queries. | +| **V. Simplicity** | ⚠️ JUSTIFIED | Plugin registry is an abstraction. Justified because: (a) the feature requires extensibility without forking core — a non-negotiable requirement; (b) `importlib.metadata` entry-points is Python stdlib — minimal abstraction; (c) without a registry, adding each plugin would require modifying `server.py` and `cli/main.py` permanently, producing a worse monolith. See Complexity Tracking below. | + +*Post-Phase 1 re-check*: ✅ Design satisfies all five principles. No new violations introduced. + +## Project Structure + +### Documentation (this feature) + +```text +specs/001-cgc-plugin-extension/ +├── plan.md # This file +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ +│ ├── plugin-interface.md # Plugin author contract +│ └── cicd-pipeline.md # CI/CD service registration contract +└── tasks.md # Phase 2 output (/speckit.tasks command) +``` + +### Source Code (repository root) + +```text +# Core CGC modifications (existing package) +src/codegraphcontext/ +├── plugin_registry.py # NEW: PluginRegistry class, isolation wrappers +├── http_transport.py # NEW: Plain JSON-RPC HTTP transport (uvicorn + starlette) +├── cli/ +│ └── main.py # MODIFIED: --transport option, plugin loading at startup +└── server.py # MODIFIED: extract handle_request(), plugin tool loading + +# New plugin packages +plugins/ +├── cgc-plugin-otel/ +│ ├── pyproject.toml +│ ├── Dockerfile +│ └── src/cgc_plugin_otel/ +│ ├── __init__.py # PLUGIN_METADATA +│ ├── cli.py # get_plugin_commands() → ("otel", typer.Typer) +│ ├── mcp_tools.py # get_mcp_tools(), get_mcp_handlers() +│ ├── receiver.py # gRPC OTLP receiver (grpcio + opentelemetry-proto) +│ ├── span_processor.py # PHP attribute extraction + correlation logic +│ └── neo4j_writer.py # Async batch writer with dead-letter queue +│ +└── cgc-plugin-xdebug/ + ├── pyproject.toml + ├── Dockerfile + └── src/cgc_plugin_xdebug/ + ├── __init__.py # PLUGIN_METADATA + ├── cli.py # get_plugin_commands() → ("xdebug", typer.Typer) + ├── mcp_tools.py # get_mcp_tools(), get_mcp_handlers() + ├── dbgp_server.py # TCP DBGp listener + XML stack frame parser + └── neo4j_writer.py # Frame upsert + CALLED_BY chain + deduplication + +# Tests (additions to existing structure) +tests/ +├── unit/ +│ ├── plugin/ +│ │ ├── test_plugin_registry.py # PluginRegistry unit tests (mocked) +│ │ ├── test_otel_processor.py # Span extraction logic +│ │ └── test_xdebug_parser.py # DBGp XML parsing + deduplication +│ └── test_http_transport.py # HTTP transport unit tests (US6) +├── integration/ +│ ├── plugin/ +│ │ ├── test_plugin_load.py # Plugin discovery + load integration +│ │ └── test_otel_integration.py # OTLP receive → graph write +│ └── test_http_transport_integration.py # HTTP transport integration (US6) +└── e2e/ + ├── plugin/ + │ └── test_plugin_lifecycle.py # Full install/use/uninstall user journey + └── test_mcp_container.py # MCP container E2E test (US6) + +# CI/CD +.github/ +├── services.json # NEW: service list for Docker matrix +└── workflows/ + ├── docker-publish.yml # MODIFIED: matrix over services.json + └── test-plugins.yml # NEW: per-plugin fast test suite + +# Deployment +docker-compose.yml # MODIFIED: add otel services +docker-compose.dev.yml # MODIFIED: add xdebug service +config/ +├── otel-collector/ +│ └── config.yaml # NEW: OTel Collector pipeline config +└── neo4j/ + └── init.cypher # MODIFIED: add plugin schema constraints + +Dockerfile.mcp # NEW: hosted MCP server image + +k8s/ +├── cgc-mcp/ +│ ├── deployment.yaml # NEW: MCP server deployment +│ └── service.yaml # NEW: MCP server ClusterIP service +└── cgc-plugin-otel/ + ├── deployment.yaml + └── service.yaml + +# Sample applications (US5) +samples/ +├── docker-compose.yml # Extends plugin-stack + 3 sample apps +├── smoke-all.sh # Automated 6-phase validation script +├── README.md # Full walkthrough with architecture diagram +├── KNOWN-LIMITATIONS.md # FQN correlation gap documentation +├── php-laravel/ +│ ├── Dockerfile # PHP 8.3 + OTEL auto-instrumentation + Xdebug +│ ├── composer.json +│ ├── README.md +│ └── app/ # Controllers, Services, Repositories +├── python-fastapi/ +│ ├── Dockerfile # Python 3.12 + opentelemetry-instrument + uvicorn +│ ├── requirements.txt +│ ├── README.md +│ └── app/ # FastAPI routers, services, repositories +└── ts-express-gateway/ + ├── Dockerfile # Multi-stage TS build → Node runtime + ├── package.json + ├── tsconfig.json + ├── README.md + └── src/ # Express routes, services (HTTP proxy) +``` + +**Structure Decision**: Multi-package layout under `plugins/` with independent +`pyproject.toml` per plugin. This matches the research recommendation (R-010) and is the +standard Python ecosystem pattern for monorepo plugin families. Plugin packages are +installable independently (`pip install codegraphcontext[otel]`) or via optional extras +in the root `pyproject.toml`. Each plugin that exposes a container service has its own +`Dockerfile` in the plugin directory. + +## Complexity Tracking + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|---|---|---| +| Plugin registry abstraction | Feature explicitly requires extensibility without forking core. Three current plugins + third-party extensibility require a clean registration boundary. | Hardcoding plugins in `server.py`/`main.py` defeats the extensibility requirement entirely. There is no simpler path to the stated goal. | +| gRPC server in OTEL plugin | OTLP protocol uses gRPC. The Python opentelemetry-sdk is tracer-side only and cannot act as a receiver. | Pure HTTP OTLP would require the same gRPC-level effort and provides less tooling ecosystem support. The OTel Collector (sidecar) already handles the edge; gRPC is the right interface for collector → processor. | +| Multiple new graph node types | Runtime layers produce genuinely different data (spans, frames). Reusing existing `Method`/`Class` nodes for runtime data would corrupt the static layer. | Cannot collapse runtime nodes into static nodes — they represent different semantic things (observed execution vs. declared code). The `source` property differentiates them without schema explosion. | diff --git a/specs/001-cgc-plugin-extension/quickstart.md b/specs/001-cgc-plugin-extension/quickstart.md new file mode 100644 index 00000000..d2e1bf83 --- /dev/null +++ b/specs/001-cgc-plugin-extension/quickstart.md @@ -0,0 +1,231 @@ +# Quickstart: CGC Plugin Extension System + +**Feature**: 001-cgc-plugin-extension +**Audience**: Developers setting up CGC-X locally and contributors building plugins + +--- + +## Prerequisites + +- Python 3.10+ +- pip / virtualenv +- Docker + Docker Compose (for container services) +- A running Neo4j instance (or use the provided docker-compose) + +--- + +## 1. Run the Full CGC-X Stack (Docker Compose) + +The fastest way to get the full stack running: + +```bash +# Clone the repo +git clone https://github.com/CodeGraphContext/CodeGraphContext +cd CodeGraphContext + +# Copy and configure environment +cp .env.example .env +# Edit .env: set NEO4J_PASSWORD and DOMAIN + +# Start the plugin stack +docker compose -f docker-compose.plugin-stack.yml up -d + +# Start with Xdebug listener (dev profile — adds xdebug service) +docker compose -f docker-compose.plugin-stack.yml -f docker-compose.dev.yml up -d +``` + +**Services started**: +| Service | URL / Port | Purpose | +|---|---|---| +| Neo4j | bolt://localhost:7687 | Shared graph database | +| CGC core | MCP at localhost:8080 | Static code indexing | +| CGC MCP (HTTP) | http://localhost:8045 | Hosted MCP server (JSON-RPC) | +| OTEL plugin | gRPC at localhost:5317 | Runtime span ingestion | +| Xdebug listener (dev) | TCP at localhost:9003 | Dev-time stack traces | + +--- + +## 2. Install CGC with Plugins (Python — Development Mode) + +For local development or when running without Docker: + +```bash +# Create a virtual environment +python -m venv .venv +source .venv/bin/activate + +# Install CGC core + all plugins in editable mode +pip install -e . +pip install -e plugins/cgc-plugin-otel +pip install -e plugins/cgc-plugin-xdebug + +# Verify plugins loaded +cgc --help +# Should show: otel, xdebug command groups alongside built-in commands +``` + +**Install specific plugins only** (production use): +```bash +pip install codegraphcontext[otel] # core + OTEL plugin +pip install codegraphcontext[all] # core + all plugins +``` + +--- + +## 3. Verify Plugin Discovery + +```bash +# List all loaded plugins +cgc plugin list + +# Expected output: +# ✓ cgc-plugin-otel v0.1.0 3 tools (otel_query_spans, otel_list_services, otel_cross_layer_query) 3 commands +# ✓ cgc-plugin-xdebug v0.1.0 2 tools (xdebug_list_chains, xdebug_query_chain) 3 commands (dev only) +``` + +--- + +## 4. Index a Repository + +```bash +# Index a local PHP/Laravel project +cgc index /path/to/your/laravel-project + +# Verify nodes were created +cgc query "MATCH (c:Class) RETURN c.name LIMIT 5" +``` + +--- + +## 5. Enable Runtime Intelligence (OTEL Plugin) + +Add to your Laravel application's `.env`: +```ini +OTEL_PHP_AUTOLOAD_ENABLED=true +OTEL_SERVICE_NAME=my-service +OTEL_TRACES_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +``` + +Send a request to your application. Verify spans appear in the graph: +```bash +cgc otel query-spans --route /api/orders --limit 5 +``` + +Or via MCP tool: +```json +{ + "tool": "otel_query_spans", + "arguments": {"http_route": "/api/orders", "limit": 5} +} +``` + +--- + +## 6. Enable Dev-Time Traces (Xdebug Plugin) + +Ensure your PHP application has Xdebug installed with these settings: +```ini +xdebug.mode=debug,trace +xdebug.client_host=localhost ; or Docker host IP +xdebug.client_port=9003 +xdebug.start_with_request=trigger +``` + +Trigger a trace by setting the `XDEBUG_TRIGGER` cookie in your browser, then query: +```bash +cgc xdebug list-chains --limit 10 +``` + +--- + +## 7. Cross-Layer Query Example + +After indexing code + collecting runtime spans, run this cross-layer query to find +static code never observed at runtime: + +```bash +cgc query " +MATCH (m:Method) +WHERE NOT EXISTS { MATCH (m)<-[:CORRELATES_TO]-(:Span) } + AND NOT EXISTS { MATCH (m)<-[:RESOLVES_TO]-(:StackFrame) } + AND m.fqn IS NOT NULL +RETURN m.fqn, m.class_name +ORDER BY m.class_name, m.fqn +LIMIT 20 +" +``` + +--- + +## 8. Run the Hosted MCP Server (HTTP Transport) + +Deploy the MCP server as a network service accessible to multiple AI clients: + +```bash +# Start the MCP server with HTTP transport via Docker Compose +docker compose -f docker-compose.plugin-stack.yml up -d cgc-mcp neo4j + +# Verify the server is healthy +curl http://localhost:8045/healthz +# Expected: {"status":"ok","tools":N} + +# Send an MCP request (initialize) +curl -X POST http://localhost:8045/mcp \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' + +# List available tools +curl -X POST http://localhost:8045/mcp \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' +``` + +**Or run locally without Docker**: +```bash +# Start MCP server in HTTP mode (default port 8045) +cgc mcp start --transport http + +# Configure AI clients to connect to http://localhost:8045/mcp +``` + +**Notes**: +- No application-level authentication — use a reverse proxy or network controls for access management +- CORS is configurable via `CGC_CORS_ORIGIN` env var (default: `*`) +- `/healthz` returns 503 when Neo4j is unreachable + +--- + +## 9. Build and Push Container Images + +```bash +# Trigger a release build (creates all plugin images) +git tag v0.1.0 +git push origin v0.1.0 +# GitHub Actions automatically builds and pushes: +# ghcr.io//cgc-core:0.1.0 +# ghcr.io//cgc-plugin-otel:0.1.0 + +# Monitor at: github.com//CodeGraphContext/actions +``` + +--- + +## 10. Write Your Own Plugin + +```bash +# Use the plugin scaffold (coming in a future task) +# For now, copy the stub plugin: +cp -r plugins/cgc-plugin-stub plugins/cgc-plugin-myname + +# Edit pyproject.toml: change name, entry points, dependencies +# Edit src/cgc_plugin_myname/__init__.py: update PLUGIN_METADATA +# Implement cli.py and mcp_tools.py following the plugin-interface.md contract +# Install and test: +pip install -e plugins/cgc-plugin-myname +cgc plugin list # Should show your plugin +``` + +See `specs/001-cgc-plugin-extension/contracts/plugin-interface.md` for the full +plugin contract specification. diff --git a/specs/001-cgc-plugin-extension/research.md b/specs/001-cgc-plugin-extension/research.md new file mode 100644 index 00000000..796b1a16 --- /dev/null +++ b/specs/001-cgc-plugin-extension/research.md @@ -0,0 +1,243 @@ +# Research: CGC Plugin Extension System + +**Feature**: 001-cgc-plugin-extension +**Date**: 2026-03-14 +**Status**: Complete — all NEEDS CLARIFICATION resolved + +--- + +## R-001: Plugin Discovery Mechanism + +**Decision**: Use Python `importlib.metadata.entry_points()` (stdlib, Python 3.10+) with +two named groups: `cgc_cli_plugins` and `cgc_mcp_plugins`. + +**Rationale**: Entry points are the Python ecosystem's standard plugin discovery contract. +They require zero runtime overhead beyond package installation — no config files, no +manual registration, no import scanning. Every tool in the Python ecosystem (pytest, +flask, flake8) uses this pattern. It is stdlib in Python 3.10+ (no extra dependency). + +**How it works**: +- Plugin packages declare entry points in their own `pyproject.toml` +- `pip install` indexes entry point metadata into the environment +- CGC calls `entry_points(group="cgc_cli_plugins")` at startup to discover all installed + plugins that contribute CLI commands +- CGC calls `entry_points(group="cgc_mcp_plugins")` to discover MCP tool contributors +- Each group resolves to a callable that CGC invokes to receive the plugin's registration + +**Alternatives considered**: +- Filesystem scanning (explicit plugin dir) — more brittle, non-standard, breaks with + virtual environments +- Config file listing plugins — requires manual edits (violates FR-002 "zero edits") +- Import path hooks — too low-level, fragile, hard to debug + +--- + +## R-002: CLI Plugin Interface + +**Decision**: Each CLI plugin entry point resolves to a function +`get_plugin_commands() -> Tuple[str, typer.Typer]` that returns a +`(command_group_name, typer_app_instance)` tuple. CGC calls +`app.add_typer(plugin_app, name=cmd_name)` for each loaded plugin. + +**Rationale**: Typer's `add_typer()` is the idiomatic way to compose command groups. The +pattern requires the plugin to own its Typer app (clean separation), and CGC to own the +top-level `app` (clean host). Returning a tuple rather than a dict is simpler for the +common case (one command group per plugin) and is consistently typed. + +**Startup sequence**: +``` +CLI main.py imports → PluginRegistry discovers cgc_cli_plugins entries → +calls each get_plugin_commands() → app.add_typer() for each → Typer starts +``` + +**Alternatives considered**: +- Plugin directly calls `app.add_typer()` — creates bidirectional coupling; plugin + imports core at registration time which can cause circular imports +- Plugin returns a Click group — Typer wraps Click but mixing levels is error-prone; + Typer's add_typer is cleaner + +--- + +## R-003: MCP Plugin Interface + +**Decision**: Each MCP plugin entry point resolves to a function +`get_mcp_tools(server_context: dict) -> dict[str, ToolDefinition]` that returns a +mapping of tool name → tool definition dict (same schema as core `tool_definitions.py`). +CGC's `MCPServer._load_plugin_tools()` merges these into its tools manifest and routes +calls via a unified `handle_tool_call()` dispatcher. + +**Server context passed to plugins** (minimal, read-only intent): +```python +{ + "db_manager": self.db_manager, # shared database connection + "version": "x.y.z", # CGC core version string +} +``` + +**Rationale**: Plugins receive the `db_manager` so they can share the existing database +connection rather than opening independent connections (violating the constitution's +single-database principle). Passing only what is needed (not `self`) prevents plugins +from calling internal server methods they shouldn't access. + +**Tool handler registration**: The plugin's `get_mcp_tools()` return value maps +tool names to JSON Schema definitions. The plugin ALSO registers handler callables +in a separate `get_mcp_handlers()` function (or combined in a single object). The +server stores handlers in `self.plugin_tool_handlers` dict and routes calls there +before checking built-in handlers. + +**Alternatives considered**: +- Subclass MCPServer per plugin — couples plugin to server implementation; not viable + for third-party plugins +- Plugin monkey-patches server — completely unsafe and untestable +- gRPC plugin protocol — overkill for in-process plugins; entry-points are sufficient + +--- + +## R-004: Plugin Version Compatibility + +**Decision**: Each plugin's `__init__.py` declares `PLUGIN_METADATA` dict with a +`cgc_version_constraint` key using PEP-440 version specifier syntax (e.g. +`">=0.3.0,<1.0"`). CGC's `PluginRegistry` validates this against the installed +`codegraphcontext` package version using `packaging.specifiers.SpecifierSet`. + +**On mismatch**: plugin is skipped with a WARNING log; all compatible plugins still load; +no error is raised to the user unless zero plugins load. + +**Rationale**: `packaging` is already an indirect dependency of pip and is present in +all virtual environments. PEP-440 specifiers are the Python standard for version +constraints. Soft-fail (warn, skip) rather than hard-fail ensures partial plugin +ecosystems remain usable. + +**Alternatives considered**: +- Semver-only checking — PEP-440 is a superset and already the ecosystem standard +- No version checking — risks silent breakage when core APIs change + +--- + +## R-005: Plugin Isolation (Error Containment) + +**Decision**: Wrap each plugin load in a `try/except Exception` block. Use a +`PluginRegistry` class that catches `ImportError`, `AttributeError`, `TimeoutError`, and +generic `Exception` at each stage (import, metadata read, command/tool registration). +A broken plugin logs an error and sets `failed_plugins[name] = reason`; it NEVER +propagates an exception to the host process. + +**Timeout**: On Unix, `signal.SIGALRM` with a 5-second timeout prevents hanging imports. +(Windows lacks SIGALRM — on Windows, timeout is skipped with a warning.) + +**Startup summary**: After all plugins are processed, CGC logs: +``` +CGC started with 19 built-in tools and 4 plugin tools (1 plugin failed). + ✓ cgc-plugin-otel 4 tools + ✗ cgc-plugin-xdebug SKIPPED: missing dependency 'dbgp' +``` + +**Rationale**: The spec requires (FR-003) that plugin failures do not prevent CGC core +from starting. Isolation at the `PluginRegistry` boundary is the cleanest enforcement. + +--- + +## R-006: OTEL Span Receiver Architecture + +**Decision**: Deploy the standard OpenTelemetry Collector (`otel/opentelemetry-collector-contrib`) +as a sidecar. It receives OTLP from applications and forwards to the OTEL plugin service +via OTLP gRPC. The plugin service implements a Python gRPC server using `grpcio` + +`opentelemetry-proto` protobuf definitions. + +**Rationale**: The Python `opentelemetry-sdk` is tracer-side only — it cannot act as an +OTLP gRPC receiver endpoint. Using the official OTel Collector as a sidecar provides +batching, retry, filtering, and sampling for free before spans reach the custom processor. +This is the established production pattern (used by Datadog, Honeycomb, Jaeger agents). + +**Key packages**: +- `grpcio>=1.57.0` — gRPC server implementation +- `opentelemetry-proto>=0.43b0` — generated protobuf/gRPC classes +- `neo4j>=5.15.0` — async Python driver + +**Write pattern**: Async batch writer using `asyncio.Queue` with configurable +`max_batch_size=100` and `max_wait_ms=5000`. MERGE on `(trace_id, span_id)` for +idempotency. Dead-letter queue for resilience when Neo4j is temporarily unavailable. + +**Alternatives considered**: +- Pure Python OTLP HTTP receiver (no gRPC) — simpler but less efficient; OTel Collector + already handles the gRPC ↔ HTTP translation if needed +- Direct OTLP from app → Python service — fragile; the Collector adds resilience + +--- + +## R-007: Xdebug DBGp Listener + +**Decision**: Implement a minimal TCP server using Python's stdlib `socket` and +`xml.etree.ElementTree` modules. No external DBGp library is required — the protocol +is XML over TCP and is simple enough to implement directly. + +**Protocol flow**: PHP Xdebug connects → init packet received → `run` command sent → +on each breakpoint, send `stack_get` → parse XML response → upsert `StackFrame` nodes +and `CALLED_BY` edges → send `run` → repeat until connection closes. + +**Deduplication**: Hash the call chain (`sha256(class::method|...) [:16]`) with an +LRU cache (size configurable, default 10,000). If hash seen recently, skip upsert. +This prevents the same execution path from creating duplicate graph structure. + +**Dev-only deployment**: The Xdebug plugin starts its TCP listener only when enabled +(`CGC_PLUGIN_XDEBUG_ENABLED=true`). In production Docker Compose, the `xdebug` service +is absent from the default compose file; it exists only in `docker-compose.dev.yml`. + +**Rationale**: Xdebug is a dev tool. Running a DBGp listener in production is a security +risk. The plugin MUST default to disabled and require explicit opt-in. + +--- + +## R-008: CI/CD Pipeline Architecture + +**Decision**: GitHub Actions matrix strategy with `fail-fast: false`. Services defined +in `.github/services.json` as a JSON array. Shared logic for checkout, Docker login, +version tag extraction, and metadata is in the matrix job — matrix jobs inherit shared +steps via `needs` dependencies from a `setup` job that outputs the services matrix. + +**Tagging strategy**: `docker/metadata-action@v5` generates: +- Semver tags from git tags (`v1.2.3` → `1.2.3`, `1.2`, `1`) +- `latest` on default branch pushes +- Branch name tags for non-default branches + +**Health check**: After `docker/build-push-action@v5` with `push: false` (local build), +load image and run a service-specific smoke test. Only push if smoke test passes. + +**Adding a new service**: Add one JSON object to `.github/services.json`. Zero workflow +logic changes. + +**Key action versions** (current as of research date): +- `docker/setup-buildx-action@v3` +- `docker/build-push-action@v5` +- `docker/login-action@v3` +- `docker/metadata-action@v5` + +**Alternatives considered**: +- One workflow file per service — massive duplication, violates FR-030 +- Reusable workflow (workflow_call) — more complex than needed; matrix is sufficient + +--- + +## R-009: Monorepo Package Layout + +**Decision**: Plugin packages live in `plugins/` subdirectory, each as an independently +installable Python package with its own `pyproject.toml`. Plugin services that run as +standalone containers (OTEL, Xdebug) also have a `Dockerfile` in their directory. + +**Development installation**: +```bash +pip install -e . # CGC core +pip install -e plugins/cgc-plugin-otel +pip install -e plugins/cgc-plugin-xdebug +``` + +After this, `cgc --help` shows plugin commands automatically. + +**Production installation** (users who want only specific plugins): +```bash +pip install codegraphcontext # Core only +pip install codegraphcontext[otel] # Core + OTEL plugin (via extras) +pip install codegraphcontext[all] # Core + all plugins +``` + +This is achieved by declaring plugins as optional extras in the root `pyproject.toml`. diff --git a/specs/001-cgc-plugin-extension/spec.md b/specs/001-cgc-plugin-extension/spec.md new file mode 100644 index 00000000..5591b8c4 --- /dev/null +++ b/specs/001-cgc-plugin-extension/spec.md @@ -0,0 +1,478 @@ +# Feature Specification: CGC Plugin Extension System + +**Feature Branch**: `001-cgc-plugin-extension` +**Created**: 2026-03-14 +**Status**: Draft +**Input**: Based on research in `cgc-extended-spec.md` — extend CGC to support runtime +runtime intelligence layers via a plugin/addon pattern for CLI and MCP, with a +common CI/CD pipeline for Docker/K8s images. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Plugin Extensibility Foundation (Priority: P1) + +A CGC contributor or third-party developer wants to extend CGC with new capabilities +(new data sources, new MCP tools, new CLI commands) without modifying the core CGC +codebase. They build a self-contained addon package that declares its CLI commands and +MCP tools, publishes it separately, and CGC discovers and loads it automatically when +installed. + +**Why this priority**: All other stories depend on a functioning plugin system. Without +the foundation, the runtime, and CI/CD stories cannot be independently developed +or released. This is the architectural backbone that makes the project composable. + +**Independent Test**: Install CGC core alone and verify it starts correctly. Then install +a minimal stub plugin; verify CGC discovers the plugin, the plugin's CLI command appears +in `cgc --help`, and its MCP tool appears in the MCP tool listing — without any changes +to core CGC code. + +**Acceptance Scenarios**: + +1. **Given** CGC core is installed without any plugins, **When** a user runs the CGC + CLI, **Then** only built-in core commands appear and no plugin-related errors occur. +2. **Given** a plugin package is installed in the same environment, **When** CGC starts, + **Then** the plugin's CLI commands and MCP tools are automatically available alongside + core capabilities. +3. **Given** a plugin is installed, **When** the plugin is uninstalled, **Then** CGC + starts cleanly without the plugin's commands or tools and without crashing. +4. **Given** two plugins are installed simultaneously, **When** CGC starts, **Then** + both plugins' commands and tools are available with no naming conflicts for distinct + plugins. +5. **Given** a plugin declares an incompatible version constraint, **When** CGC loads + plugins, **Then** the incompatible plugin is skipped with a clear warning stating the + version mismatch, and all compatible plugins still load. + +--- + +### User Story 2 - Runtime Intelligence via OTEL Plugin (Priority: P2) + +A backend developer running a PHP/Laravel application wants to understand what code +actually executes at runtime, not just what the static graph shows. They enable the +OTEL plugin, point their application's telemetry at the CGC OTEL endpoint, and can then +ask their AI assistant questions that combine runtime call data with static code +structure — for example, "which methods were called in the last hour that have no test +coverage" or "show the full execution path for a POST /api/orders request." + +**Why this priority**: The OTEL plugin is the highest-value runtime layer. It is +non-invasive (standard OTEL instrumentation already used in many projects), production- +safe, and delivers cross-layer queries immediately once spans flow into the graph. + +**Independent Test**: Start CGC with the OTEL plugin enabled. Send a sample trace (or +synthetic span payload) to the plugin's ingestion endpoint. Verify that the graph now +contains runtime nodes linked to static code nodes from a pre-indexed repository, and +that a cross-layer query returns meaningful results. + +**Acceptance Scenarios**: + +1. **Given** the OTEL plugin is enabled and a repository is indexed, **When** a + telemetry-instrumented application sends request traces, **Then** runtime call data + appears in the graph within 10 seconds of the request completing. +2. **Given** runtime nodes exist in the graph, **When** an AI assistant queries + "which methods ran during request X", **Then** the MCP tool returns a linked result + showing both the runtime call chain and the corresponding static code nodes. +3. **Given** a cross-service call occurs (service A calls service B), **When** spans + from both services are received, **Then** the graph contains an edge connecting the + two services and the call is queryable as a single path. +4. **Given** health-check or noise spans are received, **When** ingestion runs, **Then** + noise spans are filtered out and do not pollute the graph. +5. **Given** the OTEL plugin is disabled or not installed, **When** CGC starts, **Then** + no OTEL-related commands or tools appear and the core graph is unaffected. + +--- + +### User Story 3 - Development Traces via Xdebug Plugin (Priority: P3) + +A PHP developer debugging a complex feature wants method-level execution traces that +capture exactly which concrete class implementations ran (not just the interface), which +is information OTEL spans don't always provide. They enable the Xdebug listener plugin +in their development environment and selectively trigger traces for specific requests. +The resulting call-chain graph is linked back to CGC's static code nodes so they can +navigate from "what ran" to "where it's defined." + +**Why this priority**: This plugin is development/staging-only and requires Xdebug on +the target application, limiting its audience. It delivers deep, precise traces but is +not needed in production. It depends on the plugin foundation (P1) but is independent +of the OTEL plugin (P2). + +**Independent Test**: Start CGC with the Xdebug plugin enabled in development mode. +Trigger an Xdebug connection from a PHP process. Verify that stack frame nodes appear +in the graph linked to the corresponding static method nodes from a pre-indexed +repository. + +**Acceptance Scenarios**: + +1. **Given** the Xdebug plugin is enabled and a repository is indexed, **When** a PHP + process triggers an Xdebug trace, **Then** the full call stack appears in the graph + as linked frame nodes within 5 seconds of the trace completing. +2. **Given** the same call chain occurs repeatedly, **When** ingestion processes the + repeated traces, **Then** the graph contains deduplicated nodes (no duplicate chains) + and the repetition count is reflected rather than duplicated structure. +3. **Given** a frame resolves to a method that CGC has indexed, **When** the graph is + queried, **Then** the frame node is linked to the corresponding static method node, + enabling navigation from runtime execution to source definition. +4. **Given** the Xdebug plugin is not installed, **When** CGC starts, **Then** no + Xdebug-related commands or tools appear and no port is opened. + +--- + +### User Story 4 - Automated Container Builds via Common CI/CD Pipeline (Priority: P4) + +A maintainer releasing a new version of CGC or any plugin wants every service that +exposes an MCP endpoint to automatically build a versioned, production-ready container +image and publish it to a container registry. The build pipeline is shared across all +services (CGC core, OTEL plugin, Xdebug plugin), so adding a new plugin +service requires minimal CI configuration changes. The resulting images are compatible +with both Docker Compose and Kubernetes deployment patterns. + +**Why this priority**: The CI/CD pipeline enables reliable, reproducible deployment of +the plugin ecosystem. It is independent of the plugin system itself and can be delivered +after the plugins are working locally. It is foundational for anyone wanting to run +CGC-X in a self-hosted or homelab environment. + +**Independent Test**: Trigger the pipeline for a single service (CGC core or the OTEL +plugin). Verify that a tagged container image is built, passes a health-check smoke +test, and is published to the target registry with the correct version tag. Then verify +that the same pipeline configuration can build a second service with only a service +name change. + +**Acceptance Scenarios**: + +1. **Given** a version tag is pushed to the repository, **When** the pipeline runs, + **Then** container images for all enabled plugin services are built and published with + that version tag and a `latest` tag. +2. **Given** a plugin service container is started from its published image, **When** a + health check is performed, **Then** the service responds correctly within 30 seconds. +3. **Given** a new plugin service directory follows the shared conventions, **When** it + is added to the pipeline configuration, **Then** it builds and publishes alongside + existing services without changes to shared pipeline logic. +4. **Given** a build failure occurs in one service, **When** the pipeline runs, **Then** + only that service's build fails; other services complete successfully and their images + are published. +5. **Given** published images, **When** a Kubernetes manifest referencing those images + is applied to a cluster, **Then** the services start successfully and connect to their + configured graph database. + +--- + +### User Story 5 - Sample Applications for End-to-End Plugin Validation (Priority: P5) + +A developer evaluating CGC's plugin ecosystem wants to see the full pipeline in action — +index code, run an instrumented application, generate OTEL spans, and query the resulting +cross-layer graph — without building their own app first. They clone the repository, run +`docker compose up` in the `samples/` directory, execute a smoke script, and within +minutes have a populated graph with Service, Span, Function, and Class nodes visible in +Neo4j Browser. The sample apps serve as regression fixtures for future development and +as reference implementations for plugin consumers. + +**Why this priority**: All plugin infrastructure (US1-US4) is complete, but there are no +runnable demonstrations of the full pipeline. Sample apps validate the end-to-end flow, +expose integration gaps (such as the FQN correlation gap documented below), and provide +regression fixtures for future changes. + +**Independent Test**: Run `docker compose up -d` in `samples/`, then execute +`bash smoke-all.sh`. All smoke assertions pass (with the documented `correlates_to` +warning). Neo4j Browser at http://localhost:7474 shows Service, Span, Function, and Class +nodes. `cgc otel list-services` returns `sample-php`, `sample-python`, +`sample-ts-gateway`. + +**Acceptance Scenarios**: + +1. **Given** the sample apps are built and started via `docker compose up -d`, **When** + a developer runs the smoke script, **Then** all assertions pass within 120 seconds + (excluding the known `correlates_to` gap which produces a WARN, not FAIL). +2. **Given** the PHP/Laravel sample app is running with OTEL + Xdebug instrumentation, + **When** HTTP requests hit `/api/orders`, **Then** both OTEL spans (with + `code.namespace` and `code.function` attributes) and Xdebug stack frames appear in + the graph. +3. **Given** the Python/FastAPI sample app is running with OTEL instrumentation, **When** + HTTP requests hit `/api/orders`, **Then** OTEL spans appear in the graph with Python- + format FQN attributes (dotted module paths). +4. **Given** the TypeScript/Express gateway is running with OTEL instrumentation, **When** + the gateway proxies requests to backend services, **Then** CLIENT spans with + `peer.service` attributes appear in the graph, producing `CALLS_SERVICE` edges. +5. **Given** all three sample apps are indexed by CGC, **When** the graph is queried for + static code nodes, **Then** Function and Class nodes exist with `path` properties + containing `samples/`. +6. **Given** the known FQN correlation gap exists, **When** `MATCH (sp:Span)- + [:CORRELATES_TO]->(m) RETURN count(sp)` is executed, **Then** the result is 0 and the + smoke script reports WARN (not FAIL), with a reference to `KNOWN-LIMITATIONS.md`. + +--- + +### User Story 6 - Hosted MCP Server Container Image (Priority: P6) + +A platform team or individual developer wants to deploy the CGC MCP server as a +long-running network service accessible to multiple AI assistants and IDE clients +over HTTP — without requiring each client to spawn a local CGC process via stdio. +They pull the official container image, configure Neo4j credentials and an API key +via environment variables, and run it in Docker, Docker Swarm, or Kubernetes. The +server exposes a plain JSON-RPC request/response HTTP endpoint (no SSE or streaming) +that any MCP-compatible client can connect to, with CORS handled at the application +layer. Authentication is deferred to network-level controls or a reverse proxy. + +**Why this priority**: The existing MCP server only supports stdio transport, meaning +every client must run CGC as a child process. This limits deployment to local +development machines and prevents shared team infrastructure, CI/CD integration, or +cloud-hosted deployments. An HTTP transport with a production-ready container image +enables all of these use cases and is the natural next step after the plugin system +and sample apps are validated. + +**Independent Test**: Pull the published `cgc-mcp` image, run it with Neo4j +credentials. Send an MCP `initialize` request via HTTP to the published endpoint. +Verify the server responds with capabilities including all core and plugin tools. +Deploy the same image to a Kubernetes pod and verify it passes readiness probes +and serves MCP requests. + +**Acceptance Scenarios**: + +1. **Given** the `cgc-mcp` image is started with `DATABASE_TYPE`, `NEO4J_URI`, + `NEO4J_USERNAME`, and `NEO4J_PASSWORD` environment variables, **When** a client + sends an HTTP POST to `/mcp`, **Then** the server processes the MCP JSON-RPC + request and returns a valid response. +2. **Given** the server is running, **When** a client sends an `initialize` + request, **Then** the response includes all core tools AND all plugin-contributed + tools (OTEL, Xdebug) in the capabilities. +3. **Given** the server is running with plugins installed, **When** a client calls + `otel_list_services` via the HTTP endpoint, **Then** the server returns the + same results as the stdio transport would. +4. **Given** the server is deployed in Kubernetes, **When** the readiness probe + fires, **Then** the `/healthz` endpoint returns HTTP 200 within 5 seconds. +5. **Given** the server is running behind a reverse proxy or load balancer, + **When** a client sends a preflight CORS OPTIONS request, **Then** the server + responds with appropriate `Access-Control-Allow-*` headers. +6. **Given** the stdio transport is still needed for local IDE integrations, + **When** `cgc mcp start` is run without `--transport`, **Then** the server + defaults to stdio mode (backwards compatible). + +--- + +### Edge Cases + +- What happens when a plugin depends on a specific graph schema version and the core has + been upgraded with schema changes? +- How does CGC handle a plugin that registers a CLI command name or MCP tool name + already used by another loaded plugin? +- What happens if the graph database is unavailable when a plugin attempts to write + ingested data? +- How does the system behave when the OTEL plugin receives a very high volume of spans + (burst traffic) that exceeds ingestion capacity? +- What happens when Xdebug sends stack frames for a file path that CGC has not indexed? +- How are sensitive values (database credentials, API keys) managed in container images + so they are never baked into the image layer? +- What happens when OTEL spans carry `code.namespace` and `code.function` attributes but + CGC's static graph stores `Function` nodes (not `Method` nodes) without an `fqn` + property? (Known gap — `CORRELATES_TO` and `RESOLVES_TO` edges will not form until + FQN computation is added to the graph builder.) +- What happens when the hosted MCP server receives concurrent requests from + multiple AI clients? → Single-process async (uvicorn default asyncio event loop); + concurrent requests are handled via coroutines. Tool calls are short-lived I/O-bound + Cypher queries and do not block each other. +- How does the HTTP transport behave when the Neo4j database connection is lost + mid-request? → `/healthz` returns HTTP 503 with `{"status":"unhealthy"}` when + the database is unreachable. In-flight tool calls return a JSON-RPC error response. + +## Requirements *(mandatory)* + +### Functional Requirements + +**Plugin System Core** + +- **FR-001**: CGC MUST provide a plugin registration interface that allows independently + installable packages to declare CLI commands and MCP tools without modifying core code. +- **FR-002**: CGC MUST auto-discover installed plugins at startup and load them without + requiring manual configuration file edits. +- **FR-003**: CGC MUST isolate plugin failures so that a broken or incompatible plugin + does not prevent CGC core or other plugins from starting. +- **FR-004**: CGC MUST enforce plugin version compatibility checks and skip plugins that + declare an unsupported version range, reporting a clear diagnostic message. +- **FR-005**: CGC MUST ensure plugin-registered CLI commands appear in the top-level + help output, grouped under a visible "plugins" section or annotated as plugin-provided. +- **FR-006**: CGC MUST ensure plugin-registered MCP tools appear in the MCP tool listing + alongside core tools with their plugin source identified in metadata. + +**CLI Plugin Interface** + +- **FR-007**: The plugin interface MUST define a standard contract for registering CLI + command groups, including command name, arguments, options, and handler. +- **FR-008**: Plugins MUST be able to add new top-level CLI command groups without + conflicting with core command names. + +**MCP Plugin Interface** + +- **FR-009**: The plugin interface MUST define a standard contract for registering MCP + tools, including tool name, description, input schema, and handler function. +- **FR-010**: Plugins MUST be able to share the same graph database connection managed + by CGC core rather than opening independent connections. + +**OTEL Processor Plugin** + +- **FR-011**: The OTEL plugin MUST expose an ingestion endpoint that accepts telemetry + spans from a standard OpenTelemetry collector. +- **FR-012**: The OTEL plugin MUST extract structured runtime data from spans (service + identity, code namespace, called function, HTTP route, database query) and write it + to the graph as typed runtime nodes and relationships. +- **FR-013**: The OTEL plugin MUST attempt to correlate runtime nodes to existing static + code nodes in the graph where the function identity can be resolved. +- **FR-014**: The OTEL plugin MUST detect and represent cross-service calls as graph + edges between service nodes. +- **FR-015**: The OTEL plugin MUST support configurable span filtering to exclude + high-noise spans (health checks, metrics polling) from graph storage. +- **FR-016**: The OTEL plugin MUST expose at least one MCP tool that enables querying + the execution path for a specific request or route. + +**Xdebug Listener Plugin** + +- **FR-017**: The Xdebug plugin MUST expose a TCP listener that accepts DBGp protocol + connections from Xdebug-enabled PHP processes. +- **FR-018**: The Xdebug plugin MUST capture the full call stack on each trace event + and write stack frame nodes and call-chain relationships to the graph. +- **FR-019**: The Xdebug plugin MUST deduplicate identical call chains so repeated + execution of the same path does not create redundant graph structure. +- **FR-020**: The Xdebug plugin MUST attempt to resolve stack frames to static method + nodes already indexed by CGC core. +- **FR-021**: The Xdebug plugin MUST be configurable as a development/staging-only + service, excluded from production deployments without changing core configuration. + +**CI/CD Pipeline** + +- **FR-026**: The pipeline MUST build a versioned container image for each plugin + service when a version tag is pushed to the repository. +- **FR-027**: Container images MUST pass a basic health-check smoke test before being + published to the registry. +- **FR-028**: The pipeline MUST publish images with both a specific version tag and a + `latest` tag to the configured container registry. +- **FR-029**: A build failure in one service image MUST NOT prevent other service images + from completing their build and publish steps. +- **FR-030**: The pipeline MUST support a shared build configuration so that adding a + new plugin service requires only adding the service name to a list, not duplicating + pipeline logic. +- **FR-031**: Container images MUST NOT embed sensitive credentials; all secrets MUST + be provided at runtime via environment variables. +- **FR-032**: Each published image MUST include a container health-check definition that + verifies the service is ready to accept connections. +- **FR-033**: Published images MUST be compatible with Kubernetes pod specifications + (no host-mode networking requirements, configurable via environment variables only). + +**Sample Applications** + +- **FR-034**: The repository MUST include at least three sample applications (PHP/Laravel, + Python/FastAPI, TypeScript/Express) that exercise the OTEL plugin's span ingestion + pipeline end-to-end. +- **FR-035**: Each sample application MUST include a Dockerfile, dependency manifest, + OTEL auto-instrumentation configuration, and a README documenting its purpose and + FQN format. +- **FR-036**: A shared `docker-compose.yml` in `samples/` MUST orchestrate all sample + apps alongside the plugin stack (Neo4j, OTEL Collector, CGC services) using a single + `docker compose up` command. +- **FR-037**: An automated smoke script (`samples/smoke-all.sh`) MUST validate the + end-to-end pipeline by asserting the presence of Service, Span, Function, and Class + nodes in the graph after indexing and traffic generation. +- **FR-038**: Sample apps MUST document known limitations (specifically the FQN + correlation gap) in `samples/KNOWN-LIMITATIONS.md` so that developers understand why + `CORRELATES_TO` edges are absent and what future work will resolve it. + +**Hosted MCP Server** + +- **FR-039**: The MCP server MUST support a plain JSON-RPC request/response HTTP + transport (no SSE or streaming) in addition to the existing stdio transport, + selectable via a `--transport` CLI option (default: `stdio` for backwards + compatibility). +- **FR-040**: The HTTP transport MUST expose a single endpoint (`/mcp`) that accepts + MCP JSON-RPC requests as HTTP POST bodies and returns JSON-RPC responses. +- **FR-041**: *(Removed — authentication deferred to network-level controls or + reverse proxy.)* +- **FR-042**: The HTTP transport MUST expose a `/healthz` endpoint that returns + HTTP 200 with `{"status":"ok","tools":N}` when the server is ready and has a valid + database connection, and HTTP 503 with `{"status":"unhealthy"}` when the database + is unreachable. +- **FR-043**: The HTTP transport MUST handle CORS preflight requests and respond + with configurable `Access-Control-Allow-Origin` (via `CGC_CORS_ORIGIN` env var, + default: `*`). +- **FR-044**: A dedicated `Dockerfile.mcp` MUST produce a container image that runs + the MCP server in HTTP transport mode as a long-running service, without requiring + Node.js, HAProxy, or any external protocol translation layer. +- **FR-045**: The MCP container image MUST include all core tools and all installed + plugin tools (OTEL, Xdebug) in the tool listing returned by `tools/list`. +- **FR-046**: The MCP container image MUST NOT embed credentials; all secrets + (database password) MUST be provided at runtime via environment variables or + mounted files. +- **FR-047**: The MCP container image MUST be deployable in Docker, Docker Swarm, + and Kubernetes without host-mode networking or privileged capabilities. + +### Key Entities + +- **Plugin**: A self-contained, independently installable package that contributes CLI + commands and/or MCP tools to CGC. Has a declared name, version, compatibility range, + and lists of registered commands and tools. +- **PluginRegistry**: The runtime component within CGC core that discovers, validates, + and loads installed plugins. Tracks which plugins are active and resolves conflicts. +- **CLICommand**: A command or command group contributed by a plugin. Has a name, + description, argument schema, and an executing handler. +- **MCPTool**: An MCP-protocol tool contributed by a plugin. Has a name, description, + input schema, and a handler. Source plugin is identified in its metadata. +- **RuntimeNode**: A graph node produced by the OTEL or Xdebug plugin representing an + observed execution event (span, stack frame). Carries a `source` property identifying + its origin layer. +- **ContainerImage**: A versioned, publishable artifact for a plugin service. Produced + by the CI/CD pipeline and tagged with the release version. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A developer can create a working plugin that adds a CLI command and an MCP + tool to CGC in under 2 hours, using only the published plugin interface documentation + and without reading CGC core source code. +- **SC-002**: Installing or uninstalling a plugin requires no changes to CGC core + configuration files — zero manual edits. +- **SC-003**: CGC with all plugins enabled starts in under 15 seconds on standard + developer hardware. +- **SC-004**: Runtime span data from an instrumented request appears in the graph within + 10 seconds of the request completing under normal load conditions. +- **SC-005**: An AI assistant using the combined graph (static + runtime) can + answer cross-layer queries (e.g., "what code paths are never executed at runtime") that + are impossible with static analysis alone — validated by documented canonical query + examples that all return correct results. +- **SC-006**: The CI/CD pipeline builds and publishes all plugin service images in a + single pipeline run triggered by a version tag — zero manual steps required after + tagging. +- **SC-007**: Any published plugin service image passes its health check within 30 + seconds of container startup. +- **SC-008**: A new plugin service can be added to the CI/CD pipeline by a contributor + who changes only the service list in pipeline configuration — no pipeline logic + changes required. +- **SC-009**: Duplicate call-chain ingestion (the same execution path observed multiple + times) does not increase graph node count — deduplication is 100% effective for + identical chains. +- **SC-010**: All plugin service images run successfully in a Kubernetes environment + using only standard Kubernetes primitives (Deployments, Services, ConfigMaps, Secrets). +- **SC-011**: Running `bash samples/smoke-all.sh` after `docker compose up -d` in + `samples/` passes all smoke assertions within 120 seconds, with the `correlates_to` + assertion producing WARN (not FAIL) due to the documented FQN gap. +- **SC-012**: The `cgc-mcp` container image starts in under 15 seconds, passes its + `/healthz` check within 5 seconds of readiness, and correctly serves MCP `tools/list` + and `tools/call` requests over HTTP — validated by a curl-based integration test + against the running container. + +## Assumptions + +- The existing CGC codebase uses Python 3.10+ and the plugin interface will be + implemented in Python using the standard entry-points discovery mechanism. +- The graph database (FalkorDB or Neo4j) is already running and accessible to all + plugins via the connection managed by CGC core. +- Plugin authors are expected to be Python developers familiar with the CGC graph schema. +- The OTEL plugin is the primary runtime layer for production use; Xdebug is dev/staging + only, consistent with the research document's stated intent. +- CI/CD pipeline targets GitHub Actions as the execution environment, consistent with + the project's existing workflows. +- Container registry target is determined by project maintainers at implementation time + (Docker Hub, GHCR, or self-hosted). + +## Clarifications + +### Session 2026-03-20 + +- Q: Should the HTTP transport implement full MCP Streamable HTTP (with SSE for server-initiated messages) or plain JSON-RPC request/response over POST? → A: Plain JSON-RPC request/response over POST (no SSE). SSE/streaming can be added as future work if needed. +- Q: What concurrency model should the HTTP transport use for handling multiple AI clients? → A: Single-process async (uvicorn default asyncio event loop). Tool calls are short-lived I/O-bound Cypher queries. +- Q: Should the HTTP endpoint include API key authentication (`CGC_API_KEY`)? → A: No. Authentication deferred to network-level controls or reverse proxy. Removed FR-041, updated acceptance scenarios and SC-012. +- Q: How should `/healthz` behave when Neo4j is unreachable? → A: Return HTTP 503 with `{"status":"unhealthy"}`. In-flight tool calls return JSON-RPC error responses. diff --git a/specs/001-cgc-plugin-extension/tasks.md b/specs/001-cgc-plugin-extension/tasks.md new file mode 100644 index 00000000..55aa6843 --- /dev/null +++ b/specs/001-cgc-plugin-extension/tasks.md @@ -0,0 +1,388 @@ +--- + +description: "Task list for CGC Plugin Extension System" +--- + +# Tasks: CGC Plugin Extension System + +**Input**: Design documents from `specs/001-cgc-plugin-extension/` +**Prerequisites**: plan.md ✅ | spec.md ✅ | research.md ✅ | data-model.md ✅ | contracts/ ✅ + +**Tests**: Included — required by Constitution Principle III (Testing Pyramid, NON-NEGOTIABLE). +Tests MUST be written and observed to FAIL before the corresponding implementation task. + +**Organization**: Tasks grouped by user story to enable independent implementation and testing. + +## Format: `[ID] [P?] [Story?] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (US1–US5) +- Exact file paths included in every task description + +## Path Conventions + +- Core CGC: `src/codegraphcontext/` +- Plugin packages: `plugins/cgc-plugin-/src/cgc_plugin_/` +- Tests: `tests/unit/plugin/`, `tests/integration/plugin/`, `tests/e2e/plugin/` +- CI/CD: `.github/workflows/`, `.github/services.json` +- Deployment: `docker-compose.yml`, `docker-compose.dev.yml`, `k8s/` + +--- + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Initialize all plugin package scaffolding and root configuration before any +story work begins. + +- [X] T001 Create `plugins/` directory tree: `plugins/cgc-plugin-otel/src/cgc_plugin_otel/`, `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/`, `plugins/cgc-plugin-stub/src/cgc_plugin_stub/` with empty `__init__.py` placeholders +- [X] T002 [P] Write `plugins/cgc-plugin-otel/pyproject.toml` — package name `cgc-plugin-otel`, entry-points groups `cgc_cli_plugins` and `cgc_mcp_plugins`, deps: `grpcio>=1.57.0`, `opentelemetry-proto>=0.43b0`, `opentelemetry-sdk>=1.20.0`, `typer[all]>=0.9.0`, `neo4j>=5.15.0` +- [X] T003 [P] Write `plugins/cgc-plugin-xdebug/pyproject.toml` — package name `cgc-plugin-xdebug`, entry-points groups `cgc_cli_plugins` and `cgc_mcp_plugins`, deps: `typer[all]>=0.9.0`, `neo4j>=5.15.0` (stdlib-only implementation) +- [X] T005 [P] Write `plugins/cgc-plugin-stub/pyproject.toml` — package name `cgc-plugin-stub`, entry-points groups `cgc_cli_plugins` and `cgc_mcp_plugins`, dep: `typer[all]>=0.9.0` only (minimal test fixture) +- [X] T006 Add `packaging>=23.0` dependency and optional extras `[otel]`, `[xdebug]`, `[all]` to root `pyproject.toml`, each extra pointing at its corresponding plugin package in `plugins/` + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Core infrastructure that MUST be complete before any user story can be implemented. +The `PluginRegistry` class, graph schema migration, and test infrastructure are shared by all stories. + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete. + +> **NOTE: Write tests FIRST (T008), ensure they FAIL before implementing T007** + +- [X] T007 [P] Add plugin schema constraints and indexes to `config/neo4j/init.cypher` — `UNIQUE` constraints for Service.name, Trace.trace_id, Span.span_id, StackFrame.frame_id; indexes on Span.trace_id, Span.class_name, Span.http_route, StackFrame.fqn (per data-model.md) +- [X] T008 Write `tests/unit/plugin/test_plugin_registry.py` — unit tests (all entry points mocked) covering: discovers plugins from both entry-point groups, validates PLUGIN_METADATA required fields, skips plugin with incompatible cgc_version_constraint, skips plugin with conflicting name (second plugin), catches ImportError without crashing host, catches exception in get_plugin_commands() without crashing host, reports loaded/failed counts correctly. **Run and confirm FAILING before T009.** +- [X] T009 Implement `src/codegraphcontext/plugin_registry.py` — `PluginRegistry` class with: `discover_cli_plugins()` (reads `cgc_cli_plugins` group), `discover_mcp_plugins()` (reads `cgc_mcp_plugins` group), `_validate_metadata()` (checks required fields + cgc_version_constraint via `packaging.specifiers.SpecifierSet`), `_safe_load()` (try/except + SIGALRM 5s timeout on Unix), `_safe_call()` (try/except wrapper for get_plugin_commands/get_mcp_tools/get_mcp_handlers), `loaded_plugins: dict`, `failed_plugins: dict`, startup summary log line +- [X] T010 Update `tests/run_tests.sh` to include `tests/unit/plugin/` and `tests/integration/plugin/` in the `fast` suite alongside existing unit + integration paths + +**Checkpoint**: PluginRegistry unit tests pass. Schema migration ready. Fast suite covers plugin tests. + +--- + +## Phase 3: User Story 1 — Plugin Extensibility Foundation (Priority: P1) 🎯 MVP + +**Goal**: CGC discovers and loads installed plugins automatically; CLI and MCP both surface +plugin-contributed commands and tools; broken plugins never crash the host process. + +**Independent Test**: `pip install -e plugins/cgc-plugin-stub` → `cgc --help` shows `stub` +command group → `cgc stub hello` works → MCP tool `stub_hello` appears in tools/list → +`pip uninstall cgc-plugin-stub` → CGC restarts cleanly with no stub artifacts. + +> **NOTE: Write integration tests (T011) FIRST, ensure they FAIL before T012–T015** + +- [X] T011 Write `tests/integration/plugin/test_plugin_load.py` — integration tests using the stub plugin (installed as editable in conftest fixture): stub CLI command appears in `app.registered_commands` after registry runs; stub MCP tool name appears in server.tools dict; second incompatible-version stub is skipped with warning; two conflicting-name stubs load only first; registry reports correct counts. **Run and confirm FAILING before T012.** +- [X] T012 [P] [US1] Implement `plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py` — `PLUGIN_METADATA` dict: name `cgc-plugin-stub`, version `0.1.0`, cgc_version_constraint `>=0.1.0`, description `Stub plugin for testing` +- [X] T013 [P] [US1] Implement `plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py` — `get_plugin_commands()` returning `("stub", stub_app)` where `stub_app` has one command `hello` that echoes "Hello from stub plugin" +- [X] T014 [P] [US1] Implement `plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py` — `get_mcp_tools()` returning one tool `stub_hello` with inputSchema `{name: string}`; `get_mcp_handlers()` returning handler that returns `{"greeting": f"Hello {name}"}` +- [X] T015 [US1] Modify `src/codegraphcontext/cli/main.py` — add `_load_plugin_cli_commands(registry: PluginRegistry)` function that calls `app.add_typer()` for each entry in `registry.loaded_plugins`; call at module startup after core command registration; add `cgc plugin list` sub-command showing loaded/failed plugins with name, version, tool count +- [X] T016 [US1] Modify `src/codegraphcontext/server.py` — instantiate `PluginRegistry` in `MCPServer.__init__()`, call `_load_plugin_tools()` that merges plugin tool definitions into `self.tools` dict (with conflict check), store plugin handlers in `self.plugin_tool_handlers: dict`, update `handle_tool_call()` to check `self.plugin_tool_handlers` before built-in handler map + +**Checkpoint**: `pip install -e plugins/cgc-plugin-stub && cgc plugin list` shows stub; MCP tools/list includes `stub_hello`; uninstall leaves CGC clean. + +--- + +## Phase 4: User Story 2 — Runtime Intelligence via OTEL Plugin (Priority: P2) + +**Goal**: OTEL plugin receives telemetry spans, writes Service/Trace/Span nodes to the +graph, correlates spans to static Method nodes, and exposes MCP tools for runtime queries. + +**Independent Test**: With a pre-indexed PHP repository, send a synthetic OTLP span payload +to the OTEL plugin endpoint. Query `MATCH (s:Span) RETURN count(s)` → non-zero. +Query `MATCH (s:Span)-[:CORRELATES_TO]->(m:Method) RETURN s.name, m.fqn LIMIT 5` → returns linked results. + +> **NOTE: Write unit tests (T017) FIRST, ensure they FAIL before T018–T022** + +- [X] T017 Write `tests/unit/plugin/test_otel_processor.py` — unit tests (mocked db_manager, no gRPC): `extract_php_context()` parses code.namespace+code.function into fqn; `extract_php_context()` handles missing attributes gracefully (returns None fqn); `is_cross_service_span()` returns True for CLIENT kind spans with peer.service set; `should_filter_span()` returns True for health-check routes matching config; `build_span_dict()` computes duration_ms correctly from ns timestamps. **Run and confirm FAILING before T018.** +- [X] T018 [P] [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py` — `PLUGIN_METADATA` dict: name `cgc-plugin-otel`, version `0.1.0`, cgc_version_constraint `>=0.1.0` +- [X] T019 [P] [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py` — `get_plugin_commands()` returning `("otel", otel_app)` with commands: `query-spans --route TEXT --limit INT`, `list-services`, `status` (shows whether receiver is running) +- [X] T020 [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py` — `extract_php_context(span_attrs: dict) -> dict` (parses code.namespace, code.function, http.route, http.method, db.statement, db.system into typed dict); `build_fqn(namespace, function) -> str | None`; `is_cross_service_span(span_kind, span_attrs) -> bool`; `should_filter_span(span_attrs, filter_routes: list[str]) -> bool` (configurable noise filter) +- [X] T021 [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py` — `AsyncOtelWriter` class: async `write_batch(spans: list[dict])` using `asyncio.Queue(maxsize=10000)` and periodic flush (batch size 100, timeout 5s); MERGE queries for Service, Trace, Span nodes; CHILD_OF (parent_span_id), PART_OF (trace), ORIGINATED_FROM (service), CALLS_SERVICE (CLIENT kind), CORRELATES_TO (fqn match against existing Method nodes); dead-letter queue with `asyncio.Queue(maxsize=100000)` for Neo4j unavailability; `_background_retry_task()` coroutine +- [X] T022 [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py` — `OTLPSpanReceiver` class implementing `TraceServiceServicer` (grpcio + opentelemetry-proto); `Export()` method queues spans for batch processing; `main()` starts gRPC server on `OTEL_RECEIVER_PORT` (default 5317) + launches `process_span_batch()` background task; graceful shutdown on SIGTERM +- [X] T023 [P] [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py` — `get_mcp_tools()` returning: `otel_query_spans` (args: http_route, service, limit), `otel_list_services` (no args), `otel_cross_layer_query` (args: query_type enum: `never_observed|cross_service_calls|recent_executions`); `get_mcp_handlers()` with corresponding Cypher-backed handlers using `server_context["db_manager"]` +- [X] T024 [US2] Create `config/otel-collector/config.yaml` — OTLP gRPC+HTTP receivers (ports 4317, 4318); batch processor (timeout 5s, send_batch_size 512); filter processor dropping spans where `http.route` matches `/health`, `/metrics`, `/ping`; OTLP exporter forwarding to `otel-processor:5317` (insecure TLS) +- [X] T025 [US2] Add OTEL services to `docker-compose.yml` — `otel-collector` service (image: `otel/opentelemetry-collector-contrib:latest`, ports 4317-4318, depends on otel-processor); `cgc-otel-processor` service (build: `plugins/cgc-plugin-otel`, env: NEO4J_URI/USERNAME/PASSWORD/LISTEN_PORT/LOG_LEVEL, depends on neo4j healthcheck, Traefik labels) +- [X] T026 [US2] Write `tests/integration/plugin/test_otel_integration.py` — with real Neo4j fixture (or mock db_manager): call `write_batch()` with synthetic span dicts; assert Service node created with correct name; assert Span node created with correct span_id; assert CHILD_OF relationship created for parent_span_id; assert CORRELATES_TO created when fqn matches pre-existing Method node; assert filtered spans (health route) produce zero graph nodes + +**Checkpoint**: OTEL plugin loads, gRPC receiver accepts a synthetic span, Service+Span nodes appear in graph with CORRELATES_TO link to static Method. + +--- + +## Phase 5: User Story 3 — Development Traces via Xdebug Plugin (Priority: P3) + +**Goal**: Xdebug plugin runs a TCP DBGp listener, captures PHP call stacks, deduplicates +chains, writes StackFrame nodes to the graph, and links frames to static Method nodes. + +**Independent Test**: With a pre-indexed PHP repository, simulate a DBGp TCP connection +sending a synthetic stack_get XML response. Verify StackFrame nodes appear in the graph +with CALLED_BY chain relationships and RESOLVES_TO links to Method nodes. + +> **NOTE: Write unit tests (T027) FIRST, ensure they FAIL before T028–T031** + +- [X] T027 Write `tests/unit/plugin/test_xdebug_parser.py` — unit tests (no TCP): `parse_stack_xml(xml_str) -> list[dict]` returns correct frame list from sample DBGp XML; `compute_chain_hash(frames) -> str` returns same hash for identical frame lists and different hash for different lists; `build_frame_id(class_name, method_name, file_path, line) -> str` returns deterministic unique string; dedup check returns True for hash in LRU cache and False for new hash. **Run and confirm FAILING before T028.** +- [X] T028 [P] [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py` — `PLUGIN_METADATA` dict: name `cgc-plugin-xdebug`, version `0.1.0`, cgc_version_constraint `>=0.1.0`; note in description that this is dev/staging only +- [X] T029 [P] [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py` — `get_plugin_commands()` returning `("xdebug", xdebug_app)` with commands: `start` (starts listener, requires `CGC_PLUGIN_XDEBUG_ENABLED=true`), `stop`, `status`, `list-chains --limit INT` +- [X] T030 [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py` — `DBGpServer` class: `listen(host, port)` opens TCP socket with `SO_REUSEADDR`; `handle_connection(conn)` reads DBGp init packet, sends `run` command, loops: sends `stack_get -i {seq}`, parses XML response via `parse_stack_xml()`, calls `neo4j_writer.write_chain()`, sends `run`; `parse_stack_xml(xml: str) -> list[dict]` using `xml.etree.ElementTree`; server only starts when env var `CGC_PLUGIN_XDEBUG_ENABLED=true` +- [X] T031 [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py` — `XdebugWriter` class: `lru_cache: dict[str, int]` (hash → observation_count, max `DEDUP_CACHE_SIZE=10000`); `write_chain(frames: list[dict], db_manager)`: computes chain_hash, checks LRU — if seen, increments observation_count on existing StackFrame and returns; else MERGEs StackFrame nodes for each frame, creates CALLED_BY chain from depth ordering, attempts RESOLVES_TO match against `Method {fqn: $fqn}` for each frame +- [X] T032 [P] [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py` — `get_mcp_tools()` returning: `xdebug_list_chains` (args: limit, min_observations), `xdebug_query_chain` (args: class_name, method_name); `get_mcp_handlers()` with Cypher-backed handlers +- [X] T033 [US3] Add `xdebug-listener` service to `docker-compose.dev.yml` — build: `plugins/cgc-plugin-xdebug`, env: NEO4J_URI/USERNAME/PASSWORD/LISTEN_HOST/LISTEN_PORT=9003/DEDUP_CACHE_SIZE/LOG_LEVEL=DEBUG/CGC_PLUGIN_XDEBUG_ENABLED=true, ports: `9003:9003`, depends on neo4j healthcheck + +**Checkpoint**: Xdebug plugin loads with `CGC_PLUGIN_XDEBUG_ENABLED=true`, synthetic DBGp XML input produces StackFrame nodes with CALLED_BY chain and RESOLVES_TO Method links. + +--- + +## Phase 6: User Story 4 — Automated Container Builds via Common CI/CD Pipeline (Priority: P4) + +**Goal**: GitHub Actions matrix pipeline builds, smoke tests, and publishes versioned Docker +images for all plugin services. Adding a new service requires only editing `.github/services.json`. + +**Independent Test**: Push a test tag; verify GitHub Actions builds all services in parallel; +verify each image's smoke test passes; verify images are tagged with semver + `latest`; +verify a failure in one service does not cancel other builds. + +- [X] T039 [P] [US5] Create `plugins/cgc-plugin-otel/Dockerfile` — `FROM python:3.12-slim`, non-root `USER cgc`, `COPY` and `pip install --no-cache-dir`, `EXPOSE 5317`, `HEALTHCHECK --interval=30s --timeout=10s CMD python -c "import grpc; print('ok')"`, `CMD ["python", "-m", "cgc_plugin_otel.receiver"]`; no `ENV` with secret values +- [X] T040 [P] [US5] Create `plugins/cgc-plugin-xdebug/Dockerfile` — `FROM python:3.12-slim`, non-root user, `EXPOSE 9003`, `HEALTHCHECK CMD python -c "import socket; socket.socket()"`, `CMD ["python", "-m", "cgc_plugin_xdebug.dbgp_server"]`; requires `CGC_PLUGIN_XDEBUG_ENABLED=true` at runtime +- [X] T042 [US5] Create `.github/services.json` — JSON array with entries for: `cgc-core` (path: `.`, dockerfile: `Dockerfile`, health_check: `version`), `cgc-plugin-otel` (path: `plugins/cgc-plugin-otel`, health_check: `grpc_ping`), `cgc-plugin-xdebug` (path: `plugins/cgc-plugin-xdebug`, health_check: `tcp_connect`) per `contracts/cicd-pipeline.md` schema +- [X] T043 [US5] Create `.github/workflows/docker-publish.yml` — `setup` job reads `.github/services.json` and outputs matrix; `build-images` job with `strategy: {matrix: ${{ fromJson(...) }}, fail-fast: false}`: checkout, `docker/setup-buildx-action@v3`, `docker/login-action@v3` (GHCR, skipped on PR), `docker/metadata-action@v5` (semver+latest tags), `docker/build-push-action@v5` with `push: false` + `outputs: type=docker` for smoke test, smoke test per `health_check` type, then `docker/build-push-action@v5` with `push: true` if not PR and smoke test passed; `build-summary` job reports overall status +- [X] T044 [P] [US5] Create `.github/workflows/test-plugins.yml` — GitHub Actions workflow triggered on PR: matrix over plugin directories, runs `pip install -e . -e plugins/${{ matrix.plugin }}` then `pytest tests/unit/plugin/ tests/integration/plugin/ -v` per plugin; fail-fast: false +- [X] T045 [P] [US5] Create `k8s/cgc-plugin-otel/deployment.yaml` — standard `Deployment` (replicas: 1, image ref from registry, env from ConfigMap `cgc-config` for NEO4J_URI/USERNAME + Secret `cgc-secrets` for NEO4J_PASSWORD, readinessProbe via exec checking grpc import, no hostNetwork) +- [X] T046 [P] [US5] Create `k8s/cgc-plugin-otel/service.yaml` — `ClusterIP` Service exposing port 5317 (gRPC receiver) and 4318 (HTTP, forwarded from collector) +**Checkpoint**: Triggering the workflow on a test tag builds all services in parallel; one intentional Dockerfile error only fails that service's job; remaining images publish to registry with correct semver tags. + +--- + +## Final Phase: Polish & Cross-Cutting Concerns + +**Purpose**: E2E validation, cross-layer queries documentation, and developer experience +improvements that span multiple user stories. + +- [X] T048 Write `tests/e2e/plugin/test_plugin_lifecycle.py` — full user journey E2E test: install stub plugin editable → cgc starts with stub command → cgc plugin list shows stub → stub MCP tool appears in tools/list → call stub_hello via MCP → uninstall stub → cgc restarts cleanly; also: install otel plugin → start receiver → call write_batch with synthetic spans → cross-layer Cypher query returns results; run with `./tests/run_tests.sh e2e` +- [X] T049 [P] Create `docs/plugins/cross-layer-queries.md` — 5 canonical cross-layer Cypher queries validating SC-005: (1) execution path for route, (2) recent methods with no spec, (3) cross-service call chains, (4) specs describing recently-active code, (5) static code never observed at runtime; include expected result schema for each +- [X] T050 [P] Create `docs/plugins/authoring-guide.md` — minimal plugin authoring guide referencing `contracts/plugin-interface.md` and `plugins/cgc-plugin-stub/` as the worked example; covers: package scaffold, PLUGIN_METADATA, CLI contract, MCP contract, testing, publishing to PyPI +- [X] T051 [P] Update root `CLAUDE.md` agent context with new plugin directories, plugin entry-point groups (`cgc_cli_plugins`, `cgc_mcp_plugins`), and the `plugins/` layout — run `.specify/scripts/bash/update-agent-context.sh claude` +- [X] T052 Run full `quickstart.md` validation: install all three plugins editable, execute every command in `specs/001-cgc-plugin-extension/quickstart.md` end-to-end, verify all succeed; update quickstart if any step is incorrect + +--- + +## Phase 7: User Story 5 — Sample Applications for End-to-End Plugin Validation (Priority: P5) + +**Goal**: Three sample applications (PHP/Laravel, Python/FastAPI, TypeScript/Express) with +shared Docker Compose infrastructure and an automated smoke script that validates the full +pipeline: index code → run instrumented app → generate OTEL spans → query cross-layer graph. + +**Independent Test**: `cd samples/ && docker compose up -d && bash smoke-all.sh` — all +assertions pass (correlates_to warns). Neo4j Browser shows Service, Span, Function, Class +nodes. `cgc otel list-services` returns `sample-php`, `sample-python`, `sample-ts-gateway`. + +### Phase 7a: PHP/Laravel Sample App (T053-T056) — parallel with 7b, 7c + +- [X] T053 [P] [US5] Create `samples/php-laravel/` directory structure: `app/Http/Controllers/`, `app/Services/`, `app/Repositories/`, `routes/`, `database/` — standard Laravel layout +- [X] T054 [P] [US5] Implement PHP controllers, services, repositories: `OrderController` (GET/POST `/api/orders`), `OrderService`, `OrderRepository` (SQLite), `HealthController` (`/health`) — cross-class call hierarchy producing meaningful call graph +- [X] T055 [P] [US5] Create `samples/php-laravel/Dockerfile` — PHP 8.3 + Composer, OTEL auto-instrumentation (`open-telemetry/opentelemetry-auto-laravel`), Xdebug extension configured for remote debugging, env: `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, `OTEL_SERVICE_NAME=sample-php` +- [X] T056 [P] [US5] Create `samples/php-laravel/composer.json` + `samples/php-laravel/README.md` — dependencies, FQN format documentation (`Namespace\Class::method`), route table + +### Phase 7b: Python/FastAPI Sample App (T057-T060) — parallel with 7a, 7c + +- [X] T057 [P] [US5] Create `samples/python-fastapi/` directory structure: `app/`, `app/services/`, `app/repositories/` — Python package layout +- [X] T058 [P] [US5] Implement FastAPI app: `OrderRouter` (GET/POST `/api/orders`), `OrderService`, `OrderRepository` (SQLite via aiosqlite), `HealthRouter` (`/health`) — service/repository pattern with cross-module calls +- [X] T059 [P] [US5] Create `samples/python-fastapi/Dockerfile` — Python 3.12-slim, `opentelemetry-instrument` wrapping `uvicorn`, env: `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, `OTEL_SERVICE_NAME=sample-python` +- [X] T060 [P] [US5] Create `samples/python-fastapi/requirements.txt` + `samples/python-fastapi/README.md` — dependencies, Python FQN format documentation (`module.Class.method` vs PHP `Namespace\Class::method`) + +### Phase 7c: TypeScript/Express Gateway (T061-T063) — parallel with 7a, 7b + +- [X] T061 [P] [US5] Create `samples/ts-express-gateway/` directory structure: `src/`, `src/routes/`, `src/services/` — TypeScript project layout +- [X] T062 [P] [US5] Implement Express gateway: `/api/dashboard` (aggregates from PHP + Python backends via HTTP), `/api/orders` (proxies to PHP backend), `/health` — W3C trace context propagation via `@opentelemetry/api`, CLIENT spans with `peer.service` attribute +- [X] T063 [P] [US5] Create `samples/ts-express-gateway/Dockerfile` (multi-stage: build TS → run Node), `package.json`, `tsconfig.json`, `samples/ts-express-gateway/README.md` — documents cross-service span generation and `CALLS_SERVICE` edge formation + +### Phase 7d: Shared Infrastructure (T064-T068) — after 7a-7c + +- [X] T064 [US5] Create `samples/KNOWN-LIMITATIONS.md` — documents FQN correlation gap: OTEL writer matches `(m:Method {fqn: sp.fqn})` but CGC creates `Function` nodes without `fqn` property; explains that `CORRELATES_TO` and `RESOLVES_TO` edges will not form; references graph_builder.py:379; states this is a known limitation, not a bug; references future FQN story +- [X] T065 [US5] Create `samples/docker-compose.yml` — uses `include` to extend `docker-compose.plugin-stack.yml` from project root; adds 3 app services (`sample-php`, `sample-python`, `sample-ts-gateway`); depends_on otel-collector healthcheck; shared network +- [X] T066 [US5] Create `samples/smoke-all.sh` — 6-phase automated validation: (1) wait for services healthy, (2) index sample code via `cgc index`, (3) generate traffic (curl to all routes), (4) wait for span ingestion (poll with timeout), (5) assert via Cypher queries (service_count>=3, span_orders>0, static_functions>0, static_classes>0, cross_service>0, trace_links>0, correlates_to==0 as WARN), (6) summary with pass/warn/fail counts +- [X] T067 [US5] Create `samples/README.md` — full walkthrough: prerequisites, architecture diagram (ASCII), `docker compose up` instructions, smoke script usage, Neo4j Browser exploration guide, per-app route tables, link to KNOWN-LIMITATIONS.md +- [X] T068 [US5] Write `tests/e2e/plugin/test_sample_apps.py` — E2E test wrapping smoke-all.sh: `subprocess.run(["bash", "samples/smoke-all.sh"])`, asserts exit code 0, parses output for FAIL lines; skipped if Docker not available (`pytest.mark.skipif`) + +**Checkpoint**: `cd samples/ && docker compose up -d` starts all services; `bash smoke-all.sh` passes all assertions (correlates_to warns); Neo4j Browser shows populated graph. + +--- + +## Phase 8: User Story 6 — Hosted MCP Server Container Image (Priority: P6) + +**Goal**: The CGC MCP server supports plain JSON-RPC HTTP transport natively (no +supergateway/Node.js), with CORS and health checks. No application-level auth — +authentication deferred to reverse proxy or network controls. A dedicated Docker +image runs it as a long-running service deployable to Docker, Docker Swarm, and +Kubernetes. Single-process async via uvicorn default asyncio event loop. + +**Independent Test**: `docker compose up -d cgc-mcp neo4j` → wait for healthy → +`curl -X POST http://localhost:8045/mcp -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}'` +returns JSON with all core + plugin tools. + +### Phase 8a: HTTP Transport (T069-T073) — sequential + +> **NOTE: Write tests (T069) FIRST, ensure they FAIL before T070-T072** + +- [X] T069 [US6] Write `tests/unit/test_http_transport.py` — unit tests (no network): HTTP handler parses MCP JSON-RPC from request body and returns JSON-RPC response; `/healthz` returns 200 with `{"status":"ok","tools":N}` when DB connected; `/healthz` returns 503 with `{"status":"unhealthy"}` when DB unreachable; CORS preflight returns correct headers; unknown routes return 404. **Run and confirm FAILING before T070.** +- [X] T070 [US6] Add `--transport` option to `cgc mcp start` in `src/codegraphcontext/cli/main.py` — accepts `stdio` (default, existing behavior) or `http`; when `http`, reads `CGC_MCP_PORT` (default 8045) from env; launches HTTP server instead of stdio loop +- [X] T071 [US6] Implement `src/codegraphcontext/http_transport.py` — `HTTPTransport` class using uvicorn + starlette (already a dependency): `POST /mcp` route that deserializes JSON-RPC, calls `MCPServer.handle_request()`, returns JSON-RPC response; `GET /healthz` route that returns `{"status":"ok","tools":N}` (HTTP 200) when DB connected or `{"status":"unhealthy"}` (HTTP 503) when DB unreachable; CORS middleware with configurable origin via `CGC_CORS_ORIGIN` (default `*`); single-process async (uvicorn default asyncio event loop, no workers) +- [X] T072 [US6] Refactor `src/codegraphcontext/server.py` — extract request routing from the stdin loop into a `handle_request(method, params, request_id)` method that both the stdio loop and HTTP transport can call; existing stdio behavior unchanged +- [X] T073 [US6] Write `tests/integration/test_http_transport_integration.py` — start HTTP transport on a random port, send MCP requests via `httpx`, verify: `initialize` returns capabilities, `tools/list` includes core + plugin tools, `tools/call` with `stub_hello` returns greeting, `/healthz` returns 200 when DB connected, `/healthz` returns 503 when DB unreachable + +### Phase 8b: Container Image (T074-T077) — after 8a + +- [X] T074 [P] [US6] Create `Dockerfile.mcp` — multi-stage build from existing `Dockerfile`, installs core + all plugins (`cgc-plugin-otel`, `cgc-plugin-xdebug`), `EXPOSE 8045`, `HEALTHCHECK` via `/healthz`, `CMD ["cgc", "mcp", "start", "--transport", "http"]`; non-root user, no embedded credentials +- [X] T075 [P] [US6] Add `cgc-mcp` service to `docker-compose.plugin-stack.yml` — build from `Dockerfile.mcp`, env: `DATABASE_TYPE`, `NEO4J_URI`, `NEO4J_USERNAME`, `NEO4J_PASSWORD`, `CGC_CORS_ORIGIN`, `CGC_MCP_PORT`; ports `8045:8045`; depends_on neo4j healthy; replaces existing `cgc-core` service (which restarts in a loop) +- [X] T076 [P] [US6] Create `k8s/cgc-mcp/deployment.yaml` — Deployment with readinessProbe on `/healthz`, env from ConfigMap + Secret, image ref from registry; `k8s/cgc-mcp/service.yaml` — ClusterIP exposing port 8045 +- [X] T077 [US6] Add `cgc-mcp` to `.github/services.json` for CI/CD matrix build — path `./`, dockerfile `Dockerfile.mcp`, health_check `http_get` + +### Phase 8c: Documentation and Testing (T078-T080) — after 8b + +- [X] T078 [US6] Create `docs/deployment/MCP_SERVER_HOSTING.md` — deployment guide covering: Docker standalone, Docker Compose with Neo4j, Docker Swarm, Kubernetes; env var reference; client configuration (Claude Desktop, VS Code, Cursor, Claude Code) for remote HTTP MCP endpoint; reverse proxy auth and TLS recommendations +- [X] T079 [US6] Write `tests/e2e/test_mcp_container.py` — E2E test: build `Dockerfile.mcp`, start container with docker-compose, send MCP requests via curl/httpx, assert `tools/list` returns core + plugin tools, assert `tools/call` executes, assert `/healthz` 200; skipped if Docker not available +- [X] T080 [US6] Update `samples/docker-compose.yml` to include `cgc-mcp` service as an alternative to `cgc-core`, with documentation in `samples/README.md` showing how to connect AI clients to the hosted endpoint + +**Checkpoint**: `docker compose up -d cgc-mcp neo4j` → `/healthz` returns 200 → `curl -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' http://localhost:8045/mcp` returns tool listing → same image deploys to K8s pod. + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies — start immediately; T002-T005 in parallel +- **Foundational (Phase 2)**: Depends on Setup (T001 for dirs, T006 for root pyproject) + - T007 (schema) and T010 (test runner) are independent of each other — run in parallel + - T008 (unit tests) must be written before T009 (PluginRegistry implementation) +- **US1 (Phase 3)**: Depends on Foundational (T009 PluginRegistry complete) + - T011 (integration tests) written before T012-T016 + - T012, T013, T014 (stub plugin files) independent of each other — run in parallel + - T015 and T016 (core modifications) can run in parallel once T009 is done +- **US2 (Phase 4)**: Depends on US1 complete (plugin loading infrastructure) + - T017 (unit tests) before T018-T023 + - T018, T019 (metadata + CLI) independent — parallel + - T020 → T021 → T022 (processor → writer → receiver — sequential) + - T023 (MCP tools) independent of T020-T022 — can run in parallel with T021 + - T024 (OTel Collector config), T025 (docker-compose) independent — parallel after T022 +- **US3 (Phase 5)**: Depends on US1 complete; independent of US2 + - T027 (unit tests) before T028-T032 + - T028, T029 (metadata + CLI) parallel + - T030 → T031 (dbgp_server → neo4j_writer — sequential; writer depends on parsed frames) + - T032 (MCP tools) independent — parallel with T031 +- **US4 (Phase 6)**: Depends on US2 + US3 complete (Dockerfiles need working services) + - T039, T040 (Dockerfiles) parallel + - T042 (services.json) before T043 (workflow) + - T044 (test workflow) parallel with T043 + - T045, T046 (K8s manifests) parallel, independent of T043-T044 +- **US5 (Phase 7)**: Depends on US2 + US4 complete (needs working OTEL plugin + Docker images) + - Phase 7a (T053-T056), 7b (T057-T060), 7c (T061-T063) all parallel — three independent apps + - Phase 7d (T064-T068) sequential after 7a-7c — shared infrastructure depends on all apps + - T064 (KNOWN-LIMITATIONS) → T065 (docker-compose) → T066 (smoke script) → T067 (README) → T068 (E2E test) +- **US6 (Phase 8)**: Depends on US1 complete (plugin loading for tool listing); independent of US2-US5 + - Phase 8a (T069-T073) sequential — tests first, then transport, then server refactor + - Phase 8b (T074-T077) after 8a — T074, T075, T076 parallel; T077 after T074 + - Phase 8c (T078-T080) after 8b — T078 parallel with T079; T080 last +- **Polish (Final Phase)**: Depends on all user stories complete + - T049, T050, T051 all parallel + - T052 (quickstart validation) last — sequentially after T048-T051 + +### User Story Dependencies + +- **US1 (P1)**: No story dependencies — first to implement +- **US2 (P2)**: Depends on US1 complete +- **US3 (P3)**: Depends on US1 complete — independent of US2 +- **US4 (P4)**: Depends on US2 + US3 complete (container services need working implementations) +- **US5 (P5)**: Depends on US2 + US4 complete (needs working OTEL plugin + Docker infrastructure) +- **US6 (P6)**: Depends on US1 complete (plugin loading); independent of US2-US5 (can be parallelized) + +### Within Each User Story + +- Unit/integration tests MUST be written and FAIL before corresponding implementation +- `__init__.py` (metadata) before CLI and MCP modules +- CLI and MCP modules can be written in parallel +- Core logic (processor, writer, server) before MCP handlers that use it +- Docker/compose additions after core implementation is working + +--- + +## Parallel Execution Examples + +### Phase 1 (Setup) +``` +Parallel: T002, T003, T005 — three plugin pyproject.toml files, different paths +Then: T001 (dirs), T006 (root pyproject) +``` + +### Phase 2 (Foundational) +``` +Parallel: T007 (schema migration), T010 (test runner update) +Sequential: T008 (write unit tests) → T009 (implement PluginRegistry) +``` + +### US2 (OTEL Plugin) +``` +Write + fail: T017 +Parallel: T018, T019 (metadata + CLI) +Sequential: T020 → T021 → T022 (processor → writer → receiver) +Parallel with T021: T023 (MCP tools — uses db_manager directly, not receiver) +Parallel: T024, T025 (config + docker-compose) +Then: T026 (integration tests) +``` + +### US4 (CI/CD) +``` +Parallel: T039, T040 (two Dockerfiles) +Sequential: T042 → T043 (services.json must exist before workflow reads it) +Parallel: T044, T045, T046 (test workflow + K8s manifests) +``` + +### US6 (Hosted MCP Server) +``` +Sequential: T069 (write tests, confirm FAIL) → T070 → T071 → T072 → T073 +Parallel: T074, T075, T076 (Dockerfile, docker-compose, K8s manifests) +Then: T077 (services.json update) +Parallel: T078, T079 (docs + E2E test) +Then: T080 (samples integration) +``` + +--- + +## Implementation Strategy + +### MVP First (US1 Only) + +1. Complete Phase 1: Setup (T001–T006) +2. Complete Phase 2: Foundational (T007–T010) +3. Complete Phase 3: US1 Plugin Foundation (T011–T016) +4. **STOP and VALIDATE**: `cgc plugin list` works; stub plugin loads; MCP tools list includes stub; broken plugin doesn't crash CGC +5. Deploy/demo: plugin system is usable by third-party authors + +### Incremental Delivery + +1. Setup + Foundational → PluginRegistry ready +2. US1 → Plugin system works → **demo: install any plugin** +3. US2 → Runtime intelligence → **demo: "show what ran during this request"** +4. US3 → Dev traces → **demo: "show concrete implementations that ran"** +5. US4 → CI/CD → **demo: `git tag v0.1.0` builds all images automatically** +6. US5 → Sample apps → **demo: `docker compose up && bash smoke-all.sh` — full pipeline validated** +7. US6 → Hosted MCP → **demo: `curl -d '...' http://cgc-mcp:8045/mcp` — remote AI clients connect** + +### Parallel Team Strategy + +With 2 developers after US1 is complete: +- Developer A: US2 (OTEL Plugin) +- Developer B: US3 (Xdebug Plugin) + +Both complete independently, then US4 (CI/CD) begins. + +--- + +## Notes + +- `[P]` tasks = different files, no dependencies on incomplete tasks in the same phase +- `[US?]` maps each task to its user story for traceability and independent delivery +- Tests MUST be written and FAIL before implementation — this is NON-NEGOTIABLE per Constitution Principle III +- Each phase has a named Checkpoint — validate before moving to the next phase +- Verify `./tests/run_tests.sh fast` passes after completing each phase +- Plugin name prefix convention for MCP tools: `_` (e.g., `otel_query_spans`) +- No credentials in Dockerfiles or docker-compose.yml — all via environment variables +- Xdebug plugin: requires `CGC_PLUGIN_XDEBUG_ENABLED=true` at runtime; absent = no TCP port opened diff --git a/src/codegraphcontext/cli/main.py b/src/codegraphcontext/cli/main.py index 9374662e..eb589ef4 100644 --- a/src/codegraphcontext/cli/main.py +++ b/src/codegraphcontext/cli/main.py @@ -24,6 +24,7 @@ from codegraphcontext.server import MCPServer from codegraphcontext.core.database import DatabaseManager +from codegraphcontext.plugin_registry import PluginRegistry from .setup_wizard import run_neo4j_setup_wizard, configure_mcp_client from . import config_manager # Import the new helper functions @@ -102,6 +103,58 @@ def get_version() -> str: mcp_app = typer.Typer(help="MCP client configuration commands") app.add_typer(mcp_app, name="mcp") +# --------------------------------------------------------------------------- +# Plugin CLI integration +# --------------------------------------------------------------------------- + +_plugin_registry: PluginRegistry | None = None + +plugin_app = typer.Typer(help="Manage CGC plugins.") +app.add_typer(plugin_app, name="plugin") + + +@plugin_app.command("list") +def plugin_list(): + """Show all loaded and failed plugins.""" + global _plugin_registry + if _plugin_registry is None: + console.print("[yellow]Plugin registry not initialised.[/yellow]") + return + + table = Table(title="CGC Plugins", box=box.SIMPLE) + table.add_column("Name", style="cyan") + table.add_column("Status", style="bold") + table.add_column("Version") + table.add_column("Tools / Command") + table.add_column("Reason", style="dim") + + for name, info in _plugin_registry.loaded_plugins.items(): + meta = info.get("metadata", {}) + version = meta.get("version", "?") + tools = ", ".join(info.get("mcp_tools", [])) + cmd = info.get("cli_command", "") + detail = tools or cmd or "—" + table.add_row(name, "[green]loaded[/green]", version, detail, "") + + for name, reason in _plugin_registry.failed_plugins.items(): + table.add_row(name, "[red]failed[/red]", "—", "—", reason) + + console.print(table) + + +def _load_plugin_cli_commands(registry: PluginRegistry) -> None: + """Attach plugin-contributed Typer command groups to the root app.""" + global _plugin_registry + _plugin_registry = registry + for cmd_name, typer_app in registry.cli_commands: + app.add_typer(typer_app, name=cmd_name) + + +# Discover and register plugin CLI commands at import time. +_registry = PluginRegistry() +_registry.discover_cli_plugins() +_load_plugin_cli_commands(_registry) + @mcp_app.command("setup") def mcp_setup(): """ @@ -119,17 +172,50 @@ def mcp_setup(): configure_mcp_client() @mcp_app.command("start") -def mcp_start(): +def mcp_start( + transport: str = typer.Option( + "stdio", + "--transport", + help="Transport mode: 'stdio' (default, for IDE integrations) or 'http' (JSON-RPC over HTTP).", + show_default=True, + ), +): """ Start the CodeGraphContext MCP server. - - Starts the server which listens for JSON-RPC requests from stdin. - This is used by IDE integrations (VS Code, Cursor, etc.). + + Starts the server which listens for JSON-RPC requests from stdin (stdio + mode) or over HTTP (http mode). stdio mode is used by IDE integrations + (VS Code, Cursor, etc.). http mode listens on the port specified by the + CGC_MCP_PORT environment variable (default 8045). """ + if transport not in ("stdio", "http"): + console.print(f"[bold red]Error:[/bold red] Unknown transport '{transport}'. Use 'stdio' or 'http'.") + raise typer.Exit(code=1) + console.print("[bold green]Starting CodeGraphContext Server...[/bold green]") _load_credentials() server = None + + if transport == "http": + port = int(os.environ.get("CGC_MCP_PORT", "8045")) + console.print(f"[bold cyan]HTTP transport — listening on port {port}[/bold cyan]") + try: + from codegraphcontext.http_transport import HTTPTransport + server = MCPServer() + http = HTTPTransport(server) + http.start(port=port) + except ValueError as e: + console.print(f"[bold red]Configuration Error:[/bold red] {e}") + console.print("Please run `cgc neo4j setup` or use FalkorDB (default).") + except KeyboardInterrupt: + console.print("\n[bold yellow]Server stopped by user.[/bold yellow]") + finally: + if server: + server.shutdown() + return + + # --- stdio (default) --- loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: diff --git a/src/codegraphcontext/http_transport.py b/src/codegraphcontext/http_transport.py new file mode 100644 index 00000000..7f3cacda --- /dev/null +++ b/src/codegraphcontext/http_transport.py @@ -0,0 +1,121 @@ +# src/codegraphcontext/http_transport.py +"""HTTP transport layer for the CGC MCP server. + +Exposes the MCP JSON-RPC interface over plain HTTP POST at ``/mcp`` and a +liveness probe at ``/healthz``. Authentication is intentionally absent — +callers should rely on a reverse proxy or network-level controls. + +Environment variables +--------------------- +CGC_CORS_ORIGIN + Allowed CORS origin passed to ``CORSMiddleware``. Defaults to ``*``. +""" +from __future__ import annotations + +import json +import os +from typing import TYPE_CHECKING, Any + +import uvicorn +from fastapi import FastAPI, Request, Response +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse + +if TYPE_CHECKING: + from .server import MCPServer + + +class HTTPTransport: + """Wraps an :class:`~codegraphcontext.server.MCPServer` behind a FastAPI app. + + Args: + server: A fully-initialised ``MCPServer`` instance. + """ + + def __init__(self, server: "MCPServer") -> None: + self.server = server + self._app = self._build_app() + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _build_app(self) -> FastAPI: + """Construct and configure the FastAPI application.""" + cors_origin: str = os.environ.get("CGC_CORS_ORIGIN", "*") + + app = FastAPI(title="CodeGraphContext MCP HTTP Transport", docs_url=None, redoc_url=None) + + app.add_middleware( + CORSMiddleware, + allow_origins=[cors_origin], + allow_credentials=False, + allow_methods=["GET", "POST", "OPTIONS"], + allow_headers=["Content-Type"], + ) + + @app.post("/mcp") + async def mcp_endpoint(request: Request) -> Response: + """Deserialise a JSON-RPC request, dispatch to MCPServer, return response.""" + try: + body = await request.body() + payload: dict[str, Any] = json.loads(body) + except Exception: + return JSONResponse( + status_code=400, + content={ + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32700, "message": "Parse error"}, + }, + ) + + method: str = payload.get("method", "") + params: dict[str, Any] = payload.get("params", {}) or {} + request_id: Any = payload.get("id") + + response = await self.server.handle_request(method, params, request_id) + + if response is None: + # Notification — return 204 No Content. + return Response(status_code=204) + + return JSONResponse(content=response) + + @app.get("/healthz") + async def healthz() -> Response: + """Liveness probe. Returns 200 when DB is reachable, 503 otherwise.""" + connected: bool = self.server.db_manager.is_connected() + tool_count: int = len(self.server.tools) + if connected: + return JSONResponse( + status_code=200, + content={"status": "ok", "tools": tool_count}, + ) + return JSONResponse( + status_code=503, + content={"status": "unhealthy"}, + ) + + return app + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + @property + def app(self) -> FastAPI: + """The underlying FastAPI application (useful for testing).""" + return self._app + + def start(self, port: int = 8045) -> None: + """Run the HTTP server synchronously using uvicorn. + + This method blocks until the server is stopped. It runs in the + default asyncio event loop provided by uvicorn (single-process, + no workers). + + Args: + port: TCP port to listen on. + """ + uvicorn.run(self._app, host="0.0.0.0", port=port, log_level="info") diff --git a/src/codegraphcontext/plugin_registry.py b/src/codegraphcontext/plugin_registry.py new file mode 100644 index 00000000..999d3e94 --- /dev/null +++ b/src/codegraphcontext/plugin_registry.py @@ -0,0 +1,283 @@ +""" +Plugin registry for CodeGraphContext. + +Discovers and loads plugins declared via Python entry points: + - Group ``cgc_cli_plugins``: plugins contributing Typer CLI command groups + - Group ``cgc_mcp_plugins``: plugins contributing MCP tools + +Plugins are isolated: a broken plugin logs a warning and is skipped without +affecting CGC core or other plugins. +""" +from __future__ import annotations + +import logging +import signal +import sys +from typing import Any + +from importlib.metadata import entry_points, version as pkg_version, PackageNotFoundError +from packaging.specifiers import SpecifierSet, InvalidSpecifier + +logger = logging.getLogger(__name__) + +_REQUIRED_METADATA_FIELDS = ("name", "version", "cgc_version_constraint", "description") +_LOAD_TIMEOUT_SECONDS = 5 + + +def _get_cgc_version() -> str: + try: + return pkg_version("codegraphcontext") + except PackageNotFoundError: + return "0.0.0" + + +class PluginRegistry: + """ + Discovers, validates, and loads CGC plugins at startup. + + Usage:: + + registry = PluginRegistry() + registry.discover_cli_plugins() # populates cli_commands + registry.discover_mcp_plugins(ctx) # populates mcp_tools + mcp_handlers + + Results are available via: + - ``registry.cli_commands`` list of (name, typer.Typer) + - ``registry.mcp_tools`` dict of tool_name → ToolDefinition + - ``registry.mcp_handlers`` dict of tool_name → callable + - ``registry.loaded_plugins`` dict of name → registration info + - ``registry.failed_plugins`` dict of name → failure reason + """ + + def __init__(self) -> None: + self.cli_commands: list[tuple[str, Any]] = [] + self.mcp_tools: dict[str, dict] = {} + self.mcp_handlers: dict[str, Any] = {} + self.loaded_plugins: dict[str, dict] = {} + self.failed_plugins: dict[str, str] = {} + self._cgc_version = _get_cgc_version() + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def discover_cli_plugins(self) -> None: + """Discover and load all ``cgc_cli_plugins`` entry points.""" + eps = self._get_entry_points("cgc_cli_plugins") + for ep in eps: + self._load_cli_plugin(ep) + self._log_summary() + + def discover_mcp_plugins(self, server_context: dict | None = None) -> None: + """Discover and load all ``cgc_mcp_plugins`` entry points.""" + if server_context is None: + server_context = {} + eps = self._get_entry_points("cgc_mcp_plugins") + for ep in eps: + self._load_mcp_plugin(ep, server_context) + + # ------------------------------------------------------------------ + # Internal loaders + # ------------------------------------------------------------------ + + def _load_cli_plugin(self, ep: Any) -> None: + plugin_name = ep.name + mod = self._safe_import(plugin_name, ep) + if mod is None: + return + + # Validate metadata + reason = self._validate_metadata(plugin_name, mod) + if reason: + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + # Check for name conflict + if plugin_name in self.loaded_plugins: + msg = f"name conflict with already-loaded plugin '{plugin_name}'" + self.failed_plugins[plugin_name + "_duplicate"] = msg + logger.warning("Plugin '%s' (second instance) skipped: %s", plugin_name, msg) + return + + # Call get_plugin_commands() + get_cmds = getattr(mod, "get_plugin_commands", None) + if get_cmds is None: + reason = "missing get_plugin_commands() function" + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + result = self._safe_call(plugin_name, get_cmds) + if result is None: + return + + try: + cmd_name, typer_app = result + except (TypeError, ValueError) as exc: + reason = f"get_plugin_commands() returned invalid format: {exc}" + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + self.cli_commands.append((cmd_name, typer_app)) + self.loaded_plugins[plugin_name] = { + "status": "loaded", + "metadata": mod.PLUGIN_METADATA, + "cli_command": cmd_name, + } + logger.info("Plugin '%s' loaded CLI command group '%s'", plugin_name, cmd_name) + + def _load_mcp_plugin(self, ep: Any, server_context: dict) -> None: + plugin_name = ep.name + mod = self._safe_import(plugin_name, ep) + if mod is None: + return + + reason = self._validate_metadata(plugin_name, mod) + if reason: + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + get_tools = getattr(mod, "get_mcp_tools", None) + get_handlers = getattr(mod, "get_mcp_handlers", None) + + if get_tools is None: + reason = "missing get_mcp_tools() function" + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + tools = self._safe_call(plugin_name, get_tools, server_context) + if tools is None: + return + + handlers: dict = {} + if get_handlers is not None: + h = self._safe_call(plugin_name, get_handlers, server_context) + if h is not None: + handlers = h + + registered = 0 + for tool_name, tool_def in tools.items(): + if tool_name in self.mcp_tools: + logger.warning( + "Plugin '%s': tool '%s' conflicts with existing tool — skipped", + plugin_name, tool_name, + ) + continue + self.mcp_tools[tool_name] = tool_def + if tool_name in handlers: + self.mcp_handlers[tool_name] = handlers[tool_name] + registered += 1 + + if plugin_name not in self.loaded_plugins: + self.loaded_plugins[plugin_name] = { + "status": "loaded", + "metadata": mod.PLUGIN_METADATA, + "mcp_tools": list(tools.keys()), + } + else: + self.loaded_plugins[plugin_name]["mcp_tools"] = list(tools.keys()) + + logger.info("Plugin '%s' loaded %d MCP tool(s)", plugin_name, registered) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _get_entry_points(self, group: str) -> list: + try: + return list(entry_points(group=group)) + except Exception as exc: + logger.error("Failed to query entry points for group '%s': %s", group, exc) + return [] + + def _safe_import(self, plugin_name: str, ep: Any) -> Any | None: + """Load an entry point with timeout and full exception isolation.""" + _alarm_set = False + try: + if hasattr(signal, "SIGALRM"): + def _timeout_handler(signum, frame): + raise TimeoutError( + f"Plugin '{plugin_name}' import timed out after " + f"{_LOAD_TIMEOUT_SECONDS}s" + ) + signal.signal(signal.SIGALRM, _timeout_handler) + signal.alarm(_LOAD_TIMEOUT_SECONDS) + _alarm_set = True + + mod = ep.load() + return mod + + except TimeoutError as exc: + reason = str(exc) + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' load timeout: %s", plugin_name, reason) + return None + except ImportError as exc: + reason = f"ImportError: {exc}" + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' import failed (missing dependency?): %s", plugin_name, exc) + return None + except AttributeError as exc: + reason = f"AttributeError: {exc}" + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' entry point invalid (bad module path?): %s", plugin_name, exc) + return None + except Exception as exc: + reason = f"{type(exc).__name__}: {exc}" + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' unexpected load error: %s", plugin_name, exc, exc_info=True) + return None + finally: + if _alarm_set and hasattr(signal, "SIGALRM"): + signal.alarm(0) + + def _safe_call(self, plugin_name: str, func: Any, *args: Any) -> Any | None: + """Call a plugin function with full exception isolation.""" + try: + return func(*args) + except Exception as exc: + func_name = getattr(func, "__name__", repr(func)) + reason = f"{type(exc).__name__} in {func_name}: {exc}" + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' call failed: %s", plugin_name, exc, exc_info=True) + return None + + def _validate_metadata(self, plugin_name: str, mod: Any) -> str: + """Return an error reason string, or empty string if valid.""" + metadata = getattr(mod, "PLUGIN_METADATA", None) + if metadata is None: + return "missing PLUGIN_METADATA in __init__.py" + + for field in _REQUIRED_METADATA_FIELDS: + if field not in metadata: + return f"PLUGIN_METADATA missing required field '{field}'" + + constraint_str = metadata.get("cgc_version_constraint", "") + try: + specifier = SpecifierSet(constraint_str) + except InvalidSpecifier: + return f"invalid cgc_version_constraint '{constraint_str}'" + + if self._cgc_version not in specifier: + return ( + f"version mismatch: plugin requires CGC {constraint_str}, " + f"installed is {self._cgc_version}" + ) + + return "" + + def _log_summary(self) -> None: + n_loaded = len(self.loaded_plugins) + n_failed = len(self.failed_plugins) + if n_loaded == 0 and n_failed == 0: + return + parts = [f"{n_loaded} plugin(s) loaded"] + if n_failed: + parts.append(f"{n_failed} skipped/failed") + logger.info("CGC plugins: %s", ", ".join(parts)) + for name, reason in self.failed_plugins.items(): + logger.warning(" ✗ %s — %s", name, reason) diff --git a/src/codegraphcontext/server.py b/src/codegraphcontext/server.py index 2f96783b..f7dbede2 100644 --- a/src/codegraphcontext/server.py +++ b/src/codegraphcontext/server.py @@ -32,6 +32,7 @@ query_handlers, watcher_handlers ) +from .plugin_registry import PluginRegistry DEFAULT_EDIT_DISTANCE = 2 DEFAULT_FUZZY_SEARCH = False @@ -86,9 +87,33 @@ def __init__(self, loop=None): def _init_tools(self): """ - Defines the complete tool manifest for the LLM. + Defines the complete tool manifest for the LLM, including plugin tools. """ - self.tools = TOOLS + self.tools = dict(TOOLS) # mutable copy + + server_context = { + "db_manager": self.db_manager, + "version": self._get_version(), + } + + plugin_registry = PluginRegistry() + plugin_registry.discover_mcp_plugins(server_context) + + self.plugin_tool_handlers: Dict[str, Any] = {} + for tool_name, tool_def in plugin_registry.mcp_tools.items(): + if tool_name in self.tools: + continue # built-in tools take precedence + self.tools[tool_name] = tool_def + if tool_name in plugin_registry.mcp_handlers: + self.plugin_tool_handlers[tool_name] = plugin_registry.mcp_handlers[tool_name] + + @staticmethod + def _get_version() -> str: + try: + from importlib.metadata import version + return version("codegraphcontext") + except Exception: + return "0.0.0" def get_database_status(self) -> dict: """Returns the current connection status of the Neo4j database.""" @@ -204,8 +229,75 @@ async def handle_tool_call(self, tool_name: str, args: Dict[str, Any]) -> Dict[s # Run the synchronous tool function in a separate thread to avoid # blocking the main asyncio event loop. return await asyncio.to_thread(handler, **args) + + # Fall through to plugin handlers + plugin_handler = self.plugin_tool_handlers.get(tool_name) + if plugin_handler: + return await asyncio.to_thread(plugin_handler, **args) + + return {"error": f"Unknown tool: {tool_name}"} + + async def handle_request( + self, + method: str, + params: Dict[str, Any], + request_id: Any, + ) -> Optional[Dict[str, Any]]: + """Routes a single JSON-RPC request and returns the response dict. + + Returns ``None`` for notification methods that require no response + (e.g. ``notifications/initialized``). + + Args: + method: The JSON-RPC method name. + params: The ``params`` field of the request (may be empty dict). + request_id: The ``id`` field of the request, or ``None`` for + notifications. + + Returns: + A JSON-RPC response dict, or ``None`` when no response is needed. + """ + if method == 'initialize': + return { + "jsonrpc": "2.0", "id": request_id, + "result": { + "protocolVersion": "2025-03-26", + "serverInfo": { + "name": "CodeGraphContext", "version": "0.1.0", + "systemPrompt": LLM_SYSTEM_PROMPT + }, + "capabilities": {"tools": {"listTools": True}}, + } + } + elif method == 'tools/list': + return { + "jsonrpc": "2.0", "id": request_id, + "result": {"tools": list(self.tools.values())} + } + elif method == 'tools/call': + tool_name = params.get('name') + args = params.get('arguments', {}) + result = await self.handle_tool_call(tool_name, args) + + if "error" in result: + return { + "jsonrpc": "2.0", "id": request_id, + "error": {"code": -32000, "message": "Tool execution error", "data": result} + } + return { + "jsonrpc": "2.0", "id": request_id, + "result": {"content": [{"type": "text", "text": json.dumps(result, indent=2)}]} + } + elif method == 'notifications/initialized': + # Notification — no response needed. + return None else: - return {"error": f"Unknown tool: {tool_name}"} + if request_id is not None: + return { + "jsonrpc": "2.0", "id": request_id, + "error": {"code": -32601, "message": f"Method not found: {method}"} + } + return None async def run(self): """ @@ -214,7 +306,7 @@ async def run(self): # info_logger("MCP Server is running. Waiting for requests...") print("MCP Server is running. Waiting for requests...", file=sys.stderr, flush=True) self.code_watcher.start() - + loop = asyncio.get_event_loop() while True: try: @@ -223,59 +315,14 @@ async def run(self): if not line: debug_logger("Client disconnected (EOF received). Shutting down.") break - + request = json.loads(line.strip()) method = request.get('method') params = request.get('params', {}) request_id = request.get('id') - - response = {} - # Route the request based on the JSON-RPC method. - if method == 'initialize': - response = { - "jsonrpc": "2.0", "id": request_id, - "result": { - "protocolVersion": "2025-03-26", - "serverInfo": { - "name": "CodeGraphContext", "version": "0.1.0", - "systemPrompt": LLM_SYSTEM_PROMPT - }, - "capabilities": {"tools": {"listTools": True}}, - } - } - elif method == 'tools/list': - # Return the list of tools defined in _init_tools. - response = { - "jsonrpc": "2.0", "id": request_id, - "result": {"tools": list(self.tools.values())} - } - elif method == 'tools/call': - # Execute a tool call and return the result. - tool_name = params.get('name') - args = params.get('arguments', {}) - result = await self.handle_tool_call(tool_name, args) - - if "error" in result: - response = { - "jsonrpc": "2.0", "id": request_id, - "error": {"code": -32000, "message": "Tool execution error", "data": result} - } - else: - response = { - "jsonrpc": "2.0", "id": request_id, - "result": {"content": [{"type": "text", "text": json.dumps(result, indent=2)}]} - } - elif method == 'notifications/initialized': - # This is a notification, no response needed. - pass - else: - # Handle unknown methods. - if request_id is not None: - response = { - "jsonrpc": "2.0", "id": request_id, - "error": {"code": -32601, "message": f"Method not found: {method}"} - } - + + response = await self.handle_request(method, params, request_id) + # Send the response to standard output if it's not a notification. if request_id is not None and response: print(json.dumps(response), flush=True) diff --git a/tests/e2e/plugin/__init__.py b/tests/e2e/plugin/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/plugin/test_plugin_lifecycle.py b/tests/e2e/plugin/test_plugin_lifecycle.py new file mode 100644 index 00000000..33f2f896 --- /dev/null +++ b/tests/e2e/plugin/test_plugin_lifecycle.py @@ -0,0 +1,447 @@ +""" +E2E Plugin Lifecycle Tests +========================== + +Full user-journey tests for the CGC plugin extension system. + +Journey 1 — Stub plugin: + install stub editable + → CGC starts with stub CLI command + → cgc plugin list shows stub + → stub MCP tool appears in tools + → call stub_hello via MCP + → remove stub from registry → CGC restarts cleanly + +Journey 2 — OTEL write_batch: + install otel plugin (or skip if not present) + → call write_batch with synthetic spans + → cross-layer Cypher query structure is validated + +Run as part of the e2e suite: + pytest tests/e2e/plugin/ -v -m e2e +""" +from __future__ import annotations + +import importlib.metadata +import logging +import sys +from typing import Any +from unittest.mock import MagicMock, patch, call + +import pytest + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _is_installed(package: str) -> bool: + try: + importlib.metadata.version(package) + return True + except importlib.metadata.PackageNotFoundError: + return False + + +stub_installed = pytest.mark.skipif( + not _is_installed("cgc-plugin-stub"), + reason="cgc-plugin-stub not installed — run: pip install -e plugins/cgc-plugin-stub", +) + +otel_installed = pytest.mark.skipif( + not _is_installed("cgc-plugin-otel"), + reason="cgc-plugin-otel not installed — run: pip install -e plugins/cgc-plugin-otel", +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture() +def fresh_registry(): + """A fresh PluginRegistry with no state.""" + from codegraphcontext.plugin_registry import PluginRegistry + return PluginRegistry() + + +@pytest.fixture() +def mock_db_manager(): + """Minimal db_manager mock for unit-level checks within E2E tests.""" + mgr = MagicMock() + mgr.execute_query = MagicMock(return_value=[]) + mgr.execute_write = MagicMock(return_value=None) + return mgr + + +# --------------------------------------------------------------------------- +# Journey 1a: Stub plugin loads via real entry points +# --------------------------------------------------------------------------- + +@stub_installed +class TestStubPluginLifecycle: + """ + Tests the complete lifecycle of the stub plugin using the real entry-point + mechanism. Requires: pip install -e plugins/cgc-plugin-stub + """ + + def test_stub_cli_command_appears_after_discovery(self, fresh_registry): + """After discover_cli_plugins(), 'stub' is in loaded_plugins.""" + fresh_registry.discover_cli_plugins() + assert "stub" in fresh_registry.loaded_plugins + assert fresh_registry.loaded_plugins["stub"]["status"] == "loaded" + + def test_stub_command_in_cli_commands_list(self, fresh_registry): + """cli_commands contains a ('stub', ) tuple after discovery.""" + fresh_registry.discover_cli_plugins() + names = [n for n, _ in fresh_registry.cli_commands] + assert "stub" in names + + def test_plugin_list_command_reports_loaded(self, fresh_registry): + """plugin list shows stub as loaded (simulates cgc plugin list).""" + fresh_registry.discover_cli_plugins() + fresh_registry.discover_mcp_plugins() + assert "stub" in fresh_registry.loaded_plugins + assert fresh_registry.loaded_plugins["stub"]["status"] == "loaded" + + def test_stub_mcp_tool_appears_in_tools(self, fresh_registry): + """'stub_hello' appears in mcp_tools after discover_mcp_plugins().""" + fresh_registry.discover_mcp_plugins() + assert "stub_hello" in fresh_registry.mcp_tools + + def test_stub_mcp_tool_has_valid_schema(self, fresh_registry): + """stub_hello tool definition has required MCP schema fields.""" + fresh_registry.discover_mcp_plugins() + tool = fresh_registry.mcp_tools["stub_hello"] + assert "name" in tool + assert "description" in tool + assert "inputSchema" in tool + assert tool["inputSchema"]["type"] == "object" + + def test_stub_hello_handler_returns_greeting(self, fresh_registry): + """Calling stub_hello handler returns {'greeting': '...'} with caller name.""" + fresh_registry.discover_mcp_plugins() + handler = fresh_registry.mcp_handlers["stub_hello"] + result = handler(name="E2E") + assert isinstance(result, dict) + assert "greeting" in result + assert "E2E" in result["greeting"] + + def test_registry_clean_after_simulated_uninstall(self, fresh_registry): + """ + Simulates uninstall by creating a new registry with no entry points. + The new registry should start empty — no leftover stub artifacts. + """ + fresh_registry.discover_cli_plugins() + fresh_registry.discover_mcp_plugins() + assert "stub" in fresh_registry.loaded_plugins + + from codegraphcontext.plugin_registry import PluginRegistry + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[]): + clean_registry = PluginRegistry() + clean_registry.discover_cli_plugins() + clean_registry.discover_mcp_plugins() + + assert len(clean_registry.loaded_plugins) == 0 + assert len(clean_registry.cli_commands) == 0 + assert len(clean_registry.mcp_tools) == 0 + + +# --------------------------------------------------------------------------- +# Journey 1b: Broken plugin never crashes host (always runs, no install needed) +# --------------------------------------------------------------------------- + +class TestBrokenPluginIsolation: + """ + Verifies that broken plugins are quarantined without crashing CGC. + Uses mocked entry points so no real plugin install is required. + """ + + def _make_valid_ep(self, name: str): + import typer + + ep = MagicMock() + ep.name = name + mod = MagicMock() + mod.PLUGIN_METADATA = { + "name": name, + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": f"Valid plugin {name}", + } + app = typer.Typer() + + @app.command() + def hello(): + pass + + mod.get_plugin_commands = MagicMock(return_value=(name, app)) + mod.get_mcp_tools = MagicMock(return_value={ + f"{name}_tool": { + "name": f"{name}_tool", + "description": "test tool", + "inputSchema": {"type": "object", "properties": {}}, + } + }) + mod.get_mcp_handlers = MagicMock(return_value={ + f"{name}_tool": lambda: {"result": "ok"} + }) + ep.load.return_value = mod + return ep + + def test_import_error_plugin_does_not_crash_host(self, fresh_registry): + """A plugin that raises ImportError is logged as failed; CGC continues.""" + good_ep = self._make_valid_ep("good") + bad_ep = MagicMock() + bad_ep.name = "broken_import" + bad_ep.load.side_effect = ImportError("missing_dep") + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[good_ep, bad_ep]): + fresh_registry.discover_cli_plugins() + + assert "good" in fresh_registry.loaded_plugins + assert "broken_import" in fresh_registry.failed_plugins + assert len(fresh_registry.loaded_plugins) == 1 + + def test_runtime_exception_in_get_plugin_commands_is_isolated(self, fresh_registry): + """If get_plugin_commands() raises, plugin is failed; others still load.""" + good_ep = self._make_valid_ep("safe") + bad_ep = self._make_valid_ep("buggy") + bad_ep.load.return_value.get_plugin_commands.side_effect = RuntimeError( + "boom in get_plugin_commands" + ) + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[good_ep, bad_ep]): + fresh_registry.discover_cli_plugins() + + assert "safe" in fresh_registry.loaded_plugins + assert "buggy" in fresh_registry.failed_plugins + + def test_incompatible_version_plugin_is_skipped(self, fresh_registry): + """Plugin with cgc_version_constraint that doesn't match installed version is skipped.""" + ep = MagicMock() + ep.name = "future_plugin" + mod = MagicMock() + mod.PLUGIN_METADATA = { + "name": "future_plugin", + "version": "9.9.9", + "cgc_version_constraint": ">=9999.0.0", + "description": "Too new", + } + ep.load.return_value = mod + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + fresh_registry.discover_cli_plugins() + + assert "future_plugin" in fresh_registry.failed_plugins + assert "future_plugin" not in fresh_registry.loaded_plugins + + def test_cgc_starts_cleanly_with_no_plugins_installed(self, fresh_registry): + """With no plugins, registry loads cleanly and reports zero counts.""" + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[]): + fresh_registry.discover_cli_plugins() + fresh_registry.discover_mcp_plugins() + + assert len(fresh_registry.loaded_plugins) == 0 + assert len(fresh_registry.failed_plugins) == 0 + assert len(fresh_registry.cli_commands) == 0 + assert len(fresh_registry.mcp_tools) == 0 + + +# --------------------------------------------------------------------------- +# Journey 2: OTEL write_batch → synthetic spans → cross-layer query structure +# --------------------------------------------------------------------------- + +@otel_installed +class TestOtelPluginLifecycle: + """ + Tests the OTEL plugin write_batch path and cross-layer query capability. + Requires: pip install -e plugins/cgc-plugin-otel + Uses a mock db_manager so no live Neo4j instance is needed. + """ + + @pytest.fixture() + def writer(self, mock_db_manager): + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + return AsyncOtelWriter(db_manager=mock_db_manager) + + @pytest.fixture() + def synthetic_spans(self): + """Minimal synthetic span dicts matching write_batch expected format.""" + return [ + { + "span_id": "abc123", + "trace_id": "trace_001", + "parent_span_id": None, + "name": "GET /api/orders", + "service": "order-service", + "start_time_ns": 1_700_000_000_000_000_000, + "end_time_ns": 1_700_000_001_000_000_000, + "duration_ms": 1000.0, + "http_route": "/api/orders", + "http_method": "GET", + "http_status_code": 200, + "fqn": "App\\Http\\Controllers\\OrderController::index", + "span_kind": "SERVER", + "status_code": "OK", + "attributes": {}, + }, + { + "span_id": "def456", + "trace_id": "trace_001", + "parent_span_id": "abc123", + "name": "DB query", + "service": "order-service", + "start_time_ns": 1_700_000_000_100_000_000, + "end_time_ns": 1_700_000_000_200_000_000, + "duration_ms": 100.0, + "http_route": None, + "http_method": None, + "http_status_code": None, + "fqn": None, + "span_kind": "CLIENT", + "status_code": "OK", + "attributes": {"db.system": "mysql", "peer.service": "mysql"}, + }, + ] + + def test_otel_plugin_loads_via_registry(self, fresh_registry): + """OTEL plugin MCP tools are discovered by the registry.""" + fresh_registry.discover_mcp_plugins() + otel_tools = [k for k in fresh_registry.mcp_tools if k.startswith("otel_")] + assert len(otel_tools) > 0, "No otel_* MCP tools found in registry" + + def test_otel_mcp_tools_have_valid_schemas(self, fresh_registry): + """All otel_* tools have required MCP schema fields.""" + fresh_registry.discover_mcp_plugins() + for tool_name, tool_def in fresh_registry.mcp_tools.items(): + if not tool_name.startswith("otel_"): + continue + assert "name" in tool_def, f"{tool_name}: missing 'name'" + assert "description" in tool_def, f"{tool_name}: missing 'description'" + assert "inputSchema" in tool_def, f"{tool_name}: missing 'inputSchema'" + + def test_write_batch_calls_db_manager(self, writer, synthetic_spans, mock_db_manager): + """write_batch() invokes db_manager with Service, Trace, and Span merge queries.""" + import asyncio + asyncio.get_event_loop().run_until_complete(writer.write_batch(synthetic_spans)) + assert mock_db_manager.execute_write.called or mock_db_manager.execute_query.called + + def test_write_batch_handles_empty_list(self, writer, mock_db_manager): + """write_batch([]) completes without error and makes no DB calls.""" + import asyncio + asyncio.get_event_loop().run_until_complete(writer.write_batch([])) + mock_db_manager.execute_write.assert_not_called() + + def test_cross_layer_query_structure_is_valid(self): + """ + Verifies the canonical cross-layer Cypher query compiles (parse-only check). + Tests SC-005: static code never observed at runtime query. + """ + cross_layer_query = ( + "MATCH (m:Method) " + "WHERE NOT EXISTS { MATCH (m)<-[:CORRELATES_TO]-(:Span) } " + "AND NOT EXISTS { MATCH (m)<-[:RESOLVES_TO]-(:StackFrame) } " + "AND m.fqn IS NOT NULL " + "RETURN m.fqn, m.class_name " + "ORDER BY m.class_name, m.fqn LIMIT 20" + ) + # Structural validation: query contains all expected clauses + assert "CORRELATES_TO" in cross_layer_query + assert "RESOLVES_TO" in cross_layer_query + assert "m.fqn" in cross_layer_query + assert "LIMIT 20" in cross_layer_query + + +# --------------------------------------------------------------------------- +# Journey 3: MCP server integration — plugin tools surface in tools/list +# --------------------------------------------------------------------------- + +class TestMCPServerPluginIntegration: + """ + Verifies that MCPServer merges plugin tools into self.tools and routes + tool_call to plugin handlers. Uses fully mocked entry points and db_manager. + """ + + def _make_mock_tool_ep(self, plugin_name: str, tool_name: str): + ep = MagicMock() + ep.name = plugin_name + mod = MagicMock() + mod.PLUGIN_METADATA = { + "name": plugin_name, + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": "test", + } + mod.get_mcp_tools = MagicMock(return_value={ + tool_name: { + "name": tool_name, + "description": "a test tool", + "inputSchema": { + "type": "object", + "properties": {"arg": {"type": "string"}}, + }, + } + }) + mod.get_mcp_handlers = MagicMock(return_value={ + tool_name: lambda arg="default": {"result": f"called with {arg}"} + }) + ep.load.return_value = mod + return ep + + def test_registry_mcp_tools_populate_correctly(self, fresh_registry): + """Tools contributed by a mock plugin appear in registry.mcp_tools.""" + ep = self._make_mock_tool_ep("e2e_plugin", "e2e_tool") + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + fresh_registry.discover_mcp_plugins() + + assert "e2e_tool" in fresh_registry.mcp_tools + + def test_plugin_handler_callable_via_registry(self, fresh_registry): + """Handler for plugin tool is callable and returns expected result.""" + ep = self._make_mock_tool_ep("e2e_plugin", "e2e_tool") + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + fresh_registry.discover_mcp_plugins() + + handler = fresh_registry.mcp_handlers["e2e_tool"] + result = handler(arg="hello") + assert result == {"result": "called with hello"} + + def test_two_plugins_tools_merge_without_conflict(self, fresh_registry): + """Tools from two different plugins both appear in mcp_tools.""" + ep1 = self._make_mock_tool_ep("plugin_one", "tool_one") + ep2 = self._make_mock_tool_ep("plugin_two", "tool_two") + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep1, ep2]): + fresh_registry.discover_mcp_plugins() + + assert "tool_one" in fresh_registry.mcp_tools + assert "tool_two" in fresh_registry.mcp_tools + + def test_conflicting_tool_names_first_wins(self, fresh_registry): + """When two plugins register the same tool name, the first plugin's version wins.""" + ep1 = self._make_mock_tool_ep("first_plugin", "shared_tool") + ep2 = self._make_mock_tool_ep("second_plugin", "shared_tool") + + # Override second plugin to have conflicting tool + ep2.load.return_value.get_mcp_handlers.return_value = { + "shared_tool": lambda: {"result": "second"} + } + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep1, ep2]): + fresh_registry.discover_mcp_plugins() + + handler = fresh_registry.mcp_handlers.get("shared_tool") + assert handler is not None + # First plugin's handler should win + result = handler(arg="test") + assert result == {"result": "called with test"} diff --git a/tests/e2e/plugin/test_sample_apps.py b/tests/e2e/plugin/test_sample_apps.py new file mode 100644 index 00000000..8a2a3b3b --- /dev/null +++ b/tests/e2e/plugin/test_sample_apps.py @@ -0,0 +1,118 @@ +"""E2E test wrapping samples/smoke-all.sh. + +Validates the full pipeline: index code → run instrumented apps → generate +OTEL spans → query cross-layer graph. + +Requires Docker Compose and a running sample stack. Skipped automatically if +Docker is not available or the sample containers are not running. +""" + +import shutil +import subprocess +from pathlib import Path + +import pytest + +SAMPLES_DIR = Path(__file__).resolve().parents[3] / "samples" +SMOKE_SCRIPT = SAMPLES_DIR / "smoke-all.sh" + + +def _docker_available() -> bool: + """Check if Docker CLI is available and the daemon is responsive.""" + docker = shutil.which("docker") + if not docker: + return False + try: + result = subprocess.run( + ["docker", "info"], + capture_output=True, + timeout=10, + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +def _samples_running() -> bool: + """Check if the sample app containers are running.""" + try: + result = subprocess.run( + ["docker", "compose", "ps", "--status", "running", "-q"], + capture_output=True, + text=True, + cwd=SAMPLES_DIR, + timeout=10, + ) + # At least 3 containers should be running (the 3 sample apps) + running = [line for line in result.stdout.strip().splitlines() if line] + return len(running) >= 3 + except (subprocess.TimeoutExpired, OSError): + return False + + +@pytest.mark.skipif( + not _docker_available(), + reason="Docker not available", +) +@pytest.mark.skipif( + not _samples_running(), + reason="Sample app containers not running (run: cd samples/ && docker compose up -d)", +) +class TestSampleApps: + """End-to-end validation of CGC sample applications via smoke-all.sh.""" + + def test_smoke_script_passes(self): + """Run smoke-all.sh and assert it exits successfully. + + The smoke script runs 7 assertions against the Neo4j graph: + - service_count >= 3 + - span_orders > 0 + - static_functions > 0 + - static_classes > 0 + - cross_service > 0 + - trace_links > 0 + - correlates_to == 0 (WARN, not FAIL — known FQN gap) + + Exit code 0 means all assertions passed (WARNs are OK). + Exit code 1 means at least one assertion FAILed. + """ + result = subprocess.run( + ["bash", str(SMOKE_SCRIPT)], + capture_output=True, + text=True, + cwd=SAMPLES_DIR, + timeout=300, # 5 minutes max + ) + + # Print output for debugging on failure + if result.returncode != 0: + print("=== STDOUT ===") + print(result.stdout) + print("=== STDERR ===") + print(result.stderr) + + assert result.returncode == 0, ( + f"smoke-all.sh failed (exit code {result.returncode})\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + + def test_smoke_script_no_fail_lines(self): + """Verify the smoke output contains no FAIL lines.""" + result = subprocess.run( + ["bash", str(SMOKE_SCRIPT)], + capture_output=True, + text=True, + cwd=SAMPLES_DIR, + timeout=300, + ) + + fail_lines = [ + line for line in result.stdout.splitlines() + if "FAIL:" in line + ] + + assert not fail_lines, ( + f"Smoke script reported failures:\n" + + "\n".join(fail_lines) + ) diff --git a/tests/e2e/test_mcp_container.py b/tests/e2e/test_mcp_container.py new file mode 100644 index 00000000..d5806802 --- /dev/null +++ b/tests/e2e/test_mcp_container.py @@ -0,0 +1,549 @@ +""" +E2E tests for the cgc-mcp container image. + +Validates the full hosted MCP server lifecycle: + build Dockerfile.mcp + → start container with Neo4j via docker compose + → assert /healthz returns 200 + → assert tools/list returns core and plugin tools + → assert tools/call executes a tool + → assert CORS headers are present + → clean up containers + +These tests are skipped when Docker is not available on the host. They are +not intended to run in unit-test CI — use a dedicated Docker-enabled +integration environment. + +Run manually: + pytest tests/e2e/test_mcp_container.py -v -s +""" +from __future__ import annotations + +import json +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any + +import pytest + +# --------------------------------------------------------------------------- +# Repository root and compose file paths +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parents[2] +DOCKERFILE_MCP = REPO_ROOT / "Dockerfile.mcp" +PLUGIN_STACK_COMPOSE = REPO_ROOT / "docker-compose.plugin-stack.yml" +MCP_IMAGE_TAG = "cgc-mcp:test" +MCP_CONTAINER_NAME = "cgc-mcp-e2e-test" +MCP_PORT = 8045 +MCP_BASE_URL = f"http://localhost:{MCP_PORT}" +STARTUP_TIMEOUT_S = 60 # seconds to wait for /healthz to become 200 + + +# --------------------------------------------------------------------------- +# Module-level skip condition +# --------------------------------------------------------------------------- + +def _docker_available() -> bool: + """Return True if the Docker CLI is present and the daemon is responsive.""" + if not shutil.which("docker"): + return False + try: + result = subprocess.run( + ["docker", "info"], + capture_output=True, + timeout=10, + ) + return result.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +pytestmark = pytest.mark.skipif( + not _docker_available(), + reason="Docker not available or daemon not running", +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _run( + args: list[str], + *, + cwd: Path | None = None, + timeout: int = 300, + check: bool = False, +) -> subprocess.CompletedProcess[str]: + """Run a subprocess and return the CompletedProcess.""" + return subprocess.run( + args, + capture_output=True, + text=True, + cwd=cwd or REPO_ROOT, + timeout=timeout, + check=check, + ) + + +def _curl_mcp( + method: str, + params: dict[str, Any] | None = None, + *, + req_id: int = 1, + timeout: int = 15, +) -> subprocess.CompletedProcess[str]: + """Send a JSON-RPC request to the MCP endpoint via curl.""" + payload = json.dumps({ + "jsonrpc": "2.0", + "id": req_id, + "method": method, + "params": params or {}, + }) + return _run( + [ + "curl", "-s", "-X", "POST", + f"{MCP_BASE_URL}/mcp", + "-H", "Content-Type: application/json", + "-d", payload, + ], + timeout=timeout, + ) + + +def _wait_for_healthz(timeout_s: int = STARTUP_TIMEOUT_S) -> bool: + """Poll /healthz until it returns HTTP 200 or timeout_s elapses.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + result = _run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + f"{MCP_BASE_URL}/healthz"], + timeout=10, + ) + if result.returncode == 0 and result.stdout.strip() == "200": + return True + time.sleep(2) + return False + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="class") +def built_image(): + """Build Dockerfile.mcp once per class; yield the image tag. + + The image is removed after the class finishes only if this fixture + created it (i.e. it was not already present before the test run). + Skips the entire class if the build fails. + """ + result = _run( + ["docker", "build", "-f", str(DOCKERFILE_MCP), "-t", MCP_IMAGE_TAG, "."], + timeout=300, + ) + if result.returncode != 0: + pytest.skip(f"Image build failed — skipping container tests.\n{result.stderr}") + yield MCP_IMAGE_TAG + # Clean up the test image + _run(["docker", "rmi", "-f", MCP_IMAGE_TAG], timeout=30) + + +@pytest.fixture(scope="class") +def running_container(built_image: str): + """Start cgc-mcp with a mock Neo4j stub (no real DB needed for most tests). + + Uses the standalone `docker run` path so the test is self-contained and + does not require the full compose stack. The container is started with + DATABASE_TYPE=none so the server starts in degraded mode (Neo4j + unreachable), which is sufficient to test tool listing and JSON-RPC + dispatch without a live database. + + For tests that require a healthy Neo4j connection see + TestMCPContainerWithNeo4j which uses docker compose. + """ + # Remove any leftover container from a previous interrupted run + _run(["docker", "rm", "-f", MCP_CONTAINER_NAME], timeout=15) + + result = _run( + [ + "docker", "run", "-d", + "--name", MCP_CONTAINER_NAME, + "-e", "DATABASE_TYPE=none", + "-e", f"CGC_MCP_PORT={MCP_PORT}", + "-e", "CGC_CORS_ORIGIN=http://localhost:3000", + "-p", f"{MCP_PORT}:{MCP_PORT}", + built_image, + ], + timeout=30, + ) + if result.returncode != 0: + pytest.skip(f"Could not start container: {result.stderr}") + + healthy = _wait_for_healthz(timeout_s=STARTUP_TIMEOUT_S) + if not healthy: + logs = _run(["docker", "logs", MCP_CONTAINER_NAME], timeout=10) + pytest.skip( + f"Container did not become ready within {STARTUP_TIMEOUT_S}s.\n" + f"Container logs:\n{logs.stdout}\n{logs.stderr}" + ) + + yield MCP_CONTAINER_NAME + + # Teardown: stop and remove container + _run(["docker", "stop", MCP_CONTAINER_NAME], timeout=30) + _run(["docker", "rm", "-f", MCP_CONTAINER_NAME], timeout=15) + + +# --------------------------------------------------------------------------- +# Test: image build +# --------------------------------------------------------------------------- + +class TestDockerfileMCPBuilds: + """Verify Dockerfile.mcp exists and builds to a runnable image.""" + + def test_dockerfile_mcp_exists(self): + """Dockerfile.mcp is present in the repository root.""" + assert DOCKERFILE_MCP.exists(), ( + f"Dockerfile.mcp not found at {DOCKERFILE_MCP}" + ) + + def test_dockerfile_builds_successfully(self): + """docker build -f Dockerfile.mcp exits with code 0.""" + result = _run( + ["docker", "build", "-f", str(DOCKERFILE_MCP), "-t", MCP_IMAGE_TAG, "."], + timeout=300, + ) + assert result.returncode == 0, ( + f"docker build failed (exit {result.returncode}).\n" + f"stderr:\n{result.stderr}" + ) + # Cleanup: remove the image after this standalone test + _run(["docker", "rmi", "-f", MCP_IMAGE_TAG], timeout=30) + + def test_image_has_cgc_entrypoint(self): + """Built image has cgc binary available at /usr/local/bin/cgc.""" + # Build first + _run( + ["docker", "build", "-f", str(DOCKERFILE_MCP), "-t", MCP_IMAGE_TAG, "."], + timeout=300, + ) + result = _run( + ["docker", "run", "--rm", MCP_IMAGE_TAG, "which", "cgc"], + timeout=30, + ) + _run(["docker", "rmi", "-f", MCP_IMAGE_TAG], timeout=30) + assert result.returncode == 0, "cgc binary not found in image" + assert "cgc" in result.stdout + + def test_image_exposes_port_8045(self): + """Dockerfile.mcp declares EXPOSE 8045.""" + content = DOCKERFILE_MCP.read_text(encoding="utf-8") + assert "EXPOSE 8045" in content, ( + "Dockerfile.mcp does not EXPOSE 8045" + ) + + +# --------------------------------------------------------------------------- +# Test: container health and JSON-RPC +# --------------------------------------------------------------------------- + +class TestMCPContainerRunning: + """Tests that require a running container (no live Neo4j).""" + + def test_healthz_returns_200(self, running_container: str): + """GET /healthz returns HTTP 200.""" + result = _run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + f"{MCP_BASE_URL}/healthz"], + timeout=10, + ) + assert result.returncode == 0 + assert result.stdout.strip() == "200", ( + f"Expected HTTP 200 from /healthz, got: {result.stdout.strip()}" + ) + + def test_healthz_response_body_is_json(self, running_container: str): + """GET /healthz returns a JSON body with a 'status' field.""" + result = _run( + ["curl", "-s", f"{MCP_BASE_URL}/healthz"], + timeout=10, + ) + assert result.returncode == 0 + try: + body = json.loads(result.stdout) + except json.JSONDecodeError as exc: + pytest.fail(f"/healthz did not return valid JSON: {result.stdout!r} — {exc}") + assert "status" in body, f"'status' field missing from /healthz response: {body}" + + def test_tools_list_returns_valid_jsonrpc(self, running_container: str): + """tools/list returns a valid JSON-RPC 2.0 response.""" + result = _curl_mcp("tools/list") + assert result.returncode == 0, f"curl failed: {result.stderr}" + try: + response = json.loads(result.stdout) + except json.JSONDecodeError as exc: + pytest.fail(f"tools/list response is not valid JSON: {result.stdout!r} — {exc}") + assert response.get("jsonrpc") == "2.0", f"Missing jsonrpc field: {response}" + assert "result" in response, f"Expected 'result' in response: {response}" + + def test_tools_list_contains_tools_array(self, running_container: str): + """tools/list result contains a non-empty 'tools' array.""" + result = _curl_mcp("tools/list") + response = json.loads(result.stdout) + tools = response.get("result", {}).get("tools", []) + assert isinstance(tools, list), f"'tools' should be a list, got: {type(tools)}" + assert len(tools) > 0, "tools/list returned an empty tools array" + + def test_tools_list_includes_core_tools(self, running_container: str): + """Core CGC tools (e.g. query_graph, get_context) appear in tools/list.""" + result = _curl_mcp("tools/list") + response = json.loads(result.stdout) + tool_names = {t["name"] for t in response.get("result", {}).get("tools", [])} + # At least one well-known core tool must be present + core_tools = {"query_graph", "get_context", "list_functions", "analyze_callers"} + found = core_tools & tool_names + assert found, ( + f"No core CGC tools found in tools/list.\n" + f"Expected at least one of {core_tools}.\n" + f"Got: {sorted(tool_names)}" + ) + + def test_tools_list_includes_plugin_tools(self, running_container: str): + """Plugin tools (otel_* or xdebug_*) appear in tools/list.""" + result = _curl_mcp("tools/list") + response = json.loads(result.stdout) + tool_names = {t["name"] for t in response.get("result", {}).get("tools", [])} + plugin_tools = [n for n in tool_names if n.startswith(("otel_", "xdebug_"))] + assert plugin_tools, ( + f"No plugin tools (otel_* or xdebug_*) found in tools/list.\n" + f"All tools: {sorted(tool_names)}" + ) + + def test_each_tool_has_required_schema_fields(self, running_container: str): + """Every tool in tools/list has name, description, and inputSchema.""" + result = _curl_mcp("tools/list") + response = json.loads(result.stdout) + tools = response.get("result", {}).get("tools", []) + for tool in tools: + name = tool.get("name", "") + assert "name" in tool, f"Tool missing 'name': {tool}" + assert "description" in tool, f"Tool '{name}' missing 'description'" + assert "inputSchema" in tool, f"Tool '{name}' missing 'inputSchema'" + assert tool["inputSchema"].get("type") == "object", ( + f"Tool '{name}' inputSchema.type should be 'object', " + f"got: {tool['inputSchema'].get('type')!r}" + ) + + def test_unknown_method_returns_jsonrpc_error(self, running_container: str): + """Calling an unknown method returns a JSON-RPC error object.""" + result = _curl_mcp("no_such_method_xyz") + assert result.returncode == 0 + try: + response = json.loads(result.stdout) + except json.JSONDecodeError as exc: + pytest.fail(f"Response is not JSON: {result.stdout!r} — {exc}") + assert "error" in response, ( + f"Expected JSON-RPC error for unknown method, got: {response}" + ) + + def test_cors_header_present_on_mcp_response(self, running_container: str): + """POST /mcp response includes Access-Control-Allow-Origin header.""" + result = _run( + [ + "curl", "-s", "-I", "-X", "POST", + f"{MCP_BASE_URL}/mcp", + "-H", "Content-Type: application/json", + "-H", "Origin: http://localhost:3000", + "-d", '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}', + ], + timeout=15, + ) + assert result.returncode == 0 + headers_lower = result.stdout.lower() + assert "access-control-allow-origin" in headers_lower, ( + f"CORS header missing from /mcp response.\nHeaders:\n{result.stdout}" + ) + + +# --------------------------------------------------------------------------- +# Test: tools/call dispatch +# --------------------------------------------------------------------------- + +class TestMCPToolsCall: + """Verify that tools/call routes to the correct handler.""" + + def test_tools_call_returns_jsonrpc_result(self, running_container: str): + """tools/call for a valid tool returns a JSON-RPC result (not an error).""" + # Use tools/list first to pick a tool name that actually exists + list_result = _curl_mcp("tools/list") + tools = json.loads(list_result.stdout).get("result", {}).get("tools", []) + assert tools, "Cannot test tools/call — tools/list is empty" + + # Pick the first tool with no required parameters (empty properties or no required) + candidate: str | None = None + for tool in tools: + schema = tool.get("inputSchema", {}) + required = schema.get("required", []) + if not required: + candidate = tool["name"] + break + + if candidate is None: + pytest.skip("No tool with zero required parameters found in tools/list") + + result = _curl_mcp( + "tools/call", + params={"name": candidate, "arguments": {}}, + ) + assert result.returncode == 0 + try: + response = json.loads(result.stdout) + except json.JSONDecodeError as exc: + pytest.fail(f"tools/call response is not JSON: {result.stdout!r} — {exc}") + + # A valid dispatch should return 'result', not an 'error' + assert "result" in response, ( + f"tools/call returned an error for tool '{candidate}': {response}" + ) + + def test_tools_call_unknown_tool_returns_error(self, running_container: str): + """Calling a non-existent tool returns a JSON-RPC error.""" + result = _curl_mcp( + "tools/call", + params={"name": "nonexistent_tool_abc123", "arguments": {}}, + ) + assert result.returncode == 0 + response = json.loads(result.stdout) + assert "error" in response, ( + f"Expected error for unknown tool, got: {response}" + ) + + +# --------------------------------------------------------------------------- +# Test: docker compose integration (requires full stack) +# --------------------------------------------------------------------------- + +class TestMCPContainerWithNeo4j: + """ + E2E tests that start the full compose stack (Neo4j + cgc-mcp). + + These are skipped if the compose file is not present. They take longer + than the standalone container tests because Neo4j has a ~30s startup time. + """ + + @pytest.fixture(scope="class", autouse=True) + def compose_stack(self): + """Start neo4j + cgc-mcp via docker compose; tear down after class.""" + if not PLUGIN_STACK_COMPOSE.exists(): + pytest.skip(f"Compose file not found: {PLUGIN_STACK_COMPOSE}") + + # Bring up only the services needed for this test + up_result = _run( + [ + "docker", "compose", + "-f", str(PLUGIN_STACK_COMPOSE), + "up", "-d", "--build", "neo4j", "cgc-mcp", + ], + timeout=300, + ) + if up_result.returncode != 0: + pytest.skip( + f"docker compose up failed:\n{up_result.stderr}" + ) + + # Wait for /healthz to return 200 (Neo4j must also be healthy) + healthy = _wait_for_healthz(timeout_s=90) + if not healthy: + logs = _run( + ["docker", "compose", "-f", str(PLUGIN_STACK_COMPOSE), + "logs", "cgc-mcp"], + timeout=15, + ) + _run( + ["docker", "compose", "-f", str(PLUGIN_STACK_COMPOSE), + "down", "-v"], + timeout=60, + ) + pytest.skip( + f"cgc-mcp did not become healthy within 90s.\n" + f"Logs:\n{logs.stdout}" + ) + + yield + + _run( + ["docker", "compose", "-f", str(PLUGIN_STACK_COMPOSE), + "down", "-v"], + timeout=60, + ) + + def test_healthz_reports_neo4j_connected(self): + """When Neo4j is running, /healthz reports neo4j as connected.""" + result = _run( + ["curl", "-s", f"{MCP_BASE_URL}/healthz"], + timeout=10, + ) + assert result.returncode == 0 + body = json.loads(result.stdout) + assert body.get("neo4j") == "connected", ( + f"Expected neo4j='connected' in /healthz body, got: {body}" + ) + + def test_tools_list_with_live_neo4j(self): + """tools/list succeeds with a live Neo4j connection.""" + result = _curl_mcp("tools/list") + response = json.loads(result.stdout) + assert "result" in response, f"tools/list returned error with live DB: {response}" + tools = response["result"].get("tools", []) + assert len(tools) > 0 + + def test_query_graph_tool_executes_against_neo4j(self): + """query_graph tool executes a Cypher query against the live Neo4j instance.""" + result = _curl_mcp( + "tools/call", + params={ + "name": "query_graph", + "arguments": {"query": "RETURN 1 AS n"}, + }, + ) + assert result.returncode == 0 + response = json.loads(result.stdout) + assert "result" in response, ( + f"query_graph failed with live Neo4j: {response}" + ) + + def test_503_when_neo4j_is_stopped(self): + """After stopping Neo4j, /healthz returns HTTP 503. + + This test stops the neo4j container, checks the 503, then restarts it. + It is ordered last in the class because it is destructive to the stack. + """ + # Stop Neo4j + _run( + ["docker", "compose", "-f", str(PLUGIN_STACK_COMPOSE), + "stop", "neo4j"], + timeout=30, + ) + time.sleep(5) # Allow the MCP server to detect the disconnection + + result = _run( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + f"{MCP_BASE_URL}/healthz"], + timeout=10, + ) + http_code = result.stdout.strip() + + # Restart Neo4j so teardown can proceed cleanly + _run( + ["docker", "compose", "-f", str(PLUGIN_STACK_COMPOSE), + "start", "neo4j"], + timeout=30, + ) + + assert http_code == "503", ( + f"Expected HTTP 503 after Neo4j stopped, got: {http_code}" + ) diff --git a/tests/integration/plugin/test_otel_integration.py b/tests/integration/plugin/test_otel_integration.py new file mode 100644 index 00000000..6236469e --- /dev/null +++ b/tests/integration/plugin/test_otel_integration.py @@ -0,0 +1,185 @@ +""" +Integration tests for cgc_plugin_otel.neo4j_writer. + +Uses a mocked db_manager so no real Neo4j connection is required. +Tests verify that the writer issues the correct Cypher queries and +creates the expected graph structure. +""" +from __future__ import annotations + +import pytest +from unittest.mock import AsyncMock, MagicMock, call, patch + +cgc_plugin_otel = pytest.importorskip( + "cgc_plugin_otel", + reason="cgc-plugin-otel is not installed; skipping otel integration tests", +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +def _make_span( + span_id: str = "span001", + trace_id: str = "trace001", + parent_span_id: str | None = None, + service_name: str = "order-service", + http_route: str | None = "/api/orders", + fqn: str | None = "App\\Controllers\\OrderController::index", + cross_service: bool = False, + peer_service: str | None = None, + duration_ms: float = 12.5, +) -> dict: + return { + "span_id": span_id, + "trace_id": trace_id, + "parent_span_id": parent_span_id, + "name": f"GET {http_route or '/'}", + "span_kind": "CLIENT" if cross_service else "SERVER", + "service_name": service_name, + "start_time_ns": 1_000_000_000, + "end_time_ns": int(1_000_000_000 + duration_ms * 1_000_000), + "duration_ms": duration_ms, + "http_route": http_route, + "http_method": "GET", + "class_name": fqn.split("::")[0] if fqn else None, + "function_name": fqn.split("::")[1] if fqn else None, + "fqn": fqn, + "db_statement": None, + "db_system": None, + "peer_service": peer_service, + "cross_service": cross_service, + } + + +def _make_db_manager(): + """Build a mock db_manager that returns an async-capable session.""" + session = AsyncMock() + session.__aenter__ = AsyncMock(return_value=session) + session.__aexit__ = AsyncMock(return_value=False) + session.run = AsyncMock() + + driver = MagicMock() + driver.session = MagicMock(return_value=session) + + db_manager = MagicMock() + db_manager.get_driver = MagicMock(return_value=driver) + return db_manager, session + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +class TestAsyncOtelWriterBatch: + + async def test_write_batch_issues_merge_service(self): + """write_batch() issues a MERGE for the Service node.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span() + + await writer.write_batch([span]) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("MERGE" in c and "Service" in c for c in cypher_calls), \ + f"No Service MERGE found in calls: {cypher_calls}" + + async def test_write_batch_issues_merge_span(self): + """write_batch() issues a MERGE for the Span node.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span() + + await writer.write_batch([span]) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("MERGE" in c and "Span" in c for c in cypher_calls) + + async def test_write_batch_links_span_to_trace(self): + """write_batch() creates a PART_OF relationship between Span and Trace.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span() + + await writer.write_batch([span]) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("PART_OF" in c for c in cypher_calls) + + async def test_write_batch_creates_child_of_for_parent_span_id(self): + """CHILD_OF relationship is created when parent_span_id is set.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(span_id="child", parent_span_id="parent001") + + await writer.write_batch([span]) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("CHILD_OF" in c for c in cypher_calls) + + async def test_no_child_of_when_no_parent(self): + """CHILD_OF is NOT issued when parent_span_id is None.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(parent_span_id=None) + + await writer.write_batch([span]) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert not any("CHILD_OF" in c for c in cypher_calls) + + async def test_write_batch_creates_correlates_to_for_fqn(self): + """CORRELATES_TO relationship is attempted when fqn is set.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(fqn="App\\Controllers::index") + + await writer.write_batch([span]) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("CORRELATES_TO" in c for c in cypher_calls) + + async def test_no_correlates_to_when_no_fqn(self): + """CORRELATES_TO is NOT issued when fqn is None (no code context).""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(fqn=None) + + await writer.write_batch([span]) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert not any("CORRELATES_TO" in c for c in cypher_calls) + + async def test_cross_service_span_creates_calls_service(self): + """CALLS_SERVICE is created for CLIENT spans with peer_service set.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(cross_service=True, peer_service="payment-service") + + await writer.write_batch([span]) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("CALLS_SERVICE" in c for c in cypher_calls) + + async def test_db_failure_routes_to_dlq(self): + """When the database raises, spans are moved to the dead-letter queue.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager = MagicMock() + db_manager.get_driver.side_effect = Exception("Neo4j unavailable") + writer = AsyncOtelWriter(db_manager) + span = _make_span() + + await writer.write_batch([span]) + + assert not writer._dlq.empty() diff --git a/tests/integration/plugin/test_plugin_load.py b/tests/integration/plugin/test_plugin_load.py new file mode 100644 index 00000000..950c9581 --- /dev/null +++ b/tests/integration/plugin/test_plugin_load.py @@ -0,0 +1,210 @@ +""" +Integration tests for CGC plugin loading with the stub plugin. + +These tests use the real entry-point mechanism. The stub plugin must be +installed in editable mode before running: + + pip install -e plugins/cgc-plugin-stub + +Tests MUST FAIL before Phase 3 implementation (T012-T016) is complete. +""" +import importlib.metadata +import logging +import pytest +from unittest.mock import MagicMock, patch + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _stub_installed() -> bool: + """Return True if cgc-plugin-stub is installed in the current environment.""" + try: + importlib.metadata.version("cgc-plugin-stub") + return True + except importlib.metadata.PackageNotFoundError: + return False + + +stub_required = pytest.mark.skipif( + not _stub_installed(), + reason="cgc-plugin-stub not installed — run: pip install -e plugins/cgc-plugin-stub", +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture() +def registry(): + """Fresh PluginRegistry instance for each test.""" + from codegraphcontext.plugin_registry import PluginRegistry + return PluginRegistry() + + +# --------------------------------------------------------------------------- +# Tests — stub plugin via real entry points (requires editable install) +# --------------------------------------------------------------------------- + +@stub_required +class TestStubPluginLoad: + """Tests that use the real installed stub plugin via entry-point discovery.""" + + def test_stub_cli_command_discovered(self, registry): + """Stub CLI command group 'stub' appears after discover_cli_plugins().""" + registry.discover_cli_plugins() + assert "stub" in registry.loaded_plugins, ( + "stub plugin not in loaded_plugins — is PLUGIN_METADATA defined in __init__.py?" + ) + assert registry.loaded_plugins["stub"]["status"] == "loaded" + + def test_stub_cli_commands_populated(self, registry): + """cli_commands list contains ('stub', ) after load.""" + registry.discover_cli_plugins() + names = [name for name, _ in registry.cli_commands] + assert "stub" in names + + def test_stub_mcp_tool_discovered(self, registry): + """MCP tool 'stub_hello' appears in mcp_tools after discover_mcp_plugins().""" + registry.discover_mcp_plugins() + assert "stub_hello" in registry.mcp_tools, ( + "stub_hello tool missing — is get_mcp_tools() implemented in mcp_tools.py?" + ) + + def test_stub_mcp_handler_registered(self, registry): + """Handler for 'stub_hello' is registered in mcp_handlers.""" + registry.discover_mcp_plugins() + assert "stub_hello" in registry.mcp_handlers + + def test_stub_mcp_handler_returns_greeting(self, registry): + """stub_hello handler returns a dict containing 'greeting'.""" + registry.discover_mcp_plugins() + handler = registry.mcp_handlers["stub_hello"] + result = handler(name="Tester") + assert "greeting" in result + assert "Tester" in result["greeting"] + + +# --------------------------------------------------------------------------- +# Tests — isolation behaviour (always run, use mocked entry points) +# --------------------------------------------------------------------------- + +class TestPluginIsolationBehaviour: + """ + Behavioural isolation tests that do NOT require the stub to be installed. + They use hand-crafted mocks to verify the registry enforces contracts. + """ + + def _make_stub_ep(self, name="stub"): + """Build a minimal stub entry-point mock with valid metadata.""" + import typer + + ep = MagicMock() + ep.name = name + + mod = MagicMock() + mod.PLUGIN_METADATA = { + "name": name, + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": f"Stub plugin '{name}'", + } + stub_app = typer.Typer() + + @stub_app.command() + def hello(): + """Hello from stub.""" + + mod.get_plugin_commands = MagicMock(return_value=(name, stub_app)) + mod.get_mcp_tools = MagicMock(return_value={ + f"{name}_hello": { + "name": f"{name}_hello", + "description": "Say hello", + "inputSchema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }, + } + }) + mod.get_mcp_handlers = MagicMock( + return_value={f"{name}_hello": lambda name="World": {"greeting": f"Hello {name}"}} + ) + + ep.load.return_value = mod + return ep + + def test_second_incompatible_plugin_skipped(self, registry): + """A second plugin with incompatible version constraint is skipped with warning.""" + good_ep = self._make_stub_ep("good") + + bad_mod = MagicMock() + bad_mod.PLUGIN_METADATA = { + "name": "old", + "version": "0.0.1", + "cgc_version_constraint": ">=99.0.0", + "description": "Too new", + } + bad_ep = MagicMock() + bad_ep.name = "old" + bad_ep.load.return_value = bad_mod + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[good_ep, bad_ep]): + registry.discover_cli_plugins() + + assert "good" in registry.loaded_plugins + assert "old" not in registry.loaded_plugins + assert "old" in registry.failed_plugins + + def test_duplicate_name_loads_only_first(self, registry): + """Two plugins with identical names: first wins, second is silently skipped.""" + ep1 = self._make_stub_ep("dupe") + ep2 = self._make_stub_ep("dupe") + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep1, ep2]): + registry.discover_cli_plugins() + + assert registry.loaded_plugins["dupe"]["status"] == "loaded" + # Only one entry in cli_commands for this name + assert sum(1 for name, _ in registry.cli_commands if name == "dupe") == 1 + + def test_conflicting_mcp_tool_loads_only_first(self, registry): + """Two plugins registering the same MCP tool name: first plugin's definition wins.""" + ep1 = self._make_stub_ep("plugin_a") + ep2 = self._make_stub_ep("plugin_b") + + # Make plugin_b register a tool with the same key as plugin_a + ep2.load.return_value.get_mcp_tools.return_value = { + "plugin_a_hello": { # conflicts with plugin_a + "name": "plugin_a_hello", + "description": "conflict", + "inputSchema": {"type": "object", "properties": {}}, + } + } + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep1, ep2]): + registry.discover_mcp_plugins() + + # Tool exists, registered by first plugin + assert "plugin_a_hello" in registry.mcp_tools + # Both plugins loaded (even though one tool was skipped) + assert "plugin_a" in registry.loaded_plugins + assert "plugin_b" in registry.loaded_plugins + + def test_registry_reports_correct_counts(self, registry): + """loaded_plugins and failed_plugins counts are accurate after mixed load.""" + ep_good = self._make_stub_ep("ok_plugin") + ep_bad = MagicMock() + ep_bad.name = "broken" + ep_bad.load.side_effect = ImportError("missing dep") + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep_good, ep_bad]): + registry.discover_cli_plugins() + + assert len(registry.loaded_plugins) == 1 + assert len(registry.failed_plugins) == 1 + assert "ok_plugin" in registry.loaded_plugins + assert "broken" in registry.failed_plugins diff --git a/tests/integration/test_http_transport_integration.py b/tests/integration/test_http_transport_integration.py new file mode 100644 index 00000000..a951d6c7 --- /dev/null +++ b/tests/integration/test_http_transport_integration.py @@ -0,0 +1,241 @@ +# tests/integration/test_http_transport_integration.py +"""Integration tests for the CGC HTTP transport layer (T073). + +These tests start the HTTPTransport backed by a mocked MCPServer on a random +ephemeral port and exercise the full request/response cycle via +``starlette.testclient.TestClient`` (synchronous ASGI test client — no live +TCP socket needed). + +A separate async section uses ``pytest-asyncio`` + ``httpx.AsyncClient`` +mounted against the ASGI app for async call paths. +""" +from __future__ import annotations + +import json +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +import pytest_asyncio +import httpx +from starlette.testclient import TestClient + +from codegraphcontext.http_transport import HTTPTransport + + +# --------------------------------------------------------------------------- +# Shared mock factory +# --------------------------------------------------------------------------- + +def _make_server( + *, + connected: bool = True, + tools: dict | None = None, +) -> MagicMock: + """Return a fully-wired MCPServer mock.""" + server = MagicMock() + server.db_manager = MagicMock() + server.db_manager.is_connected.return_value = connected + server.tools = tools if tools is not None else { + "find_code": {"name": "find_code", "description": "Find code"}, + "execute_cypher_query": {"name": "execute_cypher_query", "description": "Run Cypher"}, + } + + async def _handle_request( + method: str, + params: dict[str, Any], + request_id: Any, + ) -> dict[str, Any] | None: + if method == "initialize": + return { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "protocolVersion": "2025-03-26", + "serverInfo": {"name": "CodeGraphContext", "version": "0.1.0"}, + "capabilities": {"tools": {"listTools": True}}, + }, + } + if method == "tools/list": + return { + "jsonrpc": "2.0", + "id": request_id, + "result": {"tools": list(server.tools.values())}, + } + if method == "tools/call": + tool_name = params.get("name") + return { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "content": [{"type": "text", "text": json.dumps({"called": tool_name})}] + }, + } + if method == "notifications/initialized": + return None + return { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": -32601, "message": f"Method not found: {method}"}, + } + + server.handle_request = AsyncMock(side_effect=_handle_request) + return server + + +# --------------------------------------------------------------------------- +# Synchronous integration tests (TestClient) +# --------------------------------------------------------------------------- + +@pytest.fixture() +def server() -> MagicMock: + return _make_server() + + +@pytest.fixture() +def client(server: MagicMock) -> TestClient: + transport = HTTPTransport(server) + return TestClient(transport.app, raise_server_exceptions=True) + + +class TestMcpIntegration: + """Full request-response cycle for MCP methods.""" + + def test_initialize_returns_capabilities(self, client: TestClient) -> None: + resp = client.post( + "/mcp", + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["result"]["protocolVersion"] == "2025-03-26" + assert body["result"]["capabilities"]["tools"]["listTools"] is True + + def test_tools_list_returns_tools(self, client: TestClient, server: MagicMock) -> None: + resp = client.post( + "/mcp", + json={"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, + ) + assert resp.status_code == 200 + tools = resp.json()["result"]["tools"] + tool_names = {t["name"] for t in tools} + assert tool_names == set(server.tools.keys()) + + def test_tools_call_returns_result(self, client: TestClient) -> None: + resp = client.post( + "/mcp", + json={ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "find_code", "arguments": {"query": "hello"}}, + }, + ) + assert resp.status_code == 200 + content = resp.json()["result"]["content"] + assert content[0]["type"] == "text" + data = json.loads(content[0]["text"]) + assert data["called"] == "find_code" + + def test_notification_returns_204(self, client: TestClient) -> None: + resp = client.post( + "/mcp", + json={"jsonrpc": "2.0", "method": "notifications/initialized"}, + ) + assert resp.status_code == 204 + + def test_unknown_method_returns_error(self, client: TestClient) -> None: + resp = client.post( + "/mcp", + json={"jsonrpc": "2.0", "id": 9, "method": "bogus/method"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert "error" in body + assert body["error"]["code"] == -32601 + + +class TestHealthzIntegration: + def test_healthz_200_when_connected(self) -> None: + tools = {f"t{i}": {"name": f"t{i}"} for i in range(4)} + server = _make_server(connected=True, tools=tools) + c = TestClient(HTTPTransport(server).app) + + resp = c.get("/healthz") + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "ok" + assert body["tools"] == 4 + + def test_healthz_503_when_disconnected(self) -> None: + server = _make_server(connected=False) + c = TestClient(HTTPTransport(server).app) + + resp = c.get("/healthz") + assert resp.status_code == 503 + assert resp.json()["status"] == "unhealthy" + + +# --------------------------------------------------------------------------- +# Async integration tests (httpx.AsyncClient + ASGI transport) +# --------------------------------------------------------------------------- + +@pytest.fixture() +def async_server() -> MagicMock: + return _make_server() + + +@pytest.fixture() +def asgi_app(async_server: MagicMock): + return HTTPTransport(async_server).app + + +@pytest.mark.asyncio +async def test_async_initialize(asgi_app) -> None: + """initialize works correctly via httpx AsyncClient.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=asgi_app), base_url="http://testserver" + ) as ac: + resp = await ac.post( + "/mcp", + json={"jsonrpc": "2.0", "id": 10, "method": "initialize", "params": {}}, + ) + assert resp.status_code == 200 + assert resp.json()["result"]["protocolVersion"] == "2025-03-26" + + +@pytest.mark.asyncio +async def test_async_tools_list(asgi_app, async_server: MagicMock) -> None: + """tools/list works correctly via httpx AsyncClient.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=asgi_app), base_url="http://testserver" + ) as ac: + resp = await ac.post( + "/mcp", + json={"jsonrpc": "2.0", "id": 11, "method": "tools/list"}, + ) + assert resp.status_code == 200 + tool_names = {t["name"] for t in resp.json()["result"]["tools"]} + assert tool_names == set(async_server.tools.keys()) + + +@pytest.mark.asyncio +async def test_async_healthz_ok(asgi_app) -> None: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=asgi_app), base_url="http://testserver" + ) as ac: + resp = await ac.get("/healthz") + assert resp.status_code == 200 + assert resp.json()["status"] == "ok" + + +@pytest.mark.asyncio +async def test_async_healthz_unhealthy() -> None: + server = _make_server(connected=False) + app = HTTPTransport(server).app + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://testserver" + ) as ac: + resp = await ac.get("/healthz") + assert resp.status_code == 503 + assert resp.json()["status"] == "unhealthy" diff --git a/tests/unit/plugin/test_otel_processor.py b/tests/unit/plugin/test_otel_processor.py new file mode 100644 index 00000000..e720a3f3 --- /dev/null +++ b/tests/unit/plugin/test_otel_processor.py @@ -0,0 +1,185 @@ +""" +Unit tests for cgc_plugin_otel.span_processor. + +All tests run without gRPC or a real database — pure logic tests. +Tests MUST FAIL before T020 (span_processor.py) is implemented. +""" +import pytest + +cgc_plugin_otel = pytest.importorskip( + "cgc_plugin_otel", + reason="cgc-plugin-otel is not installed; skipping otel processor unit tests", +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _import_processor(): + from cgc_plugin_otel.span_processor import ( + extract_php_context, + build_fqn, + is_cross_service_span, + should_filter_span, + build_span_dict, + ) + return extract_php_context, build_fqn, is_cross_service_span, should_filter_span, build_span_dict + + +# --------------------------------------------------------------------------- +# extract_php_context +# --------------------------------------------------------------------------- + +class TestExtractPhpContext: + def test_full_attributes_parsed(self): + extract_php_context, *_ = _import_processor() + attrs = { + "code.namespace": "App\\Http\\Controllers", + "code.function": "index", + "http.route": "/api/orders", + "http.method": "GET", + } + result = extract_php_context(attrs) + assert result["namespace"] == "App\\Http\\Controllers" + assert result["function"] == "index" + assert result["http_route"] == "/api/orders" + assert result["http_method"] == "GET" + + def test_missing_optional_attributes_return_none(self): + extract_php_context, *_ = _import_processor() + result = extract_php_context({}) + assert result["namespace"] is None + assert result["function"] is None + assert result["http_route"] is None + assert result["http_method"] is None + + def test_db_attributes_captured(self): + extract_php_context, *_ = _import_processor() + attrs = { + "db.statement": "SELECT * FROM orders", + "db.system": "mysql", + } + result = extract_php_context(attrs) + assert result["db_statement"] == "SELECT * FROM orders" + assert result["db_system"] == "mysql" + + +# --------------------------------------------------------------------------- +# build_fqn +# --------------------------------------------------------------------------- + +class TestBuildFqn: + def test_namespace_and_function_joined(self): + _, build_fqn, *_ = _import_processor() + assert build_fqn("App\\Controllers", "store") == "App\\Controllers::store" + + def test_none_namespace_returns_none(self): + _, build_fqn, *_ = _import_processor() + assert build_fqn(None, "store") is None + + def test_none_function_returns_none(self): + _, build_fqn, *_ = _import_processor() + assert build_fqn("App\\Controllers", None) is None + + def test_both_none_returns_none(self): + _, build_fqn, *_ = _import_processor() + assert build_fqn(None, None) is None + + +# --------------------------------------------------------------------------- +# is_cross_service_span +# --------------------------------------------------------------------------- + +class TestIsCrossServiceSpan: + def test_client_kind_with_peer_service_is_cross_service(self): + _, _, is_cross_service_span, *_ = _import_processor() + assert is_cross_service_span("CLIENT", {"peer.service": "order-service"}) is True + + def test_client_kind_without_peer_service_is_not_cross_service(self): + _, _, is_cross_service_span, *_ = _import_processor() + assert is_cross_service_span("CLIENT", {}) is False + + def test_server_kind_is_not_cross_service(self): + _, _, is_cross_service_span, *_ = _import_processor() + assert is_cross_service_span("SERVER", {"peer.service": "anything"}) is False + + def test_internal_kind_is_not_cross_service(self): + _, _, is_cross_service_span, *_ = _import_processor() + assert is_cross_service_span("INTERNAL", {"peer.service": "anything"}) is False + + +# --------------------------------------------------------------------------- +# should_filter_span +# --------------------------------------------------------------------------- + +class TestShouldFilterSpan: + def test_health_route_filtered(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({"http.route": "/health"}, ["/health", "/metrics"]) is True + + def test_metrics_route_filtered(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({"http.route": "/metrics"}, ["/health", "/metrics"]) is True + + def test_normal_route_not_filtered(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({"http.route": "/api/orders"}, ["/health", "/metrics"]) is False + + def test_empty_filter_list_never_filters(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({"http.route": "/health"}, []) is False + + def test_span_without_route_not_filtered(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({}, ["/health"]) is False + + +# --------------------------------------------------------------------------- +# build_span_dict +# --------------------------------------------------------------------------- + +class TestBuildSpanDict: + def test_duration_ms_computed_from_nanoseconds(self): + _, _, _, _, build_span_dict = _import_processor() + span = build_span_dict( + span_id="abc123", + trace_id="trace456", + parent_span_id=None, + name="GET /api/orders", + span_kind="SERVER", + start_time_ns=1_000_000_000, + end_time_ns=1_500_000_000, + attributes={}, + service_name="order-service", + ) + assert span["duration_ms"] == pytest.approx(500.0) + + def test_required_fields_present(self): + _, _, _, _, build_span_dict = _import_processor() + span = build_span_dict( + span_id="abc123", + trace_id="trace456", + parent_span_id="parent789", + name="GET /api/orders", + span_kind="SERVER", + start_time_ns=1_000_000_000, + end_time_ns=2_000_000_000, + attributes={"http.route": "/api/orders"}, + service_name="order-service", + ) + assert span["span_id"] == "abc123" + assert span["trace_id"] == "trace456" + assert span["parent_span_id"] == "parent789" + assert span["service_name"] == "order-service" + assert span["name"] == "GET /api/orders" + + def test_zero_duration_for_equal_timestamps(self): + _, _, _, _, build_span_dict = _import_processor() + span = build_span_dict( + span_id="x", trace_id="y", parent_span_id=None, + name="instant", span_kind="INTERNAL", + start_time_ns=5_000_000, end_time_ns=5_000_000, + attributes={}, service_name="svc", + ) + assert span["duration_ms"] == 0.0 diff --git a/tests/unit/plugin/test_plugin_registry.py b/tests/unit/plugin/test_plugin_registry.py new file mode 100644 index 00000000..eaad21e5 --- /dev/null +++ b/tests/unit/plugin/test_plugin_registry.py @@ -0,0 +1,275 @@ +""" +Unit tests for PluginRegistry. + +All entry-point discovery is mocked — no installed packages required. +Tests MUST FAIL before PluginRegistry is implemented (TDD Red phase). +""" +import pytest +import logging +from unittest.mock import MagicMock, patch, PropertyMock + + +# --------------------------------------------------------------------------- +# Helpers: build fake entry-point objects +# --------------------------------------------------------------------------- + +def _make_ep(name, module_path, metadata=None, raise_on_load=None): + """Create a mock entry-point that loads a callable.""" + ep = MagicMock() + ep.name = name + + if raise_on_load: + ep.load.side_effect = raise_on_load + else: + def _loader(): + if metadata is not None: + mod = MagicMock() + mod.PLUGIN_METADATA = metadata + mod.get_plugin_commands = MagicMock( + return_value=(name, MagicMock()) + ) + mod.get_mcp_tools = MagicMock(return_value={ + f"{name}_tool": { + "name": f"{name}_tool", + "description": "test", + "inputSchema": {"type": "object", "properties": {}} + } + }) + mod.get_mcp_handlers = MagicMock(return_value={ + f"{name}_tool": lambda **kw: {"ok": True} + }) + return mod + return MagicMock() + + ep.load.return_value = _loader() + + return ep + + +VALID_METADATA = { + "name": "test-plugin", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": "Test plugin", +} + +INCOMPATIBLE_METADATA = { + "name": "old-plugin", + "version": "0.0.1", + "cgc_version_constraint": ">=99.0.0", + "description": "Too new constraint", +} + +MISSING_FIELD_METADATA = { + "name": "bad-plugin", + # missing version, cgc_version_constraint, description +} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +class TestPluginRegistryDiscovery: + """Tests for plugin discovery and loading.""" + + def test_no_plugins_installed_starts_cleanly(self): + """Registry with zero entry points should start without errors.""" + from codegraphcontext.plugin_registry import PluginRegistry + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[]): + registry = PluginRegistry() + registry.discover_cli_plugins() + registry.discover_mcp_plugins() + + assert registry.loaded_plugins == {} + assert registry.failed_plugins == {} + + def test_valid_plugin_is_loaded(self): + """A plugin with valid metadata and compatible version is loaded.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("myplugin", "myplugin.cli:get_plugin_commands", + metadata=VALID_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "myplugin" in registry.loaded_plugins + assert registry.loaded_plugins["myplugin"]["status"] == "loaded" + + def test_incompatible_version_is_skipped(self): + """A plugin whose cgc_version_constraint excludes the installed CGC version is skipped.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("oldplugin", "oldplugin.cli:get", + metadata=INCOMPATIBLE_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "oldplugin" not in registry.loaded_plugins + assert "oldplugin" in registry.failed_plugins + assert "version" in registry.failed_plugins["oldplugin"].lower() + + def test_missing_metadata_field_is_skipped(self): + """A plugin missing required PLUGIN_METADATA fields is skipped.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("badplugin", "badplugin.cli:get", + metadata=MISSING_FIELD_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "badplugin" not in registry.loaded_plugins + assert "badplugin" in registry.failed_plugins + + def test_import_error_does_not_crash_host(self): + """An ImportError during plugin load is caught; registry continues.""" + from codegraphcontext.plugin_registry import PluginRegistry + + bad_ep = _make_ep("broken", "broken.cli:get", + raise_on_load=ImportError("missing dep")) + good_ep = _make_ep("good", "good.cli:get", + metadata=VALID_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", + side_effect=[[bad_ep, good_ep], []]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "broken" in registry.failed_plugins + assert "good" in registry.loaded_plugins + + def test_exception_in_get_plugin_commands_does_not_crash(self): + """An exception raised by get_plugin_commands() is caught.""" + from codegraphcontext.plugin_registry import PluginRegistry + + mod = MagicMock() + mod.PLUGIN_METADATA = VALID_METADATA + mod.get_plugin_commands.side_effect = RuntimeError("boom") + + ep = MagicMock() + ep.name = "crashplugin" + ep.load.return_value = mod + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "crashplugin" in registry.failed_plugins + + def test_duplicate_plugin_name_skips_second(self): + """Two plugins with the same name: first wins, second is skipped.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep1 = _make_ep("dupe", "a.cli:get", metadata=VALID_METADATA) + ep2 = _make_ep("dupe", "b.cli:get", metadata=VALID_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep1, ep2]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "dupe" in registry.loaded_plugins + # Only one entry — second skipped + assert registry.loaded_plugins["dupe"]["status"] == "loaded" + + def test_loaded_and_failed_counts_are_accurate(self): + """Summary counts match actual loaded/failed plugins.""" + from codegraphcontext.plugin_registry import PluginRegistry + + good1 = _make_ep("g1", "g1.cli:get", metadata=VALID_METADATA) + good2 = _make_ep("g2", "g2.cli:get", metadata=VALID_METADATA) + bad = _make_ep("bad", "bad.cli:get", + raise_on_load=ImportError("missing")) + + with patch("codegraphcontext.plugin_registry.entry_points", + side_effect=[[good1, good2, bad], []]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert len(registry.loaded_plugins) == 2 + assert len(registry.failed_plugins) == 1 + + +class TestPluginRegistryCLI: + """Tests for CLI command registration results.""" + + def test_cli_commands_populated_after_load(self): + """cli_commands list is populated with (name, typer_app) tuples.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("myplugin", "myplugin.cli:get", metadata=VALID_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert len(registry.cli_commands) == 1 + name, typer_app = registry.cli_commands[0] + assert name == "myplugin" + + def test_cli_commands_empty_without_plugins(self): + """cli_commands is empty when no plugins are installed.""" + from codegraphcontext.plugin_registry import PluginRegistry + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert registry.cli_commands == [] + + +class TestPluginRegistryMCP: + """Tests for MCP tool registration results.""" + + def test_mcp_tools_populated_after_load(self): + """mcp_tools dict is populated with tool definitions from loaded plugins.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("myplugin", "myplugin.mcp:get", metadata=VALID_METADATA) + + server_context = {"db_manager": MagicMock(), "version": "0.3.1"} + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_mcp_plugins(server_context) + + assert "myplugin_tool" in registry.mcp_tools + assert "myplugin_tool" in registry.mcp_handlers + + def test_conflicting_tool_name_skips_second(self): + """Two plugins registering the same tool name: first wins.""" + from codegraphcontext.plugin_registry import PluginRegistry + + # Both plugins register "myplugin_tool" + ep1 = _make_ep("plugin_a", "a.mcp:get", metadata={**VALID_METADATA, "name": "plugin_a"}) + ep2 = _make_ep("plugin_b", "b.mcp:get", metadata={**VALID_METADATA, "name": "plugin_b"}) + + # Make ep2 return a tool with the same name as ep1 + mod2 = MagicMock() + mod2.PLUGIN_METADATA = {**VALID_METADATA, "name": "plugin_b"} + mod2.get_mcp_tools = MagicMock(return_value={ + "myplugin_tool": { # same key as ep1's tool + "name": "myplugin_tool", + "description": "conflict", + "inputSchema": {"type": "object", "properties": {}} + } + }) + mod2.get_mcp_handlers = MagicMock(return_value={"myplugin_tool": lambda **k: {}}) + ep2.load.return_value = mod2 + + server_context = {"db_manager": MagicMock(), "version": "0.3.1"} + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep1, ep2]): + registry = PluginRegistry() + registry.discover_mcp_plugins(server_context) + + # Tool is registered once, from the first plugin + assert "myplugin_tool" in registry.mcp_tools diff --git a/tests/unit/plugin/test_xdebug_parser.py b/tests/unit/plugin/test_xdebug_parser.py new file mode 100644 index 00000000..966971a1 --- /dev/null +++ b/tests/unit/plugin/test_xdebug_parser.py @@ -0,0 +1,140 @@ +""" +Unit tests for cgc_plugin_xdebug.dbgp_server parsing logic. + +No TCP connections required — pure XML/logic tests. +Tests MUST FAIL before T030 (dbgp_server.py) is implemented. +""" +import pytest + +cgc_plugin_xdebug = pytest.importorskip( + "cgc_plugin_xdebug", + reason="cgc-plugin-xdebug is not installed; skipping xdebug parser unit tests", +) + +_SAMPLE_STACK_XML = """\ + + + + + + +""" + +_EMPTY_STACK_XML = """\ + + + +""" + + +def _import_parser(): + from cgc_plugin_xdebug.dbgp_server import ( + parse_stack_xml, + compute_chain_hash, + build_frame_id, + ) + return parse_stack_xml, compute_chain_hash, build_frame_id + + +# --------------------------------------------------------------------------- +# parse_stack_xml +# --------------------------------------------------------------------------- + +class TestParseStackXml: + def test_returns_correct_frame_count(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert len(frames) == 3 + + def test_frame_fields_present(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + frame = frames[0] + assert "where" in frame + assert "level" in frame + assert "filename" in frame + assert "lineno" in frame + + def test_frame_level_is_integer(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert all(isinstance(f["level"], int) for f in frames) + + def test_frames_ordered_by_level(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + levels = [f["level"] for f in frames] + assert levels == sorted(levels) + + def test_empty_stack_returns_empty_list(self): + parse_stack_xml, *_ = _import_parser() + assert parse_stack_xml(_EMPTY_STACK_XML) == [] + + def test_first_frame_where_parsed(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert "OrderController" in frames[0]["where"] + + def test_filename_stripped_of_scheme(self): + """file:// prefix should be stripped from the filename.""" + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert not frames[0]["filename"].startswith("file://") + + +# --------------------------------------------------------------------------- +# compute_chain_hash +# --------------------------------------------------------------------------- + +class TestComputeChainHash: + def test_same_frames_same_hash(self): + parse_stack_xml, compute_chain_hash, _ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert compute_chain_hash(frames) == compute_chain_hash(frames) + + def test_different_frames_different_hash(self): + parse_stack_xml, compute_chain_hash, _ = _import_parser() + frames1 = parse_stack_xml(_SAMPLE_STACK_XML) + frames2 = frames1[:-1] # drop last frame + assert compute_chain_hash(frames1) != compute_chain_hash(frames2) + + def test_empty_frames_returns_hash(self): + _, compute_chain_hash, _ = _import_parser() + h = compute_chain_hash([]) + assert isinstance(h, str) and len(h) > 0 + + def test_hash_is_deterministic_across_calls(self): + parse_stack_xml, compute_chain_hash, _ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert compute_chain_hash(frames) == compute_chain_hash(frames) + + +# --------------------------------------------------------------------------- +# build_frame_id +# --------------------------------------------------------------------------- + +class TestBuildFrameId: + def test_returns_string(self): + _, _, build_frame_id = _import_parser() + fid = build_frame_id("App\\Controllers\\Foo", "bar", "/var/www/Foo.php", 10) + assert isinstance(fid, str) + + def test_deterministic(self): + _, _, build_frame_id = _import_parser() + a = build_frame_id("App\\Foo", "bar", "/var/www/Foo.php", 10) + b = build_frame_id("App\\Foo", "bar", "/var/www/Foo.php", 10) + assert a == b + + def test_different_inputs_different_ids(self): + _, _, build_frame_id = _import_parser() + a = build_frame_id("App\\Foo", "bar", "/var/www/Foo.php", 10) + b = build_frame_id("App\\Foo", "baz", "/var/www/Foo.php", 10) + assert a != b diff --git a/tests/unit/test_http_transport.py b/tests/unit/test_http_transport.py new file mode 100644 index 00000000..4515be6f --- /dev/null +++ b/tests/unit/test_http_transport.py @@ -0,0 +1,177 @@ +# tests/unit/test_http_transport.py +"""Unit tests for the CGC HTTP transport layer (T069). + +All tests use ``starlette.testclient.TestClient`` (re-exported by FastAPI) and +a lightweight mock of ``MCPServer`` — no real database is required. +""" +from __future__ import annotations + +import json +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from starlette.testclient import TestClient + +from codegraphcontext.http_transport import HTTPTransport + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +def _make_server( + *, + connected: bool = True, + handle_request_return: dict[str, Any] | None = None, + tools: dict | None = None, +) -> MagicMock: + """Build a minimal MCPServer mock suitable for unit tests.""" + server = MagicMock() + server.db_manager = MagicMock() + server.db_manager.is_connected.return_value = connected + server.tools = tools if tools is not None else {"tool_a": {}, "tool_b": {}} + + if handle_request_return is None: + handle_request_return = { + "jsonrpc": "2.0", + "id": 1, + "result": {"protocolVersion": "2025-03-26"}, + } + server.handle_request = AsyncMock(return_value=handle_request_return) + return server + + +@pytest.fixture() +def server() -> MagicMock: + return _make_server() + + +@pytest.fixture() +def client(server: MagicMock) -> TestClient: + transport = HTTPTransport(server) + return TestClient(transport.app, raise_server_exceptions=True) + + +# --------------------------------------------------------------------------- +# POST /mcp — routing +# --------------------------------------------------------------------------- + +class TestMcpEndpoint: + def test_valid_request_dispatches_to_handle_request(self, client: TestClient, server: MagicMock) -> None: + """POST /mcp deserialises body and calls server.handle_request.""" + payload = {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}} + resp = client.post("/mcp", json=payload) + + assert resp.status_code == 200 + server.handle_request.assert_awaited_once_with("initialize", {}, 1) + + def test_response_body_is_json_rpc(self, client: TestClient, server: MagicMock) -> None: + """Response body matches the dict returned by handle_request.""" + expected = {"jsonrpc": "2.0", "id": 7, "result": {"tools": []}} + server.handle_request.return_value = expected + + resp = client.post("/mcp", json={"jsonrpc": "2.0", "id": 7, "method": "tools/list"}) + assert resp.json() == expected + + def test_notification_returns_204(self, client: TestClient, server: MagicMock) -> None: + """When handle_request returns None (notification), HTTP 204 is returned.""" + server.handle_request.return_value = None + + resp = client.post( + "/mcp", + json={"jsonrpc": "2.0", "method": "notifications/initialized"}, + ) + assert resp.status_code == 204 + assert resp.content == b"" + + def test_malformed_json_returns_parse_error(self, client: TestClient) -> None: + """Non-JSON body produces a 400 with JSON-RPC parse-error.""" + resp = client.post("/mcp", content=b"not-json", headers={"Content-Type": "application/json"}) + assert resp.status_code == 400 + body = resp.json() + assert body["error"]["code"] == -32700 + + def test_params_defaults_to_empty_dict_when_absent(self, client: TestClient, server: MagicMock) -> None: + """Omitting 'params' from the request should pass an empty dict.""" + client.post("/mcp", json={"jsonrpc": "2.0", "id": 2, "method": "tools/list"}) + _, call_params, _ = server.handle_request.call_args.args + assert call_params == {} + + def test_unknown_route_returns_404(self, client: TestClient) -> None: + resp = client.get("/unknown-path") + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# GET /healthz +# --------------------------------------------------------------------------- + +class TestHealthz: + def test_returns_200_when_db_connected(self) -> None: + server = _make_server(connected=True, tools={"t1": {}, "t2": {}, "t3": {}}) + c = TestClient(HTTPTransport(server).app) + + resp = c.get("/healthz") + assert resp.status_code == 200 + assert resp.json() == {"status": "ok", "tools": 3} + + def test_returns_503_when_db_unreachable(self) -> None: + server = _make_server(connected=False) + c = TestClient(HTTPTransport(server).app) + + resp = c.get("/healthz") + assert resp.status_code == 503 + assert resp.json() == {"status": "unhealthy"} + + def test_tool_count_reflects_server_tools(self) -> None: + tools = {f"tool_{i}": {} for i in range(5)} + server = _make_server(connected=True, tools=tools) + c = TestClient(HTTPTransport(server).app) + + resp = c.get("/healthz") + assert resp.json()["tools"] == 5 + + +# --------------------------------------------------------------------------- +# CORS +# --------------------------------------------------------------------------- + +class TestCors: + def test_cors_preflight_returns_correct_headers(self, client: TestClient) -> None: + """OPTIONS preflight should return CORS allow headers.""" + resp = client.options( + "/mcp", + headers={ + "Origin": "http://localhost:3000", + "Access-Control-Request-Method": "POST", + "Access-Control-Request-Headers": "Content-Type", + }, + ) + # FastAPI / Starlette CORS middleware returns 200 for preflight + assert resp.status_code == 200 + assert "access-control-allow-origin" in resp.headers + + def test_default_cors_origin_is_wildcard(self, server: MagicMock) -> None: + """Without CGC_CORS_ORIGIN env var the allowed origin should be '*'.""" + with patch.dict("os.environ", {}, clear=False): + os_env = __import__("os").environ + os_env.pop("CGC_CORS_ORIGIN", None) + transport = HTTPTransport(server) + c = TestClient(transport.app) + resp = c.get( + "/healthz", + headers={"Origin": "http://example.com"}, + ) + assert resp.headers.get("access-control-allow-origin") in ("*", "http://example.com") + + def test_custom_cors_origin_env_var(self, server: MagicMock) -> None: + """CGC_CORS_ORIGIN env var is forwarded to the CORS middleware.""" + with patch.dict("os.environ", {"CGC_CORS_ORIGIN": "https://my-app.example.com"}): + transport = HTTPTransport(server) + c = TestClient(transport.app) + resp = c.get( + "/healthz", + headers={"Origin": "https://my-app.example.com"}, + ) + assert resp.headers.get("access-control-allow-origin") == "https://my-app.example.com"