diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..4b812d4a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,77 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + MCP_IMAGE_NAME: ${{ github.repository }}-mcp-server + +jobs: + release: + name: Create Draft Release + runs-on: ubuntu-latest + steps: + - name: Validate tag format + run: | + TAG="${{ github.ref_name }}" + if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Invalid tag format: ${TAG}. Expected vMAJOR.MINOR.PATCH (e.g., v0.12.0)" + exit 1 + fi + + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Determine release type + id: info + run: | + TAG="${{ github.ref_name }}" + PATCH_NUM="${TAG##*.}" + MINOR_VERSION="${TAG%.*}" + if [[ "$PATCH_NUM" == "0" ]]; then + echo "type=minor" >> "$GITHUB_OUTPUT" + else + echo "type=patch" >> "$GITHUB_OUTPUT" + fi + echo "minor_version=${MINOR_VERSION}" >> "$GITHUB_OUTPUT" + + - name: Run release script (minor) + if: steps.info.outputs.type == 'minor' + env: + GH_TOKEN: ${{ github.token }} + run: | + chmod +x hack/release.sh + hack/release.sh minor --version "${{ github.ref_name }}" --force + + - name: Run release script (patch) + if: steps.info.outputs.type == 'patch' + env: + GH_TOKEN: ${{ github.token }} + run: | + chmod +x hack/release.sh + hack/release.sh patch "${{ steps.info.outputs.minor_version }}" --version "${{ github.ref_name }}" --force + + - name: Summary + run: | + TAG="${{ github.ref_name }}" + { + echo "## Draft Release ${TAG}" + echo "" + echo "- **Release**: https://github.com/${{ github.repository }}/releases/tag/${TAG}" + echo "- **Controller image**: \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG}\`" + echo "- **MCP server image**: \`${{ env.REGISTRY }}/${{ env.MCP_IMAGE_NAME }}:${TAG}\`" + if [[ "${{ steps.info.outputs.type }}" == "minor" ]]; then + echo "- **Release branch**: \`release-${{ steps.info.outputs.minor_version }}.x\`" + fi + echo "" + echo "> **Note:** This is a draft release. Review the release notes and publish when ready." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..c9155911 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,119 @@ +# Local Development Setup + +This guide walks you through setting up the Unstructured Data Controller for local development. + +## Prerequisites + +- Docker +- [Kind](https://kind.sigs.k8s.io/) +- kubectl +- Go +- [yq](https://github.com/mikefarah/yq) +- [AWS CLI](https://aws.amazon.com/cli/) with [awslocal](https://github.com/localstack/awscli-local) +- [Docling](https://github.com/docling-project/docling-serve) (`pip install "docling-serve[ui]"`) +- [Ollama](https://ollama.com/) (for embedding models) + +## Quick start (recommended) + +Edit these files with your credentials and config before running setup: + +- `config/samples/unstructured-secret.yaml` — S3 credentials, embedding endpoint, API keys +- `config/samples/operator_v1alpha1_controllerconfig.yaml` — Docling URL, storage bucket, concurrency + +Then run: + +```bash +make local-dev-setup +``` + +This single command: + +1. Creates a Kind cluster (`unstructured-data-controller-local`) +2. Creates namespace `unstructured-controller-namespace` +3. Creates the local cache directory +4. Starts Docling (if `docling-serve` is installed) +5. Starts Ollama and pulls the embedding model +6. Starts LocalStack in Docker on `localhost:4566` (container `localstack-dev`) +7. Creates S3 buckets (ingestion, storage, output) +8. Installs CRDs +9. Applies secrets (`operator-secret` and `pipeline-secret`), ControllerConfig, and UnstructuredDataPipeline +10. Runs the controller (`make run`) + +Press Ctrl+C to stop the controller. LocalStack and the Kind cluster keep running. + +### Subsequent runs + +If setup was already done and you only need to run the controller: + +```bash +make run +``` + +Ensure LocalStack is still running: + +```bash +curl -sf http://127.0.0.1:4566/_localstack/health +``` + +If not healthy, start it again: + +```bash +./scripts/localstack.sh start +``` + +### Cleanup + +Remove the full local environment (Docling, Ollama, LocalStack container, cache, Kind cluster): + +```bash +make local-dev-cleanup +``` + +## LocalStack only + +To start or manage LocalStack without the full setup: + +```bash +./scripts/localstack.sh start # start Docker container +./scripts/localstack.sh setup # create S3 buckets +./scripts/localstack.sh status # health check and resource list +./scripts/localstack.sh stop # stop and remove container +``` + +View LocalStack API requests: + +```bash +docker logs -f localstack-dev +``` + +See [Setting up LocalStack](docs/setup-localstack.md) for manual setup steps. + +## Manual setup + +If you prefer step-by-step setup instead of `make local-dev-setup`: + +1. [Setting up LocalStack](docs/setup-localstack.md) +2. [Unstructured Data Controller with LocalStack](docs/setup-unstructured-controller.md) + +## Verification + +```bash +kubectl get controllerconfig controllerconfig -n unstructured-controller-namespace -o yaml +kubectl get unstructureddatapipeline -n unstructured-controller-namespace -o wide +kubectl get sourcecrawler,documentprocessor,chunksgenerator,vectorembeddingsgenerator,destinationsyncer -n unstructured-controller-namespace +``` + +The ControllerConfig should show `ConfigReady` with `status: "True"`. Pipeline stages should appear after the pipeline is applied. + +## Next steps + +Follow the [Creating Sample File Guide](docs/creating-sample-file.md) to upload a test file and process it through the pipeline. + +## Running e2e tests + +E2e tests use in-cluster LocalStack (not the host Docker container). See the test workflow in `.github/workflows/test-e2e.yml`. + +```bash +export IMG=/: +make docker-build docker-push test-e2e +``` diff --git a/Makefile b/Makefile index 18ed3ee6..9003e47b 100644 --- a/Makefile +++ b/Makefile @@ -142,13 +142,21 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist .PHONY: test-e2e test-e2e: setup-test-e2e ## Run the e2e tests. Expected an isolated environment using Kind. - go test -count=1 -tags e2e ./test/e2e/ -v -timeout=60m + KIND_CLUSTER=$(KIND_CLUSTER) SKIP_CLUSTER_SETUP=true go test -count=1 -tags e2e ./test/e2e/ -v -timeout=60m $(MAKE) cleanup-test-e2e .PHONY: cleanup-test-e2e cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests @$(KIND) delete cluster --name $(KIND_CLUSTER) +.PHONY: local-dev-setup +local-dev-setup: ## Full local dev setup (Kind, LocalStack Docker, CRDs, samples, run) + @chmod +x scripts/local-dev-setup.sh && ./scripts/local-dev-setup.sh + +.PHONY: local-dev-cleanup +local-dev-cleanup: ## Clean up local development environment + @chmod +x scripts/local-dev-cleanup.sh && ./scripts/local-dev-cleanup.sh + .PHONY: lint lint: golangci-lint ## Run golangci-lint and yamllint linters $(GOLANGCI_LINT) run diff --git a/hack/release.sh b/hack/release.sh new file mode 100755 index 00000000..af6f4931 --- /dev/null +++ b/hack/release.sh @@ -0,0 +1,405 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO="" +IMAGE_BASE="" +UPSTREAM_REMOTE="origin" +DRY_RUN=false +FORCE=false +VERSION_OVERRIDE="" + +# --- Colors --- +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +info() { echo -e "${BLUE}ℹ${NC} $*"; } +success() { echo -e "${GREEN}✔${NC} $*"; } +warn() { echo -e "${YELLOW}⚠${NC} $*"; } +error() { echo -e "${RED}✖${NC} $*" >&2; } +header() { echo -e "\n${BOLD}${CYAN}$*${NC}"; } +dry_run() { echo -e " ${YELLOW}[DRY RUN]${NC} $*"; } + +run_cmd() { + if $DRY_RUN; then + dry_run "$*" + else + info "Running: $*" + eval "$@" + fi +} + +usage() { + cat </dev/null; then + error "gh CLI not found. Install it: https://cli.github.com" + exit 1 + fi + if ! gh auth status &>/dev/null 2>&1; then + error "gh CLI not authenticated. Run: gh auth login" + exit 1 + fi + success "gh CLI authenticated" + + if ! git remote get-url "$UPSTREAM_REMOTE" &>/dev/null; then + error "Remote '${UPSTREAM_REMOTE}' not found." + echo " Add it with: git remote add ${UPSTREAM_REMOTE} git@github.com:/.git" + exit 1 + fi + + local remote_url + remote_url=$(git remote get-url "$UPSTREAM_REMOTE") + REPO=$(echo "$remote_url" | sed -E 's#.*github\.com[:/]##' | sed 's/\.git$//') + IMAGE_BASE="ghcr.io/${REPO}" + success "Remote '${UPSTREAM_REMOTE}': ${remote_url}" + success "Repo: ${REPO}" + + info "Fetching ${UPSTREAM_REMOTE}..." + git fetch "$UPSTREAM_REMOTE" --tags --force --quiet + success "Fetched latest refs and tags" +} + +# --- Version helpers --- +get_all_tags() { + git tag -l 'v*' | sort -V +} + +get_latest_minor() { + get_all_tags | { grep -oP 'v\K[0-9]+\.[0-9]+' || true; } | sort -t. -k1,1n -k2,2n | tail -1 +} + +get_latest_patch_for_minor() { + local minor="$1" + get_all_tags | { grep -P "^v${minor}\.[0-9]+$" || true; } | sort -V | tail -1 +} + +get_max_patch_number() { + local minor="$1" + get_all_tags | { grep -P "^v${minor}\.[0-9]+$" || true; } | { grep -oP '\.[0-9]+$' || true; } | tr -d '.' | sort -n | tail -1 +} + +get_latest_tag() { + get_all_tags | tail -1 +} + +# --- Minor release --- +do_minor() { + preflight + + header "Version detection" + + local latest_minor + latest_minor=$(get_latest_minor) + if [[ -z "$latest_minor" ]]; then + error "No existing tags found. Cannot determine next version." + exit 1 + fi + + local next_version next_minor_num release_branch prev_release_branch prev_tag + + if [[ -n "$VERSION_OVERRIDE" ]]; then + next_version="${VERSION_OVERRIDE}" + next_minor_num=$(echo "$next_version" | grep -oP 'v0\.\K[0-9]+') + release_branch="release-v0.${next_minor_num}.x" + + local prev_minor_num=$((next_minor_num - 1)) + prev_release_branch="release-v0.${prev_minor_num}.x" + prev_tag=$(get_latest_patch_for_minor "0.${prev_minor_num}") + else + local latest_minor_num + latest_minor_num=$(echo "$latest_minor" | cut -d. -f2) + next_minor_num=$((latest_minor_num + 1)) + next_version="v0.${next_minor_num}.0" + release_branch="release-v0.${next_minor_num}.x" + + prev_release_branch="release-v0.${latest_minor_num}.x" + prev_tag=$(get_latest_patch_for_minor "0.${latest_minor_num}") + fi + + info "Latest minor: v${latest_minor}" + info "Previous release branch: ${prev_release_branch}" + info "Latest tag on branch: ${prev_tag:-none}" + info "Next version: ${BOLD}${next_version}${NC}" + + local branch_exists=false + if git rev-parse "${UPSTREAM_REMOTE}/${release_branch}" &>/dev/null 2>&1; then + branch_exists=true + fi + + if ! git rev-parse "$next_version" &>/dev/null 2>&1; then + error "Tag ${next_version} does not exist. Create and push the tag first." + exit 1 + fi + + + local commits_since + if git rev-parse "${UPSTREAM_REMOTE}/${prev_release_branch}" &>/dev/null 2>&1; then + commits_since="${UPSTREAM_REMOTE}/${prev_release_branch}" + elif [[ -n "$prev_tag" ]]; then + commits_since="${prev_tag}" + fi + + if [[ -n "${commits_since:-}" ]]; then + local total_commits + total_commits=$(git rev-list --count "${commits_since}..${UPSTREAM_REMOTE}/main") + + if (( total_commits == 0 )); then + echo "" + warn "No new commits on ${UPSTREAM_REMOTE}/main since ${commits_since}" + if ! $FORCE; then + error "Nothing to release. Use --force to proceed anyway." + exit 1 + fi + else + header "Commits included (${commits_since}..${UPSTREAM_REMOTE}/main) — ${total_commits} commits" + git log --oneline "${commits_since}..${UPSTREAM_REMOTE}/main" | head -30 + if (( total_commits > 30 )); then + echo " ... and $((total_commits - 30)) more commits" + fi + echo "" + fi + fi + + if $DRY_RUN; then + header "Release plan (DRY RUN — no changes will be made)" + echo "" + dry_run "git branch ${release_branch} ${next_version} # create release branch" + dry_run "git push ${UPSTREAM_REMOTE} ${release_branch} # push release branch" + dry_run "gh release create ${next_version} --repo ${REPO} --generate-notes --draft --notes-start-tag ${prev_tag:-v0.0.0}" + echo "" + info "Images that will be built by CI:" + echo " ${IMAGE_BASE}:${next_version}" + echo " ${IMAGE_BASE}-mcp-server:${next_version}" + echo "" + info "To execute, run: $(basename "$0") minor" + return + fi + + header "Creating minor release ${next_version}" + + if $branch_exists; then + success "Branch ${release_branch} already exists — skipping" + else + run_cmd git branch "$release_branch" "$next_version" + run_cmd git push "$UPSTREAM_REMOTE" "$release_branch" + success "Created and pushed branch ${release_branch}" + fi + + if gh release view "$next_version" --repo "$REPO" &>/dev/null 2>&1; then + success "GitHub release ${next_version} already exists — skipping" + else + header "Creating draft GitHub release" + local notes_flag="" + if [[ -n "$prev_tag" ]]; then + notes_flag="--notes-start-tag ${prev_tag}" + fi + run_cmd gh release create "$next_version" \ + --repo "$REPO" \ + --generate-notes \ + --draft \ + $notes_flag + fi + + echo "" + success "Draft release ${next_version} created!" + info "Images:" + echo " ${IMAGE_BASE}:${next_version}" + echo " ${IMAGE_BASE}-mcp-server:${next_version}" + info "Release: https://github.com/${REPO}/releases/tag/${next_version}" +} + +# --- Patch release --- +do_patch() { + local minor_input="$1" + local minor + minor=$(echo "$minor_input" | sed 's/^v//') + + if [[ ! "$minor" =~ ^[0-9]+\.[0-9]+$ ]]; then + error "Invalid minor version format: ${minor_input}" + echo " Expected format: v0.X or 0.X (e.g., v0.10 or 0.10)" + exit 1 + fi + + preflight + + header "Version detection" + + local release_branch="release-v${minor}.x" + if ! git rev-parse "${UPSTREAM_REMOTE}/${release_branch}" &>/dev/null 2>&1; then + error "Release branch ${UPSTREAM_REMOTE}/${release_branch} not found." + echo " Available release branches:" + git branch -r --list "${UPSTREAM_REMOTE}/release-*" | sed 's/^/ /' + exit 1 + fi + success "Release branch: ${UPSTREAM_REMOTE}/${release_branch}" + + local latest_patch_tag + latest_patch_tag=$(get_latest_patch_for_minor "$minor") + local max_patch + max_patch=$(get_max_patch_number "$minor") + + if [[ -z "$max_patch" ]]; then + error "No existing tags found for v${minor}.x" + exit 1 + fi + + local next_version + if [[ -n "$VERSION_OVERRIDE" ]]; then + next_version="${VERSION_OVERRIDE}" + local override_patch="${next_version##*.}" + if (( override_patch > 0 )); then + latest_patch_tag="v${minor}.$((override_patch - 1))" + fi + else + local next_patch=$((max_patch + 1)) + next_version="v${minor}.${next_patch}" + fi + + info "Existing v${minor}.x tags: $(get_all_tags | grep -P "^v${minor}\.[0-9]+$" | tr '\n' ' ')" + info "Latest patch: ${latest_patch_tag}" + info "Next version: ${BOLD}${next_version}${NC}" + + if ! git rev-parse "$next_version" &>/dev/null 2>&1; then + error "Tag ${next_version} does not exist. Create and push the tag first." + exit 1 + fi + + local new_commits + new_commits=$(git rev-list --count "${latest_patch_tag}..${UPSTREAM_REMOTE}/${release_branch}") + + echo "" + if (( new_commits == 0 )); then + warn "No new commits on ${release_branch} since ${latest_patch_tag}" + if ! $FORCE; then + error "Nothing to release. Use --force to proceed anyway." + exit 1 + fi + else + header "Cherry-picks on ${release_branch} since ${latest_patch_tag} (${new_commits} commits)" + git log --oneline "${latest_patch_tag}..${UPSTREAM_REMOTE}/${release_branch}" + echo "" + fi + + if $DRY_RUN; then + header "Release plan (DRY RUN — no changes will be made)" + echo "" + dry_run "gh release create ${next_version} --repo ${REPO} --generate-notes --draft --notes-start-tag ${latest_patch_tag}" + echo "" + info "Images that will be built by CI:" + echo " ${IMAGE_BASE}:${next_version}" + echo " ${IMAGE_BASE}-mcp-server:${next_version}" + echo "" + info "To execute, run: $(basename "$0") patch ${minor_input}" + return + fi + + header "Creating patch release ${next_version}" + + if gh release view "$next_version" --repo "$REPO" &>/dev/null 2>&1; then + success "GitHub release ${next_version} already exists — skipping" + else + header "Creating draft GitHub release" + run_cmd gh release create "$next_version" \ + --repo "$REPO" \ + --generate-notes \ + --draft \ + --notes-start-tag "$latest_patch_tag" + fi + + echo "" + success "Draft release ${next_version} created!" + info "Images:" + echo " ${IMAGE_BASE}:${next_version}" + echo " ${IMAGE_BASE}-mcp-server:${next_version}" + info "Release: https://github.com/${REPO}/releases/tag/${next_version}" +} + +# --- Argument parsing --- +COMMAND="" +PATCH_MINOR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + minor) + COMMAND="minor" + shift + ;; + patch) + COMMAND="patch" + if [[ -z "${2:-}" ]]; then + error "patch requires a minor version argument (e.g., v0.10)" + echo " Usage: $(basename "$0") patch v0.X [OPTIONS]" + exit 1 + fi + PATCH_MINOR="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --upstream-remote) + UPSTREAM_REMOTE="${2:-}" + if [[ -z "$UPSTREAM_REMOTE" ]]; then + error "--upstream-remote requires a value" + exit 1 + fi + shift 2 + ;; + --version) + VERSION_OVERRIDE="${2:-}" + if [[ -z "$VERSION_OVERRIDE" ]]; then + error "--version requires a value (e.g., v0.11.0)" + exit 1 + fi + shift 2 + ;; + --force) + FORCE=true + shift + ;; + -h|--help) + usage + ;; + *) + error "Unknown argument: $1" + usage + ;; + esac +done + +if [[ -z "$COMMAND" ]]; then + usage +fi + +case "$COMMAND" in + minor) do_minor ;; + patch) do_patch "$PATCH_MINOR" ;; +esac diff --git a/scripts/local-dev-cleanup.sh b/scripts/local-dev-cleanup.sh new file mode 100755 index 00000000..cc402d6d --- /dev/null +++ b/scripts/local-dev-cleanup.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Cleanup for local-dev-setup.sh +# Usage: make local-dev-cleanup +set -e + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOCAL_KIND_CLUSTER="${LOCAL_KIND_CLUSTER:-unstructured-data-controller-local}" + +echo "Cleaning up local development environment..." + +if [ -f /tmp/docling.pid ]; then + DOCLING_PID=$(cat /tmp/docling.pid) + if ps -p "${DOCLING_PID}" > /dev/null 2>&1; then + kill "${DOCLING_PID}" 2>/dev/null || true + fi + rm -f /tmp/docling.pid /tmp/docling.log + echo "✓ Stopped Docling" +fi + +if [ -f /tmp/ollama.pid ]; then + OLLAMA_PID=$(cat /tmp/ollama.pid) + if ps -p "${OLLAMA_PID}" > /dev/null 2>&1; then + kill "${OLLAMA_PID}" 2>/dev/null || true + fi + rm -f /tmp/ollama.pid /tmp/ollama.log + echo "✓ Stopped Ollama" +fi + +LOCALSTACK_CONTAINER="${LOCALSTACK_CONTAINER:-localstack-dev}" + +if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "${LOCALSTACK_CONTAINER}"; then + docker rm -f "${LOCALSTACK_CONTAINER}" >/dev/null + echo "✓ Stopped and removed LocalStack container '${LOCALSTACK_CONTAINER}'" +else + echo "✓ No LocalStack container '${LOCALSTACK_CONTAINER}' found" +fi + +CACHE_DIR="${REPO_ROOT}/$( + yq -r '.spec.cacheDirectory' \ + "${REPO_ROOT}/config/samples/operator_v1alpha1_controllerconfig.yaml" 2>/dev/null \ + | sed 's|/$||' || echo "tmp/cache" +)" +if [ -d "${CACHE_DIR}" ]; then + rm -rf "${CACHE_DIR}" + echo "✓ Removed cache directory ${CACHE_DIR}" +fi + +kind delete cluster --name "${LOCAL_KIND_CLUSTER}" 2>/dev/null || true +echo "✓ Local development environment removed" diff --git a/scripts/local-dev-setup.sh b/scripts/local-dev-setup.sh new file mode 100755 index 00000000..2c1a4ed0 --- /dev/null +++ b/scripts/local-dev-setup.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Usage: make local-dev-setup +# +# Before running, edit: +# config/samples/unstructured-secret.yaml — credentials +# config/samples/operator_v1alpha1_controllerconfig.yaml — operator config +set -e + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +LOCAL_KIND_CLUSTER="${LOCAL_KIND_CLUSTER:-unstructured-data-controller-local}" +LOCAL_NAMESPACE="${LOCAL_NAMESPACE:-unstructured-controller-namespace}" + +CONTROLLER_CONFIG_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_controllerconfig.yaml" +SOURCE_CRAWLER_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_sourcecrawler.yaml" +DEST_SYNCER_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_destinationsyncer.yaml" +PIPELINE_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_unstructureddatapipeline.yaml" + +echo "Setting up local development environment..." + +# 1. Create Kind cluster +echo "" +echo "1. Creating Kind cluster '${LOCAL_KIND_CLUSTER}'..." +if kind get clusters 2>/dev/null | grep -q "^${LOCAL_KIND_CLUSTER}$"; then + echo "✓ Kind cluster '${LOCAL_KIND_CLUSTER}' already exists" +else + kind create cluster --name "${LOCAL_KIND_CLUSTER}" + echo "✓ Kind cluster '${LOCAL_KIND_CLUSTER}' created" +fi + +# 2. Create namespace +echo "" +echo "2. Creating namespace '${LOCAL_NAMESPACE}'..." +if kubectl get namespace "${LOCAL_NAMESPACE}" &>/dev/null; then + echo "✓ Namespace '${LOCAL_NAMESPACE}' already exists" +else + kubectl create namespace "${LOCAL_NAMESPACE}" + echo "✓ Namespace '${LOCAL_NAMESPACE}' created" +fi + +# 3. Create cache directory (from ControllerConfig sample) +echo "" +echo "3. Creating local cache directory..." +mkdir -p "${REPO_ROOT}/$( + yq -r '.spec.cacheDirectory' "${CONTROLLER_CONFIG_YAML}" \ + | sed 's|/$||' +)" +echo "✓ Cache directory created" + +# 4. Check/Start Docling +echo "" +echo "4. Checking Docling service..." +if command -v docling-serve &>/dev/null; then + if curl -s http://localhost:5001/health &>/dev/null; then + echo "✓ Docling is already running" + else + echo "Starting Docling in the background..." + nohup docling-serve run --enable-ui > /tmp/docling.log 2>&1 & + echo $! > /tmp/docling.pid + sleep 2 + if curl -s http://localhost:5001/health &>/dev/null; then + echo "✓ Docling started (log: /tmp/docling.log)" + else + echo "Warning: Docling may still be starting. Check /tmp/docling.log" + fi + fi +else + echo "⚠ Docling is not installed. Start it manually or install docling-serve." +fi + +# 5. Check/Start Ollama (for embedding models) +echo "" +echo "5. Checking Ollama service..." +if command -v ollama &>/dev/null; then + if curl -s http://localhost:11434/api/tags &>/dev/null; then + echo "✓ Ollama is already running" + else + echo "Starting Ollama in the background..." + nohup ollama serve > /tmp/ollama.log 2>&1 & + echo $! > /tmp/ollama.pid + sleep 2 + if curl -s http://localhost:11434/api/tags &>/dev/null; then + echo "✓ Ollama started (log: /tmp/ollama.log)" + else + echo "Warning: Ollama may still be starting. Check /tmp/ollama.log" + fi + fi + echo "Pulling embedding model..." + ollama pull nomic-embed-text:latest 2>/dev/null || true + ollama cp nomic-embed-text:latest nomic-ai/nomic-embed-text-v1.5 2>/dev/null || true + echo "✓ Embedding model ready" +else + echo "⚠ Ollama is not installed. Install from https://ollama.com/ for embeddings." +fi + +# 6. Start LocalStack in Docker on localhost:4566 +echo "" +echo "6. Starting LocalStack..." +export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}" +LOCALSTACK_CONTAINER="${LOCALSTACK_CONTAINER:-localstack-dev}" +LOCALSTACK_IMAGE="${LOCALSTACK_IMAGE:-localstack/localstack:4.14.0}" +LOCALSTACK_ENDPOINT="http://127.0.0.1:4566" + +if curl -sf "${LOCALSTACK_ENDPOINT}/_localstack/health" &>/dev/null; then + echo "✓ LocalStack already running at ${LOCALSTACK_ENDPOINT}" +elif docker ps -a --format '{{.Names}}' | grep -qx "${LOCALSTACK_CONTAINER}"; then + echo "Starting existing container '${LOCALSTACK_CONTAINER}'..." + docker start "${LOCALSTACK_CONTAINER}" >/dev/null +else + echo "Creating container '${LOCALSTACK_CONTAINER}' from ${LOCALSTACK_IMAGE}..." + docker run -d \ + --name "${LOCALSTACK_CONTAINER}" \ + -p 127.0.0.1:4566:4566 \ + "${LOCALSTACK_IMAGE}" >/dev/null +fi + +echo "Waiting for LocalStack at ${LOCALSTACK_ENDPOINT}..." +for _ in $(seq 1 30); do + if curl -sf "${LOCALSTACK_ENDPOINT}/_localstack/health" &>/dev/null; then + echo "✓ LocalStack is healthy" + break + fi + sleep 1 +done +if ! curl -sf "${LOCALSTACK_ENDPOINT}/_localstack/health" &>/dev/null; then + echo "Error: LocalStack did not become healthy in time" >&2 + docker logs "${LOCALSTACK_CONTAINER}" --tail=40 >&2 || true + exit 1 +fi + +# 7. Create S3 buckets +echo "" +echo "7. Creating S3 buckets..." +INGESTION_BUCKET="$(yq -r '.spec.sourceCrawlerConfig.s3Config.bucket' "${SOURCE_CRAWLER_YAML}")" +STORAGE_BUCKET="$(yq -r '.spec.dataStorageBucket' "${CONTROLLER_CONFIG_YAML}")" +OUTPUT_BUCKET="$(yq -r '.spec.destinationSyncerConfig.s3DestinationConfig.bucket' "${DEST_SYNCER_YAML}")" + +for bucket in "${INGESTION_BUCKET}" "${STORAGE_BUCKET}" "${OUTPUT_BUCKET}"; do + if awslocal s3 ls "s3://${bucket}" &>/dev/null; then + echo "✓ Bucket '${bucket}' already exists" + else + awslocal s3 mb "s3://${bucket}" + echo "✓ Bucket '${bucket}' created" + fi +done + +# 8. Install CRDs +echo "" +echo "8. Installing CRDs..." +cd "${REPO_ROOT}" && make install +echo "✓ CRDs installed" + +# 9. Apply secrets (operator-secret + pipeline-secret) +echo "" +echo "9. Creating secrets..." +kubectl apply -f "${REPO_ROOT}/config/samples/unstructured-secret.yaml" \ + -n "${LOCAL_NAMESPACE}" +echo "✓ Secrets applied" + +# 10. Apply ControllerConfig +echo "" +echo "10. Creating ControllerConfig..." +kubectl apply -f "${CONTROLLER_CONFIG_YAML}" -n "${LOCAL_NAMESPACE}" +echo "✓ ControllerConfig applied" + +# 11. Apply UnstructuredDataPipeline (creates all stage CRs automatically) +echo "" +echo "11. Creating UnstructuredDataPipeline..." +kubectl apply -f "${PIPELINE_YAML}" -n "${LOCAL_NAMESPACE}" +echo "✓ UnstructuredDataPipeline applied" + +echo "" +echo "✓ Local development environment setup complete" +echo "" +echo "Starting controller (Ctrl+C to stop)..." +cd "${REPO_ROOT}" && make run diff --git a/scripts/localstack.sh b/scripts/localstack.sh new file mode 100755 index 00000000..cb31ecf3 --- /dev/null +++ b/scripts/localstack.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# LocalStack helper for local development. +# Usage: +# ./scripts/localstack.sh start # start container (no-op if healthy) +# ./scripts/localstack.sh stop # stop and remove container +# ./scripts/localstack.sh setup # start + create S3 buckets +# ./scripts/localstack.sh status # show health and resource summary +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CONTROLLER_CONFIG_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_controllerconfig.yaml" +SOURCE_CRAWLER_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_sourcecrawler.yaml" +DEST_SYNCER_YAML="${REPO_ROOT}/config/samples/operator_v1alpha1_destinationsyncer.yaml" + +CONTAINER="${LOCALSTACK_CONTAINER:-localstack-dev}" +IMAGE="${LOCALSTACK_IMAGE:-localstack/localstack:4.14.0}" +ENDPOINT="${LOCALSTACK_ENDPOINT:-http://127.0.0.1:4566}" +AWS_REGION="${AWS_DEFAULT_REGION:-us-east-1}" + +INGESTION_BUCKET="$(yq -r '.spec.sourceCrawlerConfig.s3Config.bucket' "${SOURCE_CRAWLER_YAML}")" +DATA_STORAGE_BUCKET="$(yq -r '.spec.dataStorageBucket' "${CONTROLLER_CONFIG_YAML}")" +OUTPUT_BUCKET="$(yq -r '.spec.destinationSyncerConfig.s3DestinationConfig.bucket' "${DEST_SYNCER_YAML}")" + +usage() { + cat < + +Commands: + start Start LocalStack in Docker (skips if already healthy) + stop Stop and remove the LocalStack container + setup Start LocalStack and create S3 buckets + status Print LocalStack health and configured resources + +Environment overrides: + LOCALSTACK_CONTAINER Docker container name (default: localstack-dev) + LOCALSTACK_IMAGE Docker image (default: localstack/localstack:4.14.0) + LOCALSTACK_ENDPOINT LocalStack URL (default: http://127.0.0.1:4566) +EOF +} + +require_command() { + if ! command -v "$1" &>/dev/null; then + echo "Error: '$1' is required but not installed." >&2 + exit 1 + fi +} + +aws_local() { + AWS_ACCESS_KEY_ID=test \ + AWS_SECRET_ACCESS_KEY=test \ + AWS_DEFAULT_REGION="${AWS_REGION}" \ + aws --endpoint-url="${ENDPOINT}" "$@" +} + +localstack_healthy() { + curl -sf "${ENDPOINT}/_localstack/health" &>/dev/null +} + +wait_for_localstack() { + echo "Waiting for LocalStack at ${ENDPOINT}..." + for _ in $(seq 1 30); do + if localstack_healthy; then + echo "✓ LocalStack is healthy" + return 0 + fi + sleep 1 + done + echo "Error: LocalStack did not become healthy in time" >&2 + docker logs "${CONTAINER}" --tail=40 >&2 || true + return 1 +} + +cmd_start() { + require_command docker + require_command curl + + if localstack_healthy; then + echo "✓ LocalStack already running at ${ENDPOINT}" + return 0 + fi + + if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER}"; then + echo "Starting existing container '${CONTAINER}'..." + docker start "${CONTAINER}" >/dev/null + else + echo "Creating container '${CONTAINER}' from ${IMAGE}..." + docker run -d \ + --name "${CONTAINER}" \ + -p 127.0.0.1:4566:4566 \ + "${IMAGE}" >/dev/null + fi + + wait_for_localstack +} + +cmd_stop() { + require_command docker + + if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER}"; then + docker rm -f "${CONTAINER}" >/dev/null + echo "✓ Stopped and removed '${CONTAINER}'" + else + echo "✓ No '${CONTAINER}' container found" + fi +} + +cmd_setup() { + require_command aws + require_command yq + + cmd_start + + echo "" + echo "Creating S3 buckets..." + for bucket in "${INGESTION_BUCKET}" "${DATA_STORAGE_BUCKET}" "${OUTPUT_BUCKET}"; do + if aws_local s3 ls "s3://${bucket}" &>/dev/null; then + echo "✓ Bucket '${bucket}' already exists" + else + aws_local s3 mb "s3://${bucket}" >/dev/null + echo "✓ Bucket '${bucket}' created" + fi + done + + echo "" + echo "✓ LocalStack setup complete" + echo " Endpoint: ${ENDPOINT}" +} + +cmd_status() { + require_command aws + + if localstack_healthy; then + echo "✓ LocalStack healthy at ${ENDPOINT}" + curl -s "${ENDPOINT}/_localstack/health" | sed 's/^/ /' + else + echo "✗ LocalStack not reachable at ${ENDPOINT}" + return 1 + fi + + echo "" + echo "S3 buckets:" + aws_local s3 ls 2>/dev/null | sed 's/^/ /' || echo " (none)" +} + +main() { + local command="${1:-}" + case "${command}" in + start) cmd_start ;; + stop) cmd_stop ;; + setup) cmd_setup ;; + status) cmd_status ;; + -h|--help|help|"") usage ;; + *) + echo "Error: unknown command '${command}'" >&2 + usage >&2 + exit 1 + ;; + esac +} + +main "$@"