auto-ingest/docker-compose.yml at main · scottjoyner/auto-ingest · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
# Auto-Ingest Docker Compose
# Services: ingest, worker, sync, content, cron jobs
# All paths mounted from host via NAS_ROOT (.env)

services:
  # === Ingest Service ===
  # Runs run_ingest_all.sh continuously. Has flock-based concurrency guard.
  ingest-service:
    build: .
    image: auto-ingest:latest
    container_name: auto-ingest-service
    command: ["bash", "-c", "while true; do /app/run_ingest_all.sh; sleep 300; done"]
    restart: unless-stopped
    extra_hosts:
      - "host.docker.internal:host-gateway"
    environment:
      SCAN_ROOTS: ${SCAN_ROOTS}
      DASHCAM_ROOT: ${DASHCAM_ROOT}
      NEO4J_URI: ${NEO4J_URI}
      NEO4J_USER: ${NEO4J_USER}
      NEO4J_PASSWORD: ${NEO4J_PASSWORD}
      NEO4J_DB: ${NEO4J_DB}
      LOCAL_TZ: America/New_York
      EMBED_MODEL_NAME: sentence-transformers/all-MiniLM-L6-v2
      MODEL_PREF: large-v3,faster-whisper:large-v3
      LOG_LEVEL: INFO
    volumes:
      - ./:/app
      - ${NAS_ROOT}:/nas
    networks:
      - auto-ingest-net

  # === Ingest Worker (distributed) ===
  # Claims .job files from NAS drop queue and executes them
  ingest-worker:
    build: .
    image: auto-ingest:latest
    container_name: auto-ingest-worker
    command: ["bash", "-c", "while true; do /app/deploy/worker_ingest.sh; sleep 30; done"]
    restart: unless-stopped
    extra_hosts:
      - "host.docker.internal:host-gateway"
    environment:
      DROP_ROOT: ${DROP_ROOT}
      AUDIO_ROOT: ${AUDIO_ROOT}
      DASHCAM_ROOT: ${DASHCAM_ROOT}
      BODYCAM_ROOT: ${BODYCAM_ROOT}
      TRANSCRIPT_ROOT: ${TRANSCRIPT_ROOT}
      NEO4J_URI: ${NEO4J_URI}
      NEO4J_USER: ${NEO4J_USER}
      NEO4J_PASSWORD: ${NEO4J_PASSWORD}
      NEO4J_DB: ${NEO4J_DB}
    volumes:
      - ./:/app
      - ${NAS_ROOT}:/nas
    networks:
      - auto-ingest-net

  # === Sync Service ===
  # Continuously syncs legacy drop from deathstar into canonical roots
  sync-service:
    build: .
    image: auto-ingest:latest
    container_name: auto-sync-service
    command: ["bash", "-c", "while true; do /app/deploy/sync_from_legacy_drop.sh >> /var/log/sync.log 2>&1; sleep 600; done"]
    restart: unless-stopped
    extra_hosts:
      - "host.docker.internal:host-gateway"
    environment:
      LEGACY_DROP_ROOT: ${LEGACY_DROP_ROOT}
      AUDIO_ROOT: ${AUDIO_ROOT}
      DASHCAM_ROOT: ${DASHCAM_ROOT}
      BODYCAM_ROOT: ${BODYCAM_ROOT}
      TRANSCRIPT_ROOT: ${TRANSCRIPT_ROOT}
      YOLO_ROOT: ${YOLO_ROOT}
      META_ROOT: ${META_ROOT}
    volumes:
      - ./:/app
      - ${NAS_ROOT}:/nas
    networks:
      - auto-ingest-net

  # === Content OS Service ===
  # Runs content-os status loop for content workflow monitoring
  content-service:
    build: .
    image: auto-ingest:latest
    container_name: content-generation-service
    command: ["bash", "-c", "while true; do /usr/local/bin/content-os status; sleep 1800; done"]
    restart: unless-stopped
    environment:
      CONTENT_OS_LLM_BASE_URL: ${CONTENT_OS_LLM_BASE_URL}
      CONTENT_OS_LLM_API_KEY: ${CONTENT_OS_LLM_API_KEY}
      CONTENT_OS_LLM_MODEL: ${CONTENT_OS_LLM_MODEL}
      CONTENT_OS_LLM_TEMPERATURE: ${CONTENT_OS_LLM_TEMPERATURE}
      CONTENT_OS_LLM_MAX_TOKENS: ${CONTENT_OS_LLM_MAX_TOKENS}
    volumes:
      - ./:/app
    networks:
      - auto-ingest-net

  # === Ingest Cron ===
  # Runs sync + ingest on a schedule (every 5 min)
  ingest-cron:
    build: .
    image: auto-ingest:latest
    entrypoint: ["/app/deploy/start-cron.sh"]
    environment:
      CRON_FILE: /app/deploy/cron/ingest.crontab
      SCAN_ROOTS: ${SCAN_ROOTS}
      DASHCAM_ROOT: ${DASHCAM_ROOT}
      NEO4J_URI: ${NEO4J_URI}
      NEO4J_USER: ${NEO4J_USER}
      NEO4J_PASSWORD: ${NEO4J_PASSWORD}
      NEO4J_DB: ${NEO4J_DB}
    volumes:
      - ./:/app
      - ${NAS_ROOT}:/nas
    networks:
      - auto-ingest-net

  # === Content Cron ===
  # Runs content workflow batch every 30 min
  content-cron:
    build: .
    image: auto-ingest:latest
    entrypoint: ["/app/deploy/start-cron.sh"]
    environment:
      CRON_FILE: /app/deploy/cron/content_generation.crontab
      CONTENT_OS_LLM_BASE_URL: ${CONTENT_OS_LLM_BASE_URL}
      CONTENT_OS_LLM_API_KEY: ${CONTENT_OS_LLM_API_KEY}
      CONTENT_OS_LLM_MODEL: ${CONTENT_OS_LLM_MODEL}
    volumes:
      - ./:/app
    networks:
      - auto-ingest-net

  # === Job API (lightweight trigger) ===
  # Exposes HTTP endpoints to enqueue jobs and trigger runs
  job-api:
    build: .
    image: auto-ingest:latest
    container_name: auto-ingest-job-api
    command: ["python", "/app/deploy/job_trigger_api.py"]
    restart: unless-stopped
    extra_hosts:
      - "host.docker.internal:host-gateway"
    ports:
      - "8766:8765"
    volumes:
      - ./:/app
      - ${NAS_ROOT}:/nas
    environment:
      DROP_ROOT: ${DROP_ROOT}
      NAS_ROOT: ${NAS_ROOT}
      SCAN_ROOTS: ${SCAN_ROOTS}
      AUDIO_ROOT: ${AUDIO_ROOT}
      DASHCAM_ROOT: ${DASHCAM_ROOT}
      BODYCAM_ROOT: ${BODYCAM_ROOT}
      JOB_API_PORT: "8765"
    networks:
      - auto-ingest-net

networks:
  auto-ingest-net:
    driver: bridge