Feels like an intermittent error, but I was surprised by it, so I thought I would report it.
- Using tenant "personal tenant" for cluster "fabric-prod1.defang.dev:443"
- Reading access token from file /Users/jordan/.local/state/defang/fabric-prod1.defang.dev
- fabricClient 9ucmfmnqqobj /io.defang.v1.FabricController/WhoAmI null
- fabricClient 1di31zib8m92 /io.defang.v1.FabricController/GetVersion null
- Fabric: v0.6.0-1564-gd49421f8 CLI: 3e6d4725 CLI-Min: v3.1.0
- fabricClient jyy7vyzxczsp /io.defang.v1.FabricController/CheckToS null
- tracking event "UP INVOKED": [{args []} {err <nil>} {non-interactive false} {provider auto} {CalledAs deploy} {version 3e6d4725} {debug true} {stack jordangcp}]
- fabricClient a8xpb6l1kxx9 /io.defang.v1.FabricController/Track {"anon_id":"fc285f96-a793-492b-b0b7-4107e667ad4d","event":"UP INVOKED","properties":{"CalledAs":" deploy","args":"[]","debug":"true","err":"\u003cnil\u003e","non-interactive":"false","provider":"auto","stack":"jordangcp","version":"3e6d4725"},"os":"darwin","arch":"arm64"}
name: crewai
services:
app:
build:
context: /Users/jordan/wk/samples/samples/crewai/app
dockerfile: Dockerfile
command:
- ./run.sh
depends_on:
postgres:
condition: service_started
required: true
redis:
condition: service_started
required: true
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
environment:
CELERY_BROKER_URL: redis://redis:6379/0
CELERY_RESULT_BACKEND: redis://redis:6379/0
DATABASE_URL: postgres://postgres:password@postgres:5432/postgres?sslmode=require
DJANGO_SECRET_KEY: 5k6qi_gzy65bz2soxl6k(rsrw)0z3ws!zmjzsbe+wkp1ovfayg
DJANGO_SETTINGS_MODULE: config.settings
REDIS_URL: redis://redis:6379/0
SSL_MODE: disable
healthcheck:
test:
- CMD
- curl
- -f
- http://localhost:8000/
timeout: 2s
interval: 5s
retries: 10
start_period: 10s
networks:
default: null
ports:
- mode: ingress
target: 8000
published: "8000"
protocol: tcp
chat:
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
provider:
type: model
options:
model:
- chat-default
x-defang-llm: true
embedding:
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
provider:
type: model
options:
model:
- embedding-default
x-defang-llm: true
postgres:
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
environment:
POSTGRES_PASSWORD: password
image: pgvector/pgvector:pg16
networks:
default: null
ports:
- mode: host
target: 5432
published: "5432"
protocol: tcp
x-defang-postgres: true
redis:
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
image: redis:6.2
networks:
default: null
ports:
- mode: host
target: 6379
published: "6379"
protocol: tcp
x-defang-redis: true
worker:
build:
context: /Users/jordan/wk/samples/samples/crewai/app
dockerfile: Dockerfile
command:
- celery
- -A
- config
- worker
- -l
- info
depends_on:
chat:
condition: service_started
required: true
embedding:
condition: service_started
required: true
postgres:
condition: service_started
required: true
redis:
condition: service_started
required: true
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
environment:
DATABASE_URL: postgres://postgres:password@postgres:5432/postgres?sslmode=require
DJANGO_SECRET_KEY: 5k6qi_gzy65bz2soxl6k(rsrw)0z3ws!zmjzsbe+wkp1ovfayg
DJANGO_SETTINGS_MODULE: config.settings
OPENAI_API_KEY: ""
REDIS_URL: redis://redis:6379/0
SSL_MODE: disable
networks:
default: null
networks:
default:
name: crewai_default
- Creating gcp provider
* Using the "jordangcp" stack on gcp from --stack flag and local stack file
- checking if application default credentials are available and has permission, GOOGLE_APPLICATION_CREDENTIALS=""...
- Enabling services: [cloudresourcemanager.googleapis.com]
- found valid application default credentials with required permissions
name: crewai
services:
app:
build:
context: /Users/jordan/wk/samples/samples/crewai/app
dockerfile: Dockerfile
command:
- ./run.sh
depends_on:
postgres:
condition: service_started
required: true
redis:
condition: service_started
required: true
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
environment:
CELERY_BROKER_URL: redis://redis:6379/0
CELERY_RESULT_BACKEND: redis://redis:6379/0
DATABASE_URL: postgres://postgres:password@postgres:5432/postgres?sslmode=require
DJANGO_SECRET_KEY: 5k6qi_gzy65bz2soxl6k(rsrw)0z3ws!zmjzsbe+wkp1ovfayg
DJANGO_SETTINGS_MODULE: config.settings
REDIS_URL: redis://redis:6379/0
SSL_MODE: disable
healthcheck:
test:
- CMD
- curl
- -f
- http://localhost:8000/
timeout: 2s
interval: 5s
retries: 10
start_period: 10s
networks:
default: null
ports:
- mode: ingress
target: 8000
published: "8000"
protocol: tcp
chat:
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
provider:
type: model
options:
model:
- chat-default
x-defang-llm: true
embedding:
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
provider:
type: model
options:
model:
- embedding-default
x-defang-llm: true
postgres:
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
environment:
POSTGRES_PASSWORD: password
image: pgvector/pgvector:pg16
networks:
default: null
ports:
- mode: host
target: 5432
published: "5432"
protocol: tcp
x-defang-postgres: true
redis:
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
image: redis:6.2
networks:
default: null
ports:
- mode: host
target: 6379
published: "6379"
protocol: tcp
x-defang-redis: true
worker:
build:
context: /Users/jordan/wk/samples/samples/crewai/app
dockerfile: Dockerfile
command:
- celery
- -A
- config
- worker
- -l
- info
depends_on:
chat:
condition: service_started
required: true
embedding:
condition: service_started
required: true
postgres:
condition: service_started
required: true
redis:
condition: service_started
required: true
deploy:
resources:
reservations:
cpus: 0.5
memory: "536870912"
environment:
DATABASE_URL: postgres://postgres:password@postgres:5432/postgres?sslmode=require
DJANGO_SECRET_KEY: 5k6qi_gzy65bz2soxl6k(rsrw)0z3ws!zmjzsbe+wkp1ovfayg
DJANGO_SETTINGS_MODULE: config.settings
OPENAI_API_KEY: ""
REDIS_URL: redis://redis:6379/0
SSL_MODE: disable
networks:
default: null
networks:
default:
name: crewai_default
- Getting services from bucket: defang-cd-i11ht56xksl6 projects/crewai/jordangcp/project.pb defang-upload@jordan-491820.iam.gserviceaccount.com
- fabricClient soncxc7344dv /io.defang.v1.FabricController/CanIUse {"project":"crewai","provider":4,"service_count":6,"stack":"jordangcp","region":"us-central1","provider_account_id":"jordan-491820","prefer_cd_version":"index.docker.io/defangio/cd:public-gcp-e341544d-x86_64","prefer_pulumi_version":"3.192.0","driver":"cloudbuild"}
- Using previous CD image: index.docker.io/defangio/cd:public-gcp-e341544d-x86_64
! A newer CD image is available; using previously deployed version. To upgrade, re-run with --allow-upgrade or set DEFANG_ALLOW_UPGRADE=1
- Using Pulumi version: 3.192.0
- fabricClient we4ghf79tyy1 /io.defang.v1.FabricController/ListDeployments {"project":"crewai","type":2,"stack":"jordangcp"}
! Defang cannot monitor status of the following managed service(s): [postgres redis].
To check if the managed service is up, check the status of the service which depends on it.
- Validating Dockerfile: /Users/jordan/wk/samples/samples/crewai/app/Dockerfile for service "app"
- Validating Dockerfile: /Users/jordan/wk/samples/samples/crewai/app/Dockerfile for service "worker"
* Service environment variables resolution summary:
SERVICE ENVIRONMENT SOURCE VALUE
app CELERY_BROKER_URL Compose redis://redis:6379/0
app CELERY_RESULT_BACKEND Compose redis://redis:6379/0
app DATABASE_URL Compose post**
app DJANGO_SECRET_KEY Config ******
app DJANGO_SETTINGS_MODULE Compose config.settings
app REDIS_URL Compose redis://redis:6379/0
app SSL_MODE Config ******
postgres POSTGRES_PASSWORD Config ******
worker DATABASE_URL Compose post**
worker DJANGO_SECRET_KEY Config ******
worker DJANGO_SETTINGS_MODULE Compose config.settings
worker OPENAI_API_KEY Compose
worker REDIS_URL Compose redis://redis:6379/0
worker SSL_MODE Config ******
- port 8000: ignoring 'published: 8000' in 'ingress' mode
* Packaging the project files for app at /Users/jordan/wk/samples/samples/crewai/app
- Reading .dockerignore file from .dockerignore
- Adding .dockerignore
- Adding Dockerfile
- Adding config
- Adding config/__init__.py
- Adding config/asgi.py
- Adding config/celery.py
- Adding config/settings.py
- Adding config/urls.py
- Adding config/wsgi.py
- Adding core
- Adding core/__init__.py
- Adding core/admin.py
- Adding core/ai_clients.py
- Adding core/apps.py
- Adding core/consumers.py
- Adding core/custom_llm.py
- Adding core/management
- Adding core/management/__init__.py
- Adding core/management/commands
- Adding core/management/commands/__init__.py
- Adding core/management/commands/create_initial_superuser.py
- Adding core/migrations
- Adding core/migrations/0001_enable_pgvector.py
- Adding core/migrations/0002_initial.py
- Adding core/migrations/0003_rename_vector_summary_embedding.py
- Adding core/migrations/__init__.py
- Adding core/models.py
- Adding core/routing.py
- Adding core/tasks.py
- Adding core/templates
- Adding core/templates/core
- Adding core/templates/core/home.html
- Adding core/tests.py
- Adding core/views.py
- Adding dev.sh
- Adding manage.py
- Adding requirements.txt
- Adding run.sh
- Digest for "app": sha256-zUipS1bsplCV6bPmRlJYxhtOjwSwyUK1PESonXnlUtk=
* Uploading the project files for app
* Setting up defang CD in GCP project jordan-491820, this could take a few minutes
- Enabling services: [storage.googleapis.com artifactregistry.googleapis.com run.googleapis.com iam.googleapis.com cloudresourcemanager.googleapis.com cloudbuild.googleapis.com compute.googleapis.com dns.googleapis.com secretmanager.googleapis.com sqladmin.googleapis.com servicenetworking.googleapis.com redis.googleapis.com certificatemanager.googleapis.com cloudscheduler.googleapis.com]
- Bucket "defang-cd-i11ht56xksl6" already exists
- Service account defang-cd already exists
- serviceAccount:defang-cd@jordan-491820.iam.gserviceaccount.com already has roles [roles/run.admin roles/iam.serviceAccountAdmin roles/iam.serviceAccountUser roles/artifactregistry.admin roles/compute.futureReservationViewer roles/compute.loadBalancerAdmin roles/compute.networkAdmin roles/compute.securityAdmin roles/dns.admin roles/cloudbuild.builds.editor roles/secretmanager.admin roles/resourcemanager.projectIamAdmin roles/compute.instanceAdmin.v1 roles/cloudsql.admin roles/redis.admin roles/certificatemanager.owner roles/serviceusage.serviceUsageAdmin roles/datastore.owner roles/logging.logWriter roles/cloudscheduler.admin] on resource projects/jordan-491820
- Principal serviceAccount:defang-cd@jordan-491820.iam.gserviceaccount.com already has roles [roles/storage.admin] on bucket defang-cd-i11ht56xksl6
- Service account defang-upload already exists
- Principal serviceAccount:defang-upload@jordan-491820.iam.gserviceaccount.com already has roles [roles/storage.objectUser] on bucket defang-cd-i11ht56xksl6
- Using CD image: "index.docker.io/defangio/cd:public-gcp-e341544d-x86_64"
- [DEBUG] PUT https://storage.googleapis.com/defang-cd-i11ht56xksl6/uploads/sha256-zUipS1bsplCV6bPmRlJYxhtOjwSwyUK1PESonXnlUtk%3D.tar.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=defang-upload%40jordan-491820.iam.gserviceaccount.com%2F20260406%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20260406T225650Z&X-Goog-Expires=899&X-Goog-Signature=699b1d79292cbe974e77f5c331b04d514e50729a4e7f65f08e2f23e31ac6587e5807ab097a9facd94025ab21fa6ef2ba54051329521d035144b94700fa687902edc90d1e69eae306e261e062dfe653a5b67f4fa9f78a8705ff31109a374de5ceae9dd1d0047a4abae597a8b965947d2f9e4395359503affc35470520aa8bc7bb3887af6e6fe6437123d476ee8eca616a1ec03c4176e1eb1562a7bd8c4e6972622d12c943c0dcfa8bc186b5df5a7312295938ea5df0c7113400c4824a27c47278ebe20d95f3469fea493281b13324029901c9756dc4b59c8fee37b61cebab2bdf5b51d32faa0da8124395ddb88499239d99b3cfba63ce156a1b6180c5e468a93b&X-Goog-SignedHeaders=host
! service "app": environment variable(s) ["DJANGO_SECRET_KEY" "SSL_MODE"] overridden by config
* Packaging the project files for worker at /Users/jordan/wk/samples/samples/crewai/app
- Reading .dockerignore file from .dockerignore
- Adding .dockerignore
- Adding Dockerfile
- Adding config
- Adding config/__init__.py
- Adding config/asgi.py
- Adding config/celery.py
- Adding config/settings.py
- Adding config/urls.py
- Adding config/wsgi.py
- Adding core
- Adding core/__init__.py
- Adding core/admin.py
- Adding core/ai_clients.py
- Adding core/apps.py
- Adding core/consumers.py
- Adding core/custom_llm.py
- Adding core/management
- Adding core/management/__init__.py
- Adding core/management/commands
- Adding core/management/commands/__init__.py
- Adding core/management/commands/create_initial_superuser.py
- Adding core/migrations
- Adding core/migrations/0001_enable_pgvector.py
- Adding core/migrations/0002_initial.py
- Adding core/migrations/0003_rename_vector_summary_embedding.py
- Adding core/migrations/__init__.py
- Adding core/models.py
- Adding core/routing.py
- Adding core/tasks.py
- Adding core/templates
- Adding core/templates/core
- Adding core/templates/core/home.html
- Adding core/tests.py
- Adding core/views.py
- Adding dev.sh
- Adding manage.py
- Adding requirements.txt
- Adding run.sh
- Digest for "worker": sha256-zUipS1bsplCV6bPmRlJYxhtOjwSwyUK1PESonXnlUtk=
* Uploading the project files for worker
- [DEBUG] PUT https://storage.googleapis.com/defang-cd-i11ht56xksl6/uploads/sha256-zUipS1bsplCV6bPmRlJYxhtOjwSwyUK1PESonXnlUtk%3D.tar.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=defang-upload%40jordan-491820.iam.gserviceaccount.com%2F20260406%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20260406T225651Z&X-Goog-Expires=899&X-Goog-Signature=1541897eda818986ddbb60b80463f1da6763f1a71e56deaa575cbd20dd22d750f618f9647750365dde12165fa87ca20a19d1b0b7cfef8764034b12186c0bf42a5ab9487d4f43eceb5b1a5221fa41179f577ce71453a839c33b9d7c481380517a5c7789112ca8feb8d08db09ef143bef8fcfb05c123491fabf38f997df724364d0e9d12eaa57a328df57b1eb4c5e3565497dd5849d4cece265570ad2e7446d7bedee76f6760f02002aad2834b1890e76b7e7e51fa872a4de07280a22d7c360fee6f40a1cdd93d248ff3298b98d699c81edfe19a6f1f8bb7151a049ed0605ab012523f77061cd130e9006a02053b136993bef9a7e65add894b6a73d447721b7fbe&X-Goog-SignedHeaders=host
! service "worker": environment variable(s) ["DJANGO_SECRET_KEY" "SSL_MODE"] overridden by config
! service "postgres": environment variable(s) ["POSTGRES_PASSWORD"] overridden by config
- service "app": missing compose directive: restart; assuming 'unless-stopped' (add 'restart' to silence)
! service "app": environment "DATABASE_URL" may contain sensitive information; consider using 'defang config set DATABASE_URL' to securely store this value
- service "app": environment "DATABASE_URL" may contain detected secrets of type: [URL with password]
- service "app": unsupported compose directive: healthcheck start_period
- service "chat": missing compose directive: restart; assuming 'unless-stopped' (add 'restart' to silence)
- service "embedding": missing compose directive: restart; assuming 'unless-stopped' (add 'restart' to silence)
- service "postgres": missing compose directive: restart; assuming 'unless-stopped' (add 'restart' to silence)
- service "redis": missing compose directive: restart; assuming 'unless-stopped' (add 'restart' to silence)
- service "worker": missing compose directive: restart; assuming 'unless-stopped' (add 'restart' to silence)
! service "worker": environment "DATABASE_URL" may contain sensitive information; consider using 'defang config set DATABASE_URL' to securely store this value
- service "worker": environment "DATABASE_URL" may contain detected secrets of type: [URL with password]
- fabricClient 5ef31bpdteby /io.defang.v1.FabricController/GetDelegateSubdomainZone {"project":"crewai","stack":"jordangcp"}
- Getting services from bucket: defang-cd-i11ht56xksl6 projects/crewai/jordangcp/project.pb defang-upload@jordan-491820.iam.gserviceaccount.com
- Preparing domain delegation for crewai-jordangcp.jordanstephens.defang.app
- Zone defang-crewai-jordangcp-jordanstephens-defang-app created with nameservers [ns-cloud-a1.googledomains.com. ns-cloud-a2.googledomains.com. ns-cloud-a3.googledomains.com. ns-cloud-a4.googledomains.com.]
- fabricClient 29a00gxw9r34 /io.defang.v1.FabricController/DelegateSubdomainZone {"name_server_records":["ns-cloud-a1.googledomains.com.","ns-cloud-a2.googledomains.com.","ns-cloud-a3.googledomains.com.","ns-cloud-a4.googledomains.com."],"project":"crewai","stack":"jordangcp"}
- fabricClient ho5286uftory /io.defang.v1.FabricController/CreateUploadURL {"project":"crewai","stack":"jordangcp","filename":"states-jg1p50o3xlyk.json"}
- fabricClient i5bygfg1juc0 /io.defang.v1.FabricController/CreateUploadURL {"project":"crewai","stack":"jordangcp","filename":"events-jg1p50o3xlyk.json"}
- [DEBUG] PUT https://storage.googleapis.com/defang-cd-i11ht56xksl6/uploads/a533ofz4mjtk?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=defang-upload%40jordan-491820.iam.gserviceaccount.com%2F20260406%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20260406T225653Z&X-Goog-Expires=899&X-Goog-Signature=4dc547d5019878524b4722ee59455409fe583bb88eb736d817fec6ce2dbfc019e4d0278379282c9634a702ec0fcbb418f7da84a361c05eb90e0be72112b2c748c16baf4488ac21de570c449f6d528bf0d6d4d177f9795ed5ef02fa17dfa77c670c090202f8188e854859c92b8f9d0e1b56f77c5f03b2d8a5c6f9d0326e2dd74ccca412dfa5fccdbd62b5e7a091a4076b954770e3649196a4642d04c5b61139879bcb2199601f89ebe5f356ee6eff3c006071d9061bf0d0e51d78e28750ee4cc449ea8e49f90ce1ff42557d813f05fff0ac442e497fb8f22b7555aba682ae9693d5cb9486ef316287eefce4407058b0a7cdef9eb01ceae3b3e6ca9a87903dabcb&X-Goog-SignedHeaders=host
- Starting CD in cloudbuild at: 2026-04-06T15:56:53-07:00
- fabricClient lz78l8305250 /io.defang.v1.FabricController/PutStack {"stack":{"name":"jordangcp","project":"crewai","stack_file":"REVGQU5HX1BST1ZJREVSPSJnY3AiCkdDUF9QUk9KRUNUX0lEPSJqb3JkYW4tNDkxODIwIgpHT09HTEVfUkVHSU9OPSJ1cy1jZW50cmFsMSI=","provider":4,"last_deployed_at":{"seconds":1775516214,"nanos":606505000},"provider_account_id":"jordan-491820","region":"us-central1"}}
- Failed to record deployment: already_exists: stack "jordangcp" already exists
! Unable to update deployment history; deployment will proceed anyway.
Project: crewai
postgres:
allowScaling: true
endpoints:
- postgres.google.internal:5432
etag: a533ofz4mjtk
healthcheckPath: /
privateFqdn: postgres.google.internal
project: crewai
service:
name: postgres
state: UPDATE_QUEUED
status: UPDATE_QUEUED
redis:
allowScaling: true
endpoints:
- redis.google.internal:6379
etag: a533ofz4mjtk
healthcheckPath: /
privateFqdn: redis.google.internal
project: crewai
service:
name: redis
state: UPDATE_QUEUED
status: UPDATE_QUEUED
chat:
allowScaling: true
endpoints:
- chat.google.internal:80
etag: a533ofz4mjtk
healthcheckPath: /
privateFqdn: chat.google.internal
project: crewai
service:
name: chat
state: UPDATE_QUEUED
status: UPDATE_QUEUED
embedding:
allowScaling: true
endpoints:
- embedding.google.internal:80
etag: a533ofz4mjtk
healthcheckPath: /
privateFqdn: embedding.google.internal
project: crewai
service:
name: embedding
state: UPDATE_QUEUED
status: UPDATE_QUEUED
worker:
allowScaling: true
etag: a533ofz4mjtk
healthcheckPath: /
project: crewai
service:
name: worker
state: BUILD_QUEUED
status: BUILD_QUEUED
app:
allowScaling: true
endpoints:
- app--8000.crewai-jordangcp.jordanstephens.defang.app
etag: a533ofz4mjtk
healthcheckPath: /
project: crewai
publicFqdn: app.crewai-jordangcp.jordanstephens.defang.app
service:
name: app
state: BUILD_QUEUED
status: BUILD_QUEUED
* Tailing logs for deployment ID a533ofz4mjtk ; press Ctrl+C to detach:
- Tailing ALL logs in project "crewai"
- Tail request: since:{seconds:1775516197 nanos:881009000} etag:"a533ofz4mjtk" project:"crewai" log_type:4294967295 follow:true
- waiting for services [app worker chat embedding] to reach state DEPLOYMENT_COMPLETED
- Query and tail logs since 2026-04-06 22:56:37.881009 +0000 UTC with query:
(
logName=~"logs/run.googleapis.com%2F(stdout|stderr)$" OR
logName="projects/jordan-491820/logs/cloudbuild" OR
logName="projects/jordan-491820/logs/cos_containers" OR
logName="projects/jordan-491820/logs/docker-logs"
) AND (timestamp >= "2026-04-06T22:56:37.881009Z") AND (
(
resource.type="build"
labels.build_tags =~ "jordangcp_crewai_[a-zA-Z0-9-]{1,63}_a533ofz4mjtk"
-labels.build_step="MAIN"
) OR (
resource.type="cloud_run_revision"
labels."defang-stack" = "jordangcp"
labels."defang-project" = "crewai"
labels."defang-etag" = "a533ofz4mjtk"
) OR (
resource.type="gce_instance"
labels."defang-stack" = "jordangcp"
labels."defang-project" = "crewai"
labels."defang-etag" = "a533ofz4mjtk"
)
)
* Showing only build logs and runtime errors. Press V to toggle verbose mode.
- Query and tail logs since 2026-04-06 15:56:54.666212 -0700 PDT m=+17.053029084 with query:
(
protoPayload.serviceName="run.googleapis.com" OR
protoPayload.serviceName="compute.googleapis.com" OR
protoPayload.serviceName="cloudbuild.googleapis.com"
) AND (timestamp >= "2026-04-06T22:56:54.666212Z") AND (
(
protoPayload.methodName="google.cloud.run.v2.Jobs.UpdateJob" OR "google.cloud.run.v2.Jobs.CreateJob"
protoPayload.request.job.template.labels."defang-stack"="jordangcp"
protoPayload.request.job.template.labels."defang-project"="crewai"
protoPayload.request.job.template.labels."defang-etag"="a533ofz4mjtk"
protoPayload.request.job.template.labels."defang-service"=~"^(app|worker|chat|embedding)$"
) OR (
protoPayload.methodName="/Jobs.RunJob" OR "/Jobs.CreateJob" OR "/Jobs.UpdateJob"
protoPayload.response.spec.template.metadata.labels."defang-stack"="jordangcp"
protoPayload.response.spec.template.metadata.labels."defang-project"="crewai"
protoPayload.response.spec.template.metadata.labels."defang-etag"="a533ofz4mjtk"
protoPayload.response.spec.template.metadata.labels."defang-service"=~"^(app|worker|chat|embedding)$"
) OR (
protoPayload.methodName="google.cloud.run.v2.Services.CreateService" OR "google.cloud.run.v2.Services.UpdateService" OR "google.cloud.run.v2.Services.ReplaceService" OR "google.cloud.run.v2.Services.DeleteService"
protoPayload.request.service.template.labels."defang-stack"="jordangcp"
protoPayload.request.service.template.labels."defang-project"="crewai"
protoPayload.request.service.template.labels."defang-etag"="a533ofz4mjtk"
protoPayload.request.service.template.labels."defang-service"=~"^(app|worker|chat|embedding)$"
) OR (
protoPayload.methodName="/Services.CreateService" OR "/Services.UpdateService" OR "/Services.ReplaceService" OR "/Services.DeleteService"
protoPayload.response.spec.template.metadata.labels."defang-stack"="jordangcp"
protoPayload.response.spec.template.metadata.labels."defang-project"="crewai"
protoPayload.response.spec.template.metadata.labels."defang-etag"="a533ofz4mjtk"
protoPayload.response.spec.template.metadata.labels."defang-service"=~"^(app|worker|chat|embedding)$"
) OR (
protoPayload.methodName=~"beta.compute.regionInstanceGroupManagers.(insert|patch)" AND operation.first="true"
protoPayload.request.allInstancesConfig.properties.labels.key="defang-stack"
protoPayload.request.allInstancesConfig.properties.labels.value="jordangcp"
protoPayload.request.allInstancesConfig.properties.labels.key="defang-project"
protoPayload.request.allInstancesConfig.properties.labels.value="crewai"
protoPayload.request.allInstancesConfig.properties.labels.key="defang-etag"
protoPayload.request.allInstancesConfig.properties.labels.value="a533ofz4mjtk"
protoPayload.request.allInstancesConfig.properties.labels.key="defang-service"
protoPayload.request.allInstancesConfig.properties.labels.value=~"^(app|worker|chat|embedding)$"
) OR (
protoPayload.methodName="v1.compute.instanceGroups.addInstances"
) OR (
resource.type="build"
logName=~"logs/cloudaudit.googleapis.com%2Factivity$"
)
)
- Closing log lister client
- Closing log lister client
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:57:17.581-07:00 cd ** Update started for stack jordangcp
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:57:23.907-07:00 cd ** Updating service "postgres"
2026-04-06T15:57:24.061-07:00 cd ** Updating service "redis"
2026-04-06T15:57:24.062-07:00 cd ** Updating service "chat"
2026-04-06T15:57:25.177-07:00 cd ** Building image for "app"...
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:57:24.261-07:00 cd ** Updating service "embedding"
2026-04-06T15:57:24.430-07:00 cd ** Building image for "worker"...
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
* Waiting for ["app" "worker" "chat" "embedding"] to be in state DEPLOYMENT_COMPLETED...
- service app with state ( DEPLOYMENT_PENDING ) and status:
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:58:21.665-07:00 app 2026-04-06 22:58:21,665 INFO HTTP Request: GET https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json "HTTP/1.1 200 OK"
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:58:29.812-07:00 app 2026-04-06 22:58:29,813 INFO Starting server at tcp:port=8000:interface=0.0.0.0
2026-04-06T15:58:29.812-07:00 app 2026-04-06 22:58:29,814 INFO HTTP/2 support enabled
2026-04-06T15:58:29.813-07:00 app 2026-04-06 22:58:29,814 INFO Configuring endpoint tcp:port=8000:interface=0.0.0.0
2026-04-06T15:58:29.814-07:00 app 2026-04-06 22:58:29,815 INFO Listening on TCP address 0.0.0.0:8000
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:58:30.601-07:00 app 169.254.169.126:35335 - - [06/Apr/2026:22:58:30] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
* Waiting for ["app" "worker" "chat" "embedding"] to be in state DEPLOYMENT_COMPLETED...
- service app with state ( DEPLOYMENT_PENDING ) and status: Ready condition status changed to True for Service defang-crewai-jordangcp-app-6cc666a.
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- ignored root trigger id 4abd0907-71cd-462d-a972-5070ea1464b7 for instance group insert
- Polled CD task status: done=false, err=<nil>
- ignored root trigger id 342faca6-cf04-4e6e-b654-e058032a73e8 for instance group insert
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:58:40.609-07:00 app 169.254.169.126:32293 - - [06/Apr/2026:22:58:40] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:58:45.616-07:00 app 169.254.169.126:38677 - - [06/Apr/2026:22:58:45] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:58:50.623-07:00 app 169.254.169.126:4371 - - [06/Apr/2026:22:58:50] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:58:55.630-07:00 app 169.254.169.126:10695 - - [06/Apr/2026:22:58:55] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:59:00.637-07:00 app 169.254.169.126:51925 - - [06/Apr/2026:22:59:00] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:59:05.645-07:00 app 169.254.169.126:29457 - - [06/Apr/2026:22:59:05] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:59:10.652-07:00 app 169.254.169.126:16935 - - [06/Apr/2026:22:59:10] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:59:15.659-07:00 app 169.254.169.126:26265 - - [06/Apr/2026:22:59:15] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:59:20.666-07:00 app 169.254.169.126:47751 - - [06/Apr/2026:22:59:20] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:59:25.673-07:00 app 169.254.169.126:7955 - - [06/Apr/2026:22:59:25] "GET /" 200 8563
- Polled CD task status: done=false, err=<nil>
2026-04-06T15:59:30.679-07:00 app 169.254.169.126:11859 - - [06/Apr/2026:22:59:30] "GET /" 200 8563
2026-04-06T15:59:35.686-07:00 app 169.254.169.126:6193 - - [06/Apr/2026:22:59:35] "GET /" 200 8563
- Polled CD task status: done=false, err=failed to poll build operation: rpc error: code = Unauthenticated desc = transport: per-RPC creds failed due to error: Post "https://oauth2.googleapis.com/token": context deadline exceeded
* Deployment complete. Waiting for services to be healthy...
- Closing log tailer
- Closing log tailer client
2026-04-06T15:59:40.693-07:00 app 169.254.169.126:48163 - - [06/Apr/2026:22:59:40] "GET /" 200 8563
- Closing log tailer
- Closing log tailer client
- Tail while monitoring stopped with monitoring done <nil>
- Reading access token from file /Users/jordan/.local/state/defang/fabric-prod1.defang.dev
- RegisterAction {key=/model/fabric/google/gemini-2.5-flash}
- RegisterPlugin {name=fabric}
- RegisterValue {name=/format/json}
- RegisterValue {name=/format/jsonl}
- RegisterValue {name=/format/text}
- RegisterValue {name=/format/array}
- RegisterValue {name=/format/enum}
- RegisterAction {key=/util/generate}
- Default prompt directory not found, skipping loading .prompt files {dir=./prompts}
- RegisterValue {name=genkit/defaultModel}
- RegisterValue {name=genkit/promptDir}
- RegisterAction {key=/tool/services}
- RegisterAction {key=/tool/deploy}
- RegisterAction {key=/tool/destroy}
- RegisterAction {key=/tool/logs}
- RegisterAction {key=/tool/estimate}
- RegisterAction {key=/tool/set_config}
- RegisterAction {key=/tool/select_stack}
- RegisterAction {key=/tool/create_aws_stack}
- RegisterAction {key=/tool/create_gcp_stack}
- RegisterAction {key=/tool/list_stacks}
- RegisterAction {key=/tool/current_stack}
- RegisterAction {key=/tool/remove_config}
- RegisterAction {key=/tool/list_configs}
- RegisterAction {key=/tool/read_file}
- RegisterAction {key=/tool/find_files}
- RegisterAction {key=/tool/list_files}
Error: failed to poll build operation: rpc error: code = Unauthenticated desc = transport: per-RPC creds failed due to error: Post "https://oauth2.googleapis.com/token": context deadline exceeded
failed to poll build operation: rpc error: code = Unauthenticated desc = transport: per-RPC creds failed due to error: Post "https://oauth2.googleapis.com/token": context deadline exceeded
- tracking event "CLI Error": [{err failed to poll build operation: rpc error: code = Unauthenticated desc = transport: per-RPC creds failed due to error: Post "https://oauth2.googleapis.com/token": context deadline exceeded
failed to poll build operation: rpc error: code = Unauthenticated desc = transport: per-RPC creds failed due to error: Post "https://oauth2.googleapis.com/token": context deadline exceeded}]
- fabricClient sjaphojfc83a /io.defang.v1.FabricController/Track {"anon_id":"fc285f96-a793-492b-b0b7-4107e667ad4d","event":"CLI Error","properties":{"err":"failed to poll build operation: rpc error: code = Unauthenticated desc = transport: per-RPC creds failed due to error: Post \"https://oauth2.googleapis.com/token\": context deadline exceeded\nfailed to poll build operation: rpc error: code = Unauthenticated desc = transport: per-RPC creds failed due to error: Post \"https://oauth2.googleapis.com/token\": context deadline exceeded"},"os":"darwin","arch":"arm64"}
Feels like an intermittent error, but I was surprised by it, so I thought I would report it.