Skip to content

Commit

Permalink
[AIRFLOW-XXX] Dump logs in case of kube failure (apache#5472)
Browse files Browse the repository at this point in the history
Previously we were only dumping the logs in case of _success_, which was
somewhat pointless
  • Loading branch information
ashb authored Jun 26, 2019
1 parent 82b7b8f commit 8481bb7
Showing 1 changed file with 22 additions and 15 deletions.
37 changes: 22 additions & 15 deletions scripts/ci/kubernetes/kube/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,24 @@ kubectl apply -f $DIRNAME/postgres.yaml
kubectl apply -f $DIRNAME/volumes.yaml
kubectl apply -f $BUILD_DIRNAME/airflow.yaml

dump_logs() {
echo "------- pod description -------"
kubectl describe pod $POD
echo "------- webserver init container logs - init -------"
kubectl logs $POD -c init || true
if [ "${GIT_SYNC}" = 1 ]; then
echo "------- webserver init container logs - git-sync-clone -------"
kubectl logs $POD -c git-sync-clone || true
fi
echo "------- webserver logs -------"
kubectl logs $POD -c webserver || true
echo "------- scheduler logs -------"
kubectl logs $POD -c scheduler || true
echo "--------------"
}


set +x
# wait for up to 10 minutes for everything to be deployed
PODS_ARE_READY=0
for i in {1..150}
Expand All @@ -168,11 +186,13 @@ do
fi
sleep 4
done
POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | grep airflow | head -1)

if [[ "$PODS_ARE_READY" == 1 ]]; then
echo "PODS are ready."
else
echo "PODS are not ready after waiting for a long time. Exiting..."
dump_logs
exit 1
fi

Expand All @@ -199,21 +219,8 @@ if [[ "$AIRFLOW_WEBSERVER_IS_READY" == 1 ]]; then
echo "Airflow webserver is ready."
else
echo "Airflow webserver is not ready after waiting for a long time. Exiting..."
dump_logs
exit 1
fi

POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | grep airflow | head -1)

echo "------- pod description -------"
kubectl describe pod $POD
echo "------- webserver init container logs - init -------"
kubectl logs $POD init
if [ "${GIT_SYNC}" = 1 ]; then
echo "------- webserver init container logs - git-sync-clone -------"
kubectl logs $POD git-sync-clone
fi
echo "------- webserver logs -------"
kubectl logs $POD webserver
echo "------- scheduler logs -------"
kubectl logs $POD scheduler
echo "--------------"
dump_logs

0 comments on commit 8481bb7

Please sign in to comment.