add tests for collections search links (#310)

* add tests for collections search links

* fix issues with docker and scripts. fix tests. fix issue with collection_search links

* set break-system-packages for ci

* remove slashes from branch name

---------

Co-authored-by: David W Bitner <[email protected]>

Author: vincentsarago

.github/workflows/continuous-integration.yml

@@ -10,6 +10,7 @@ env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
   DOCKER_BUILDKIT: 1
+  PIP_BREAK_SYSTEM_PACKAGES: 1
 
 jobs:
   changes:
@@ -35,12 +36,11 @@ jobs:
     - id: check
       run: |
         buildpg=false;
-        ref=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}};
+        ref=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | tr / _);
         [[ "${{ steps.filter.outputs.pgstac }}" == "true" ]] && buildpg=true || ref=main;
         echo "pgtag=${{ env.REGISTRY }}/stac-utils/pgstac-postgres:$ref" >>$GITHUB_OUTPUT;
         echo "buildpg=$buildpg" >>$GITHUB_OUTPUT;
         buildy=false;
-        ref=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}};
         [[ "${{ steps.filter.outputs.pypgstac }}" == "true" ]] && buildpy=true || ref=main;
         echo "pytag=${{ env.REGISTRY }}/stac-utils/pgstac-pyrust:$ref" >>$GITHUB_OUTPUT;
         echo "buildpy=$buildpg" >>$GITHUB_OUTPUT;

docker-compose.yml

@@ -4,6 +4,7 @@ services:
     image: pgstac
     build:
       context: .
+      network: host
       dockerfile: docker/pgstac/Dockerfile
       target: pgstac
     platform: linux/amd64
@@ -24,6 +25,7 @@ services:
     image: pypgstac
     build:
       context: .
+      network: host
       dockerfile: docker/pypgstac/Dockerfile
       target: pypgstac
     platform: linux/amd64

docker/pgstac/Dockerfile

@@ -2,7 +2,7 @@ ARG PG_MAJOR=15
 ARG POSTGIS_MAJOR=3
 
 # Base postgres image that pgstac can be installed onto
-FROM postgres:${PG_MAJOR}-bullseye as pgstacbase
+FROM postgres:${PG_MAJOR}-bullseye AS pgstacbase
 ARG POSTGIS_MAJOR
 RUN  \
     apt-get update \
@@ -19,7 +19,7 @@ RUN  \
 COPY docker/pgstac/dbinit/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 # Base postgres image with plrust installed that can be used for future development using plrust
-FROM pgstacbase as pgstacbase-plrust
+FROM pgstacbase AS pgstacbase-plrust
 ENV PLRUSTVERSION=1.2.7
 ENV RUSTVERSION=1.72.0
 ENV PLRUSTDOWNLOADURL=https://github.com/tcdi/plrust/releases/download/
@@ -63,13 +63,13 @@ USER root
 RUN apt-get install -y /${PLRUSTFILE}
 
 # The pgstacbase image with latest version of pgstac installed
-FROM pgstacbase as pgstac
+FROM pgstacbase AS pgstac
 WORKDIR /docker-entrypoint-initdb.d
 COPY docker/pgstac/dbinit/pgstac.sh 990_pgstac.sh
 COPY src/pgstac/pgstac.sql 999_pgstac.sql
 
 # The pgstacbase-plrust image with the latest version of pgstac installed
-FROM pgstacbase-plrust as pgstac-plrust
+FROM pgstacbase-plrust AS pgstac-plrust
 WORKDIR /docker-entrypoint-initdb.d
 COPY docker/pgstac/dbinit/pgstac.sh 990_pgstac.sh
 COPY src/pgstac/pgstac.sql 999_pgstac.sql

docker/pypgstac/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1-slim-bullseye as pyrustbase
+FROM rust:1-slim-bullseye AS pyrustbase
 ENV PYTHONWRITEBYTECODE=1
 ENV PYTHONBUFFERED=1
 ENV PIP_ROOT_USER_ACTION=ignore
@@ -17,14 +17,14 @@ RUN \
     && apt-get clean && apt-get -y autoremove \
     && rm -rf /var/lib/apt/lists/*
 
-FROM pyrustbase as pypgstac
+FROM pyrustbase AS pypgstac
 COPY ./src/pypgstac/pyproject.toml /tmp/pyproject.toml
 WORKDIR /tmp
 RUN \
     uv pip compile --all-extras /tmp/pyproject.toml >/tmp/requirements.txt \
     && uv pip install --system -r /tmp/requirements.txt
 COPY docker/pypgstac/bin /opt/docker/pypgstac/bin
 COPY src/pypgstac /opt/src/pypgstac
-COPY src/pgstac/migrations /opt/src/pgstac/migrations
+COPY src/pgstac /opt/src/pgstac
 WORKDIR /opt/src/pypgstac
 RUN uv pip install --system -e . && rm -rf /usr/local/cargo/registry

docker/pypgstac/bin/stageversion

@@ -40,7 +40,7 @@ cd $BASEDIR/pgstac
 
 # make the base pgstac.sql a symbolic link to the most recent version
 rm pgstac.sql
-ln -s migrations/pgstac.${VERSION}.sql pgstac.sql
+cp migrations/pgstac.${VERSION}.sql pgstac.sql
 
 # Update the version number in the appropriate places
 [[ $VERSION == 'unreleased' ]] && PYVERSION="${OLDVERSION}-dev" || PYVERSION="$VERSION"

docker/pypgstac/bin/test

@@ -4,6 +4,10 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 export SRCDIR=$SCRIPT_DIR/../../../src
 export PGSTACDIR=$SRCDIR/pgstac
 
+echo $SCRIPT_DIR
+echo $SRCDIR
+echo $PGSTACDIR
+
 if [[ "${CI}" ]]; then
     set -x
 fi
@@ -42,7 +46,7 @@ function test_formatting(){
     cd $SRCDIR/pypgstac
 
     echo "Running ruff"
-    ruff python tests
+    ruff -n python tests
 
     echo "Running mypy"
     mypy python

scripts/runinpypgstac

@@ -25,16 +25,30 @@ do
         NOCACHE="--no-cache"
     elif [[ $ARG == "--build" ]]; then
         BUILD=1
+    elif [[ $ARG == "--cpfiles" ]]; then
+        CPFILES=1
     else
         CONTAINER_ARGS+=($ARG)
     fi
 done
 if [[ $BUILD == 1 ]]; then
     echo "Building docker images..."
+    docker compose down -v --remove-orphans
     docker compose build $NOCACHE
+    docker compose up -d pgstac
+    sleep 4
 fi
 PGSTAC_RUNNING=$(docker compose ps pgstac --status running -q)
-docker compose run -T --rm -u $(id -u):$(id -g) pypgstac "${CONTAINER_ARGS[@]}"
+if [[ $CPFILES == 1 ]]; then
+    docker ps | grep pypgstacworker
+    [[ $? == 0 ]] && echo "Killing pypgstacworker" && docker kill pypgstacworker
+    docker compose run -d --rm --name pypgstacworker pypgstac /bin/bash
+    docker compose exec pypgstac "${CONTAINER_ARGS[@]}"
+    docker cp pypgstacworker:/opt/src $SCRIPT_DIR/..
+    docker kill pypgstacworker
+else
+    docker compose run -T --rm pypgstac "${CONTAINER_ARGS[@]}"
+fi
 JOBEXITCODE=$?
 [[ $PGSTAC_RUNNING == "" ]] && docker compose stop pgstac
 exit $JOBEXITCODE

scripts/stageversion

@@ -1,4 +1,4 @@
 #!/bin/bash
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 cd $SCRIPT_DIR/..
-$SCRIPT_DIR/runinpypgstac stageversion "$@"
+$SCRIPT_DIR/runinpypgstac --build --cpfiles stageversion "$@"

scripts/test

@@ -1,3 +1,3 @@
 #!/bin/bash
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-$SCRIPT_DIR/runinpypgstac test "$@"
+$SCRIPT_DIR/runinpypgstac --build test "$@"