GDAL 3.7.2 (#120)

* GDAL 3.7.2 Dockerfile.environment and tests adjustments * Unsure a proper GDALDestory * Fix MacOS builds * Minor cleanups * pthread_yield => sched_yield * README.md update * .gitkeep workflows dir * Add the rest of the jobs into the docker-compose file * Review comments, make a better vectors comparison * Replace abs diff & eps comparison with the BOOST_CHECK_SMALL function * Parmetrize env Dockerfile via Build arguments
geotrellis · Nov 6, 2023 · fd0cc7d · fd0cc7d
1 parent e6ef89d
commit fd0cc7d
Show file tree

Hide file tree

Showing 10 changed files with 166 additions and 43 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,7 +3,7 @@ version: 2.1
 executors:
   gdal-build-executor-amd64:
     docker:
-      - image: quay.io/geotrellis/gdal-warp-bindings-environment:amd64-2
+      - image: daunnc/gdal-warp-bindings-environment:3.7.2-amd64
     working_directory: /workdir
 
   gdal-build-executor-arm64:
@@ -71,10 +71,10 @@ jobs:
       CXX: c++
       CFLAGS: "-Wall -Werror -Og -ggdb3"
       JAVA_HOME: "/macintosh/jdk8u202-b08/Contents/Home"
-      GDALCFLAGS: "-I/macintosh/gdal/3.1.2/include"
+      GDALCFLAGS: "-I/macintosh/gdal/3.7.2/include"
       CXXFLAGS: "-I/usr/osxcross/SDK/MacOSX10.10.sdk/usr/include/c++/v1"
       BOOST_ROOT: "/usr/local/include/boost_1_69_0"
-      LDFLAGS: "-mmacosx-version-min=10.9 -L/macintosh/gdal/3.1.2/lib -lgdal -lstdc++ -lpthread -Wl,-rpath,/usr/local/lib"
+      LDFLAGS: "-mmacosx-version-min=10.9 -L/macintosh/gdal/3.7.2/lib -lgdal -lstdc++ -lpthread -Wl,-rpath,/usr/local/lib"
       LD_LIBRARY_PATH_ORIGIN: ""
       CROSS_ROOT: /usr/x86_64-apple-darwin14
       PATH: /usr/x86_64-apple-darwin14/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml
@@ -0,0 +1,101 @@
+version: "3.9"
+
+services:
+  # cd .github/workflows; docker compose -f ../docker-compose.yml up build-linux-amd64
+  build-linux-amd64:
+    image: daunnc/gdal-warp-bindings-environment:3.7.2-amd64
+    working_dir: /workdir
+    environment:
+      CC: gcc
+      CXX: g++
+      CFLAGS: "-Wall -Werror -Og -ggdb3 -DSO_FINI -D_GNU_SOURCE"
+      BOOST_ROOT: "/usr/local/include/boost_1_69_0"
+      JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-amd64"
+    volumes:
+      - ./../:/workdir/
+    command: 
+      - /bin/bash
+      - -c
+      - |
+        ln -s /tmp/c41078a1.tif src/experiments/data/c41078a1.tif
+        make -j4 -C src tests 
+        make -j4 -C src/experiments/thread pattern oversubscribe
+    network_mode: host
+
+  build-linux-arm64:
+    image: quay.io/geotrellis/gdal-warp-bindings-environment:arm64-2
+    working_dir: /workdir
+    environment:
+      ARCH: arm64
+      CC: aarch64-linux-gnu-gcc-8
+      CXX: aarch64-linux-gnu-g++-8
+      CFLAGS: "-Wall -Og -ggdb3 -DSO_FINI -D_GNU_SOURCE"
+      BOOST_ROOT: "/usr/local/include/boost_1_69_0"
+      JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-arm64"
+    volumes:
+      - ./../:/workdir/
+    command: 
+      - /bin/bash
+      - -c
+      - |
+        touch /workdir/src/main/java/com/azavea/gdal/GDALWarp.class
+        cp -f /tmp/workdir/*.h /workdir/src/
+        make -j4 -C src libgdalwarp_bindings-arm64.so
+    network_mode: host
+
+  build-macos-amd64:
+    image: daunnc/gdal-warp-bindings-environment:3.7.2-amd64
+    working_dir: /workdir
+    environment:
+      OSXCROSS_NO_INCLUDE_PATH_WARNINGS: 1
+      CROSS_TRIPLE: x86_64-apple-darwin14
+      OS: darwin
+      SO: dylib
+      CC: cc
+      CXX: c++
+      CFLAGS: "-Wall -Werror -Og -ggdb3"
+      JAVA_HOME: "/macintosh/jdk8u202-b08/Contents/Home"
+      GDALCFLAGS: "-I/macintosh/gdal/3.7.2/include"
+      CXXFLAGS: "-I/usr/osxcross/SDK/MacOSX10.10.sdk/usr/include/c++/v1"
+      BOOST_ROOT: "/usr/local/include/boost_1_69_0"
+      LDFLAGS: "-mmacosx-version-min=10.9 -L/macintosh/gdal/3.7.2/lib -lgdal -lstdc++ -lpthread -Wl,-rpath,/usr/local/lib"
+      LD_LIBRARY_PATH_ORIGIN: ""
+      CROSS_ROOT: /usr/x86_64-apple-darwin14
+      PATH: /usr/x86_64-apple-darwin14/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+      PATH_ORIGIN: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+    volumes:
+      - ./../:/workdir/
+    command: 
+      - /bin/bash
+      - -c
+      - |
+        touch /workdir/src/main/java/com/azavea/gdal/GDALWarp.class
+        cp -f /tmp/workdir/*.h /workdir/src/
+        make -j4 -C src libgdalwarp_bindings-amd64.dylib
+    network_mode: host
+
+  build-windows-amd64:
+    image: daunnc/gdal-warp-bindings-environment:3.7.2-amd64
+    working_dir: /workdir
+    environment:
+      CROSS_TRIPLE: x86_64-w64-mingw32
+      OS: win32
+      CFLAGS: "-Wall -Werror -Og -g"
+      JAVA_HOME: "/windows/jdk8u202-b08"
+      GDALCFLAGS: "-I/usr/local/include"
+      BOOST_ROOT: "/usr/local/include/boost_1_69_0"
+      LDFLAGS: "-L/windows/gdal/lib -lgdal_i -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic -lws2_32"
+      LD_LIBRARY_PATH: "/usr/x86_64-linux-gnu/x86_64-w64-mingw32/lib:"
+      LD_LIBRARY_PATH_ORIGIN: ""
+      CROSS_ROOT: /usr/x86_64-w64-mingw32
+      PATH: /usr/x86_64-w64-mingw32/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+    volumes:
+      - ./../:/workdir/
+    command: 
+      - /bin/bash
+      - -c
+      - |
+        touch /workdir/src/main/java/com/azavea/gdal/GDALWarp.class
+        cp -f /tmp/workdir/*.h /workdir/src/
+        make -j4 -C src gdalwarp_bindings-amd64.dll
+    network_mode: host
diff --git a/.github/workflows/.gitkeep b/.github/workflows/.gitkeep
diff --git a/Docker/Dockerfile.environment-amd64 b/Docker/Dockerfile.environment-amd64
@@ -1,6 +1,13 @@
 FROM quay.io/geotrellis/gdal-warp-bindings-crossbuild:amd64-2
 LABEL maintainer="Azavea <[email protected]>"
 
+ARG GDAL_VERSION=3.7.2
+ARG GDAL_MACOS=libgdal-3.7.2-h85f6614_6.conda
+ARG GDAL_WINDOWS=release-1916-x64-gdal-3-7-2-mapserver-8-0-1-libs.zip
+ARG PROJ_VERSION=9.3.0
+ARG LIBTIFF_VERSION=4.1.0
+ARG CURL_VERSION=7.71.1
+
 RUN apt-get update -y && \
     apt-get install build-essential pkg-config openjdk-8-jdk -y -q && \
     apt-get autoremove && \
@@ -10,18 +17,18 @@ RUN apt-get update -y && \
 # Install SQLite
 RUN apt-get install -y sqlite3 libsqlite3-dev
 
-# Build GDAL 3.1.2
+# Build GDAL ${GDAL_VERSION}
 RUN cd /usr/local/src && \
-    wget -k 'https://download.osgeo.org/gdal/3.1.2/gdal-3.1.2.tar.gz' && \
-    wget -k 'https://download.osgeo.org/proj/proj-7.1.0.tar.gz' && \
-    wget -k 'https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz' && \
-    wget -k 'https://curl.haxx.se/download/curl-7.71.1.tar.gz' && \
-    tar axvf gdal-3.1.2.tar.gz && tar axvf proj-7.1.0.tar.gz && tar axvf tiff-4.1.0.tar.gz && tar axvf curl-7.71.1.tar.gz && \
-    cd curl-7.71.1 && ./configure --prefix=/usr/local && nice -n 19 make -j33 && make install && \
-    cd ../tiff-4.1.0 && ./configure --prefix=/usr/local && nice -n 19 make -j33 && make install && \
-    cd ../proj-7.1.0 && ./configure --prefix=/usr/local && nice -n 19 make -j33 && make install && \
-    cd ../gdal-3.1.2 && ./configure --prefix=/usr/local && nice -n 19 make -j33 && make install && \
-    cd .. && rm -r curl-7.71.1/ tiff-4.1.0/ proj-7.1.0/ gdal-3.1.2/ curl-7.71.1.tar.gz tiff-4.1.0.tar.gz proj-7.1.0.tar.gz gdal-3.1.2.tar.gz
+    wget -k "https://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz" && \
+    wget -k "https://download.osgeo.org/proj/proj-${PROJ_VERSION}.tar.gz" && \
+    wget -k "https://download.osgeo.org/libtiff/tiff-${LIBTIFF_VERSION}.tar.gz" && \
+    wget -k "https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz" && \
+    tar axvf gdal-${GDAL_VERSION}.tar.gz && tar axvf proj-${PROJ_VERSION}.tar.gz && tar axvf tiff-${LIBTIFF_VERSION}.tar.gz && tar axvf curl-${CURL_VERSION}.tar.gz && \
+    cd curl-${CURL_VERSION} && ./configure --prefix=/usr/local && nice -n 19 make -j33 && make install && \
+    cd ../tiff-${LIBTIFF_VERSION} && ./configure --prefix=/usr/local && nice -n 19 make -j33 && make install && \
+    cd ../proj-${PROJ_VERSION} && mkdir -p build && cd ./build && cmake .. && cmake --build . -j 33 && cmake --build . --target install && \
+    cd ../../gdal-${GDAL_VERSION} && mkdir -p build && cd ./build && cmake .. && cmake --build . -j 33 && cmake --build . --target install && \
+    cd ../.. && rm -r curl-${CURL_VERSION}/ tiff-${LIBTIFF_VERSION}/ proj-${PROJ_VERSION}/ gdal-${GDAL_VERSION}/ curl-${CURL_VERSION}.tar.gz tiff-${LIBTIFF_VERSION}.tar.gz proj-${PROJ_VERSION}.tar.gz gdal-${GDAL_VERSION}.tar.gz
 
 # Test data
 RUN wget 'https://download.osgeo.org/geotiff/samples/usgs/c41078a1.tif' -k -O /tmp/c41078a1.tif
@@ -36,13 +43,16 @@ RUN wget 'https://boostorg.jfrog.io/artifactory/main/release/1.69.0/source/boost
 # Macintosh
 RUN mkdir -p /macintosh && \
     cd /macintosh && \
+    wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && /bin/bash ~/miniconda.sh -b && \
+    eval "$(/root/miniconda3/bin/conda shell.bash hook)" && \
     wget "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u202-b08/OpenJDK8U-jdk_x64_mac_hotspot_8u202b08.tar.gz" && \
     tar axvf OpenJDK8U-jdk_x64_mac_hotspot_8u202b08.tar.gz && \
     rm -f OpenJDK8U-jdk_x64_mac_hotspot_8u202b08.tar.gz && \
-    wget "https://anaconda.org/conda-forge/libgdal/3.1.2/download/osx-64/libgdal-3.1.2-hd7bf8dc_4.tar.bz2" && \
-    mkdir -p gdal/3.1.2 && \
-    tar axvf libgdal-3.1.2-hd7bf8dc_4.tar.bz2 -C gdal/3.1.2 && \
-    rm -f libgdal-3.1.2-hd7bf8dc_4.tar.bz2
+    wget "https://anaconda.org/conda-forge/libgdal/${GDAL_VERSION}/download/osx-64/${GDAL_MACOS}" && \
+    mkdir -p gdal/${GDAL_VERSION} && \
+    cph extract ${GDAL_MACOS} --dest gdal/${GDAL_VERSION} && \
+    rm -f ${GDAL_MACOS} && \
+    rm -r /root/miniconda3
 
 # Windows
 RUN mkdir -p /windows && \
@@ -52,11 +62,11 @@ RUN mkdir -p /windows && \
     rm -r OpenJDK8U-jdk_x64_windows_hotspot_8u202b08.zip && \
     mkdir -p /windows/gdal && \
     cd /windows/gdal && \
-    wget "http://download.gisinternals.com/sdk/downloads/release-1911-x64-gdal-3-0-4-mapserver-7-4-3-libs.zip" && \
-    unzip release-1911-x64-gdal-3-0-4-mapserver-7-4-3-libs.zip && \
-    rm -f release-1911-x64-gdal-3-0-4-mapserver-7-4-3-libs.zip
+    wget "http://download.gisinternals.com/sdk/downloads/${GDAL_WINDOWS}" && \
+    unzip ${GDAL_WINDOWS} && \
+    rm -f ${GDAL_WINDOWS}
 
 # Linkage
 RUN echo '/usr/local/lib' >> /etc/ld.so.conf && ldconfig
 
-# docker build -f Dockerfile.environment-amd64 -t quay.io/geotrellis/gdal-warp-bindings-environment:amd64-2 .
+# docker build -f Dockerfile.environment-amd64 -t quay.io/geotrellis/gdal-warp-bindings-environment:${GDAL_VERSION}-amd64 .
diff --git a/README.md b/README.md
@@ -11,12 +11,13 @@ APIs are provided for C and Java.
 
 These bindings require a GDAL installation on your machine with the appropriate matching version:
 
-| GDAL Warp Bindings | OS            |  GDAL | Shared Library {so,dylib,dll} |
-|--------------------|---------------|-------|-------------------------------|
-|              1.1.x | Linux (AMD64) | 3.1.2 | libgdal.so.27                 |
-|              1.1.x | Linux (ARM64) | 2.4.0 | libgdal.so.20                 |
-|              1.1.x | MacOS (AMD64) | 3.1.2 | libgdal.27.dylib              |
-|              1.1.x | Windows       | 3.0.4 | --                            |
+| GDAL Warp Bindings | OS                            |  GDAL | Shared Library {so,dylib,dll} |
+|--------------------|-------------------------------|-------|-------------------------------|
+|              3.7.0 | Linux, MacOS, Windows (AMD64) | 3.7.2 | libgdal.so.33.3.7.2           |
+|              1.1.x | Linux (AMD64)                 | 3.1.2 | libgdal.so.27                 |
+|              1.1.x | Linux (ARM64)                 | 2.4.0 | libgdal.so.20                 |
+|              1.1.x | MacOS (AMD64)                 | 3.1.2 | libgdal.27.dylib              |
+|              1.1.x | Windows                       | 3.0.4 | --                            |
 
 ## MacOS ##
 

diff --git a/src/bindings.cpp b/src/bindings.cpp
@@ -70,18 +70,6 @@ static uint64_t get_nanos()
 #endif
 }
 
-#if defined(__APPLE__)
-inline void pthread_yield()
-{
-    pthread_yield_np();
-}
-#elif defined(__MINGW32__)
-inline void pthread_yield()
-{
-    sleep(0);
-}
-#endif
-
 /**
  * A macro for making one attempt to perform the given operation on
  * (one of) the locked datasets.  If an attempt succeeds, then the
@@ -142,7 +130,7 @@ inline void pthread_yield()
             TRY(fn)                                                                       \
             if (!done)                                                                    \
             {                                                                             \
-                pthread_yield();                                                          \
+                sched_yield();                                                          \
             }                                                                             \
         }                                                                                 \
         if ((code == ATTEMPT_SUCCESSFUL) && ((i < attempts) || (i > 0 && attempts == 0))) \
@@ -280,6 +268,7 @@ void deinit()
     env_deinit();
     cache_deinit();
     token_deinit();
+    GDALDestroyDriverManager();
 }
 
 #if defined(SO_FINI) && defined(__linux__)

diff --git a/src/experiments/thread/rawthread.cpp b/src/experiments/thread/rawthread.cpp
@@ -220,6 +220,7 @@ int main(int argc, char **argv)
     GDALWarpAppOptionsFree(app_options);
     GDALClose(dataset);
     GDALClose(source);
+    GDALDestroyDriverManager();
     unlink(temp_path);
 
     return 0;

diff --git a/src/experiments/thread/wrapthread.cpp b/src/experiments/thread/wrapthread.cpp
@@ -262,6 +262,7 @@ int main(int argc, char **argv)
 
     deinit();
     GDALWarpAppOptionsFree(app_options);
+    GDALDestroyDriverManager();
 
     return 0;
 }
diff --git a/src/unit_tests/cache_tests.cpp b/src/unit_tests/cache_tests.cpp
@@ -167,3 +167,8 @@ BOOST_AUTO_TEST_CASE(passive_multiple_test)
     cache.get(uri_options1, 16);
     BOOST_TEST(cache.count(uri_options1) <= 8);
 }
+
+BOOST_AUTO_TEST_CASE(destroy)
+{
+    GDALDestroyDriverManager();
+}
diff --git a/src/unit_tests/dataset_tests.cpp b/src/unit_tests/dataset_tests.cpp
@@ -36,6 +36,16 @@ auto options2 = options_t{"-r", "bilinear",
 auto uri_options1 = std::make_pair(uri1, options1);
 auto uri_options2 = std::make_pair(uri1, options2);
 
+double EPSILON = 10e-9;
+
+void BOOST_TEST_VECTOR_DOUBLE(std::vector<double> actual, std::vector<double> expected, double eps = EPSILON) {
+    BOOST_TEST(actual.size() == expected.size());
+
+    for (int i = 0; i < expected.size(); i++) {
+        BOOST_CHECK_SMALL((expected[i] - actual[i]), eps);
+    }
+}
+
 BOOST_AUTO_TEST_CASE(init)
 {
     GDALAllRegister();
@@ -55,7 +65,7 @@ BOOST_AUTO_TEST_CASE(get_block_size)
 
     ld.get_block_size(locked_dataset::WARPED, 1, &width, &height);
     BOOST_TEST(width == 512);
-    BOOST_TEST(height == 128);
+    BOOST_TEST(height == 512);
 
     errno_deinit();
 }
@@ -458,7 +468,7 @@ BOOST_AUTO_TEST_CASE(get_transform_test)
     ld.get_transform(locked_dataset::WARPED, transform);
     actual.insert(actual.end(), transform, transform + 6);
 
-    BOOST_TEST(actual == expected);
+    BOOST_TEST_VECTOR_DOUBLE(actual, expected);
 
     errno_deinit();
 }
@@ -543,3 +553,8 @@ BOOST_AUTO_TEST_CASE(width_height_test)
 
     errno_deinit();
 }
+
+BOOST_AUTO_TEST_CASE(destroy)
+{
+    GDALDestroyDriverManager();
+}