diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 2943422..88f55c9 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -23,106 +23,55 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - PLAT: [i686, x86_64] - INTERFACE64: ['0', '1'] - MB_ML_VER: ['2014'] - MB_ML_LIBC: ['manylinux'] include: - - os: macos-latest - PLAT: arm64 - INTERFACE64: '1' - - os: macos-latest - PLAT: arm64 - INTERFACE64: '0' - - os: ubuntu-latest - PLAT: x86_64 - INTERFACE64: '1' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_2 - - os: ubuntu-latest - PLAT: x86_64 - INTERFACE64: '0' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_2 - - - os: ubuntu-24.04-arm - PLAT: aarch64 - INTERFACE64: '0' - MB_ML_VER: '2014' - - os: ubuntu-24.04-arm - PLAT: aarch64 - INTERFACE64: '1' - MB_ML_VER: '2014' - - os: ubuntu-24.04-arm - PLAT: aarch64 - INTERFACE64: '0' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_2 - - os: ubuntu-24.04-arm - PLAT: aarch64 - INTERFACE64: '1' - MB_ML_LIBC: musllinux - MB_ML_VER: _1_2 - - exclude: - - PLAT: i686 - os: macos-latest - - PLAT: i686 - INTERFACE64: '1' + - { os: ubuntu-latest, PLAT: i686, INTERFACE64: '0', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + + - { os: ubuntu-latest, PLAT: x86_64, INTERFACE64: '0', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + - { os: ubuntu-latest, PLAT: x86_64, INTERFACE64: '1', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + + - { os: macos-latest, PLAT: x86_64, INTERFACE64: '0', MB_ML_LIBC: macosx} + - { os: macos-latest, PLAT: x86_64, INTERFACE64: '1', MB_ML_LIBC: macosx} + + - { os: macos-latest, PLAT: arm64, INTERFACE64: '0', MB_ML_LIBC: macosx} + - { os: macos-latest, PLAT: arm64, INTERFACE64: '1', MB_ML_LIBC: macosx} + + - { os: ubuntu-latest, PLAT: x86_64, INTERFACE64: '0', MB_ML_VER: '_1_2', MB_ML_LIBC: musllinux} + - { os: ubuntu-latest, PLAT: x86_64, INTERFACE64: '1', MB_ML_VER: '_1_2', MB_ML_LIBC: musllinux} + + - { os: ubuntu-24.04-arm, PLAT: aarch64, INTERFACE64: '0', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + - { os: ubuntu-24.04-arm, PLAT: aarch64, INTERFACE64: '1', MB_ML_VER: '2014', MB_ML_LIBC: manylinux} + + - { os: ubuntu-24.04-arm, PLAT: aarch64, INTERFACE64: '0', MB_ML_VER: '_1_2', MB_ML_LIBC: musllinux} + - { os: ubuntu-24.04-arm, PLAT: aarch64, INTERFACE64: '1', MB_ML_VER: '_1_2', MB_ML_LIBC: musllinux} + env: NIGHTLY: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} - MB_PYTHON_VERSION: ${{ matrix.python-version }} - TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} MB_ML_LIBC: ${{ matrix.MB_ML_LIBC }} MB_ML_VER: ${{ matrix.MB_ML_VER }} INTERFACE64: ${{ matrix.INTERFACE64 }} BUILD_DIR: ${{ github.workspace }} PLAT: ${{ matrix.PLAT }} + OS-NAME: ${{ matrix.os }} steps: - uses: actions/checkout@v4.1.1 with: submodules: recursive fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.9 + - name: Set extra env run: | echo "DOCKER_TEST_IMAGE=$(echo multibuild/xenial_${{ matrix.PLAT}})" >> $GITHUB_ENV; - uses: maxim-lobanov/setup-xcode@v1.6.0 - if: ${{ matrix.os == 'macos-latest' }} + if: ${{ contains(matrix.os, 'macos') }} with: - xcode-version: '15.4' + xcode-version: '16.0' - name: Print some Environment variable run: | echo "PLAT: ${PLAT}" echo "DOCKER_TEST_IMAGE: ${DOCKER_TEST_IMAGE}" - - name: Install VirtualEnv - run: | - python3 -m pip install --upgrade pip - pip install virtualenv - - name: Build OpenBLAS - run: | - set -xeo pipefail - source tools/build_steps.sh - echo "------ BEFORE BUILD ---------" - before_build - if [[ "$NIGHTLY" = "true" ]]; then - echo "------ CLEAN CODE --------" - clean_code $REPO_DIR develop - echo "------ BUILD LIB --------" - build_lib "$PLAT" "$INTERFACE64" "1" - else - echo "------ CLEAN CODE --------" - clean_code $REPO_DIR $OPENBLAS_COMMIT - echo "------ BUILD LIB --------" - build_lib "$PLAT" "$INTERFACE64" "0" - fi # - name: Setup tmate session # if: ${{ failure() }} @@ -130,34 +79,33 @@ jobs: # with: # limit-access-to-actor: true - - name: Build and test wheel - run: | - if [[ "$NIGHTLY" = "true" ]]; then - # Set the pyproject.toml version: convert v0.3.24-30-g138ed79f to 0.3.34.30 - version=$(cd OpenBLAS && git describe --tags --abbrev=8 | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") - sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml - fi - if [ "macos-latest" == "${{ matrix.os }}" ]; then - source tools/build_wheel.sh - else - libc=${MB_ML_LIBC:-manylinux} - docker_image=quay.io/pypa/${libc}${MB_ML_VER}_${PLAT} - docker run --rm -e INTERFACE64="${INTERFACE64}" \ - -e MB_ML_LIBC="${MB_ML_LIBC}" \ - -v $(pwd):/openblas $docker_image \ - /bin/bash -xe /openblas/tools/build_wheel.sh - sudo chmod -R a+w dist - fi - - - uses: actions/upload-artifact@v4.3.0 + - name: Build and Test wheels + uses: pypa/cibuildwheel@v3.1.4 + with: + output-dir: dist + env: + CIBW_ARCHS: ${{matrix.PLAT}} + CIBW_BUILD_VERBOSITY: 1 + CIBW_BUILD: "cp39-${{ matrix.MB_ML_LIBC }}_${{matrix.PLAT}}" + + CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + CIBW_MUSLLINUX_X86_64_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.MB_ML_LIBC }}${{matrix.MB_ML_VER}} + + - name: Upload wheels to artifacts + uses: actions/upload-artifact@v4.3.0 with: name: wheels-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }}-${{ matrix.MB_ML_VER }} path: dist/scipy_openblas*.whl - - uses: actions/upload-artifact@v4.3.0 + + - name: Upload openblas to artifacts + uses: actions/upload-artifact@v4.3.0 with: name: openblas-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }}-${{ matrix.MB_ML_VER }} - path: libs/openblas*.tar.gz + path: dist/openblas*.tar.gz - uses: conda-incubator/setup-miniconda@v3.2.0 with: diff --git a/.gitmodules b/.gitmodules index 9a0ddc0..546361d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,3 @@ [submodule "OpenBLAS"] path = OpenBLAS url = https://github.com/xianyi/OpenBLAS.git -[submodule "multibuild"] - path = multibuild - url = https://github.com/multi-build/multibuild.git -[submodule "gfortran-install"] - path = gfortran-install - url = https://github.com/MacPython/gfortran-install.git diff --git a/build-openblas.sh b/build-openblas.sh new file mode 100755 index 0000000..79d4f25 --- /dev/null +++ b/build-openblas.sh @@ -0,0 +1,97 @@ +#! /bin/bash + + +# Most of the content in this file comes from https://github.com/multi-build/multibuild, with some modifications +# Follow the license below + + + +# .. _license: + +# ********************* +# Copyright and License +# ********************* + +# The multibuild package, including all examples, code snippets and attached +# documentation is covered by the 2-clause BSD license. + +# Copyright (c) 2013-2024, Matt Terry and Matthew Brett; all rights +# reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +set -xeo pipefail +source tools/build_steps.sh +echo "------ BEFORE BUILD ---------" +before_build + +function fill_submodule { + # Restores .git directory to submodule, if necessary + # See: + # https://stackoverflow.com/questions/41776331/is-there-a-way-to-reconstruct-a-git-directory-for-a-submodule + local repo_dir="$1" + [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 + local git_loc="$repo_dir/.git" + # For ordinary submodule, .git is a file. + [ -d "$git_loc" ] && return + # Need to recreate .git directory for submodule + local origin_url=$(cd "$repo_dir" && git config --get remote.origin.url) + local repo_copy="$repo_dir-$RANDOM" + git clone --recursive "$repo_dir" "$repo_copy" + rm -rf "$repo_dir" + mv "${repo_copy}" "$repo_dir" + (cd "$repo_dir" && git remote set-url origin $origin_url) +} + +function clean_code { + local repo_dir=${1:-$REPO_DIR} + local build_commit=${2:-$BUILD_COMMIT} + [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 + [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 + # The package $repo_dir may be a submodule. git submodules do not + # have a .git directory. If $repo_dir is copied around, tools like + # Versioneer which require that it be a git repository are unable + # to determine the version. Give submodule proper git directory + fill_submodule "$repo_dir" + (cd $repo_dir \ + && git fetch origin --tags \ + && git checkout $build_commit \ + && git clean -fxd \ + && git reset --hard \ + && git submodule update --init --recursive) +} + + +if [[ "$NIGHTLY" = "true" ]]; then + echo "------ CLEAN CODE --------" + clean_code $REPO_DIR develop + echo "------ BUILD LIB --------" + build_lib "$PLAT" "$INTERFACE64" "1" +else + echo "------ CLEAN CODE --------" + clean_code $REPO_DIR $OPENBLAS_COMMIT + echo "------ BUILD LIB --------" + build_lib "$PLAT" "$INTERFACE64" "0" +fi \ No newline at end of file diff --git a/ci-before-build.sh b/ci-before-build.sh new file mode 100755 index 0000000..a4a1594 --- /dev/null +++ b/ci-before-build.sh @@ -0,0 +1,103 @@ +#! /bin/bash + + +# Most of the content in this file comes from https://github.com/multi-build/multibuild, with some modifications +# Follow the license below + + + +# .. _license: + +# ********************* +# Copyright and License +# ********************* + +# The multibuild package, including all examples, code snippets and attached +# documentation is covered by the 2-clause BSD license. + +# Copyright (c) 2013-2024, Matt Terry and Matthew Brett; all rights +# reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: + +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +#! /bin/bash +set -xe + +if [[ "$NIGHTLY" = "true" ]]; then + # Set the pyproject.toml version: convert v0.3.24-30-g138ed79f to 0.3.34.30 + version=$(cd OpenBLAS && git describe --tags --abbrev=8 | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") + sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml +fi + + +#!/bin/bash +# Utilities for both OSX and Docker Linux +# python or python3 should be on the PATH + +# Only source common_utils once +if [ -n "$COMMON_UTILS_SOURCED" ]; then + return +fi +COMMON_UTILS_SOURCED=1 + +# Turn on exit-if-error +set -e + +MULTIBUILD_DIR=$(dirname "${BASH_SOURCE[0]}") +DOWNLOADS_SDIR=downloads +PYPY_URL=https://downloads.python.org/pypy + +if [ $(uname) == "Darwin" ]; then + IS_MACOS=1; IS_OSX=1; +else + # In the manylinux_2_24 image, based on Debian9, "python" is not installed + # so link in something for the various system calls before PYTHON_EXE is set + which python || export PATH=/opt/python/cp39-cp39/bin:$PATH + + if [ "$MB_ML_LIBC" == "musllinux" ]; then + IS_ALPINE=1; + MB_ML_VER=${MB_ML_VER:-"_1_2"} + else + # Default Manylinux version + MB_ML_VER=${MB_ML_VER:-2014} + fi +fi + +# Work round bug in travis xcode image described at +# https://github.com/direnv/direnv/issues/210 +shell_session_update() { :; } + +# Workaround for https://github.com/travis-ci/travis-ci/issues/8703 +# suggested by Thomas K at +# https://github.com/travis-ci/travis-ci/issues/8703#issuecomment-347881274 +unset -f cd +unset -f pushd +unset -f popd + +# Build OpenBLAS +source build-openblas.sh + +source tools/build_prepare.sh diff --git a/ci-repair-wheel.sh b/ci-repair-wheel.sh new file mode 100755 index 0000000..22fb2ba --- /dev/null +++ b/ci-repair-wheel.sh @@ -0,0 +1,35 @@ +#! /bin/bash +set -xe + +PYTHON=${PYTHON:-python3.9} + +if [ $(uname) == "Darwin" ]; then + $PYTHON -m pip install delocate + # move the mis-named scipy_openblas64-none-any.whl to a platform-specific name + # if [ "${PLAT}" == "arm64" ]; then + # for f in $2/*.whl; do mv $f "${f/%any.whl/macosx_11_0_$PLAT.whl}"; done + # else + # for f in $2/*.whl; do mv $f "${f/%any.whl/macosx_10_9_$PLAT.whl}"; done + # fi + delocate-wheel -w $1 -v $2 + + cp libs/openblas*.tar.gz dist/ +else + auditwheel repair -w $1 --lib-sdir /lib $2 + # rm dist/scipy_openblas*-none-any.whl + # rm {dest_dir}/*.whl + + # Add an RPATH to libgfortran: + # https://github.com/pypa/auditwheel/issues/451 + if [ "$MB_ML_LIBC" == "musllinux" ]; then + apk add zip + else + yum install -y zip + fi + unzip $1/*.whl "*libgfortran*" + patchelf --force-rpath --set-rpath '$ORIGIN' */lib/libgfortran* + zip $1/*.whl */lib/libgfortran* + mkdir -p /output + # copy libs/openblas*.tar.gz to dist/ + cp libs/openblas*.tar.gz /output/ +fi diff --git a/ci-test.sh b/ci-test.sh new file mode 100755 index 0000000..1a55363 --- /dev/null +++ b/ci-test.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Test that the wheel works with a different python +set -xe + +if [ "${PLAT}" == "arm64" ]; then + # Cannot test + exit 0 +fi + +PYTHON=python3.9 +if [ "$(uname)" == "Darwin" -a "${PLAT}" == "x86_64" ]; then + which python3.9 + PYTHON="arch -x86_64 python3.9" +fi +if [ "${INTERFACE64}" != "1" ]; then + # cibuildwheel will install the wheel automatically + # $PYTHON -m pip install --no-index --find-links /tmp/cibuildwheel/repaired_wheel scipy_openblas32 + $PYTHON -m scipy_openblas32 +else + # $PYTHON -m pip install --no-index --find-links /tmp/cibuildwheel/repaired_wheel scipy_openblas64 + $PYTHON -m scipy_openblas64 +fi diff --git a/gfortran-install b/gfortran-install deleted file mode 160000 index 3dd38d9..0000000 --- a/gfortran-install +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3dd38d9ce78b3890598cb0eff18a7bec50c06f5e diff --git a/multibuild b/multibuild deleted file mode 160000 index 24f1446..0000000 --- a/multibuild +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 24f1446f6477893b2ef9d721aabcee7a494ca2b1 diff --git a/pyproject.toml b/pyproject.toml index 90bc7f9..95988b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "scipy-openblas64" # v0.3.30 -version = "0.3.30.0.2" +version = "0.3.30.0.3" requires-python = ">=3.7" description = "Provides OpenBLAS for python packaging" readme = "README.md" @@ -39,3 +39,20 @@ install_requires = "importlib-metadata ~= 1.0 ; python_version < '3.8'" [tool.setuptools.package-data] scipy_openblas64 = ["lib/*", "include/*", "lib/pkgconfig/*", "lib/cmake/openblas/*"] + +[tool.cibuildwheel] +before-build = "bash ci-before-build.sh" +repair-wheel-command = "bash ci-repair-wheel.sh {dest_dir} {wheel}" +test-command = "cd {package} && bash ci-test.sh " +environment-pass = [ + "REPO_DIR", + "OPENBLAS_COMMIT", + "MACOSX_DEPLOYMENT_TARGET", + "NIGHTLY", + "MB_ML_LIBC", + "MB_ML_VER", + "INTERFACE64", + "BUILD_DIR", + "PLAT", + "OS-NAME", +] diff --git a/setup.py b/setup.py index 6068493..cd0a2c0 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,19 @@ from setuptools import setup +from setuptools.dist import Distribution +from wheel.bdist_wheel import bdist_wheel as _bdist_wheel -setup() + +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + + +class bdist_wheel(_bdist_wheel): + def get_tag(self): + return "py3", "none", _bdist_wheel.get_tag(self)[2] + + +setup( + distclass=BinaryDistribution, + cmdclass={"bdist_wheel": bdist_wheel}, +) diff --git a/tools/build_prepare.sh b/tools/build_prepare.sh new file mode 100644 index 0000000..cb99d44 --- /dev/null +++ b/tools/build_prepare.sh @@ -0,0 +1,72 @@ +#! /bin/bash + +set -xe + +ls libs/openblas* >/dev/null 2>&1 && true +if [ "$?" != "0" ]; then + # inside docker + cd /project +fi +PYTHON=${PYTHON:-python3.9} + +mkdir -p local/openblas +mkdir -p dist +$PYTHON -m pip install wheel auditwheel + +# This will fail if there is more than one file in libs +tar -C local/scipy_openblas64 --strip-components=2 -xf libs/openblas*.tar.gz + +# do not package the static libs and symlinks, only take the shared object +find local/scipy_openblas64/lib -maxdepth 1 -type l -delete +rm local/scipy_openblas64/lib/*.a +# Check that the pyproject.toml and the pkgconfig versions agree. +py_version=$(grep "^version" pyproject.toml | sed -e "s/version = \"//") +pkg_version=$(grep "version=" ./local/scipy_openblas64/lib/pkgconfig/scipy-openblas*.pc | sed -e "s/version=//" | sed -e "s/dev//") +if [[ -z "$pkg_version" ]]; then + echo Could not read version from pkgconfig file + exit 1 +fi +if [[ $py_version != $pkg_version* ]]; then + echo Version from pyproject.toml "$py_version" does not match version from build "pkg_version" + exit 1 +fi + +if [ $(uname) == "Darwin" ]; then + soname=$(cd local/scipy_openblas64/lib; ls libscipy_openblas*.dylib) + echo otool -D local/scipy_openblas64/lib/$soname + otool -D local/scipy_openblas64/lib/$soname + # issue 153: there is a ".0" in the install_name. Remove it + # also add a @rpath + install_name_tool -id @rpath/$soname local/scipy_openblas64/lib/$soname +fi + +rm -rf local/scipy_openblas64/lib/pkgconfig +echo "" >> LICENSE.txt +echo "----" >> LICENSE.txt +echo "" >> LICENSE.txt +if [ $(uname) == "Darwin" ]; then + cat tools/LICENSE_osx.txt >> LICENSE.txt +else + cat tools/LICENSE_linux.txt >> LICENSE.txt +fi + +if [ "${INTERFACE64}" != "1" ]; then + # rewrite the name of the project to scipy-openblas32 + # this is a hack, but apparently there is no other way to change the name + # of a pyproject.toml project + # + # use the BSD variant of sed -i and remove the backup + sed -e "s/openblas64/openblas32/" -i.bak pyproject.toml + rm *.bak + mv local/scipy_openblas64 local/scipy_openblas32 + sed -e "s/openblas_get_config64_/openblas_get_config/" -i.bak local/scipy_openblas32/__init__.py + sed -e "s/cflags =.*/cflags = '-DBLAS_SYMBOL_PREFIX=scipy_'/" -i.bak local/scipy_openblas32/__init__.py + sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__main__.py + sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__init__.py + rm local/scipy_openblas32/*.bak +fi + +rm -rf dist/* + + +echo "The build preparation is done." \ No newline at end of file diff --git a/tools/build_steps.sh b/tools/build_steps.sh index f87758b..1262e1d 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -2,10 +2,27 @@ BUILD_PREFIX=/usr/local ROOT_DIR=$(dirname $(dirname "${BASH_SOURCE[0]}")) -source ${ROOT_DIR}/multibuild/common_utils.sh MB_PYTHON_VERSION=3.9 +function any_python { + for cmd in $PYTHON_EXE python3 python; do + if [ -n "$(type -t $cmd)" ]; then + echo $cmd + return + fi + done + echo "Could not find python or python3" + exit 1 +} + +function get_os { + # Report OS as given by uname + # Use any Python that comes to hand. + $(any_python) -c 'import platform; print(platform.uname()[0])' +} + + function before_build { # Manylinux Python version set in build_lib if [ -n "$IS_OSX" ]; then @@ -19,14 +36,10 @@ function before_build { sudo chmod 777 /usr/local/include touch /usr/local/include/.dir_exists fi - source ${ROOT_DIR}/multibuild/osx_utils.sh - get_macpython_environment ${MB_PYTHON_VERSION} venv - # Since install_fortran uses `uname -a` to determine arch, - # force the architecture - arch -${PLAT} bash -s << EOF -source ${ROOT_DIR}/gfortran-install/gfortran_utils.sh -install_gfortran -EOF + # get_macpython_environment ${MB_PYTHON_VERSION} venv + python3.9 -m venv venv + source venv/bin/activate + alias gfortran=gfortran-15 # Deployment target set by gfortran_utils echo "Deployment target $MACOSX_DEPLOYMENT_TARGET" @@ -42,12 +55,6 @@ function clean_code_local { local build_commit=${2:-$BUILD_COMMIT} [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 - # The package $repo_dir may be a submodule. git submodules do not - # have a .git directory. If $repo_dir is copied around, tools like - # Versioneer which require that it be a git repository are unable - # to determine the version. Give submodule proper git directory - # XXX no need to do this - # fill_submodule "$repo_dir" pushd $repo_dir echo in $repo_dir git fetch origin --tags @@ -106,27 +113,12 @@ function build_lib { local interface64=${2:-$INTERFACE64} local nightly=${3:0} local manylinux=${MB_ML_VER:-1} - # Make directory to store built archive if [ -n "$IS_OSX" ]; then # Do build, add gfortran hash to end of name do_build_lib "$plat" "gf_${GFORTRAN_SHA:0:7}" "$interface64" "$nightly" - return + else + do_build_lib "$plat" "" "$interface64" "$nightly" fi - # Manylinux wrapper - local libc=${MB_ML_LIBC:-manylinux} - local docker_image=quay.io/pypa/${libc}${manylinux}_${plat} - docker pull $docker_image - # Docker sources this script, and runs `do_build_lib` - docker run --rm \ - -e BUILD_PREFIX="$BUILD_PREFIX" \ - -e PLAT="${plat}" \ - -e INTERFACE64="${interface64}" \ - -e NIGHTLY="${nightly}" \ - -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ - -e MB_ML_VER=${manylinux} \ - -e MB_ML_LIBC=${libc} \ - -v $PWD:/io \ - $docker_image /io/tools/docker_build_wrap.sh } function patch_source { @@ -269,12 +261,49 @@ function do_build_lib { $BUILD_PREFIX/lib/cmake/openblas } + +function build_lib_on_travis { + # OSX or manylinux build + # + # Input arg + # plat - one of i686, x86_64, arm64 + # interface64 - 1 if build with INTERFACE64 and SYMBOLSUFFIX + # nightly - 1 if building for nightlies + # + # Depends on globals + # BUILD_PREFIX - install suffix e.g. "/usr/local" + # MB_ML_VER + set -x + local plat=${1:-$PLAT} + local interface64=${2:-$INTERFACE64} + local nightly=${3:0} + local manylinux=${MB_ML_VER:-1} + + # Manylinux wrapper + local libc=${MB_ML_LIBC:-manylinux} + local docker_image=quay.io/pypa/${libc}${manylinux}_${plat} + docker pull $docker_image + # run `do_build_lib` in the docker image + docker run --rm \ + -e BUILD_PREFIX="$BUILD_PREFIX" \ + -e PLAT="${plat}" \ + -e INTERFACE64="${interface64}" \ + -e NIGHTLY="${nightly}" \ + -e PYTHON_VERSION="$MB_PYTHON_VERSION" \ + -e MB_ML_VER=${manylinux} \ + -e MB_ML_LIBC=${libc} \ + -v $PWD:/io \ + $docker_image /io/tools/docker_build_wrap.sh +} + + + function build_on_travis { if [ ${TRAVIS_EVENT_TYPE} == "cron" ]; then - build_lib "$PLAT" "$INTERFACE64" 1 + build_lib_on_travis "$PLAT" "$INTERFACE64" 1 version=$(cd OpenBLAS && git describe --tags --abbrev=8 | sed -e "s/^v\(.*\)-g.*/\1/" | sed -e "s/-/./g") sed -e "s/^version = .*/version = \"${version}\"/" -i.bak pyproject.toml else - build_lib "$PLAT" "$INTERFACE64" 0 + build_lib_on_travis "$PLAT" "$INTERFACE64" 0 fi } diff --git a/tools/build_wheel.sh b/tools/build_wheel.sh index 776a3ae..394dcb6 100644 --- a/tools/build_wheel.sh +++ b/tools/build_wheel.sh @@ -1,111 +1,48 @@ +#! /bin/bash # Needs: # $INTERFACE64 ("1" or "0") # $PLAT (x86_64, i686, arm64, aarch64, s390x, ppc64le) +# The code below is for Travis use only. set -xe -ls libs/openblas* >/dev/null 2>&1 && true -if [ "$?" != "0" ]; then - # inside docker +if [[ ! -e tools/build_prepare.sh ]];then cd /openblas fi -PYTHON=${PYTHON:-python3.9} -mkdir -p local/openblas -mkdir -p dist -$PYTHON -m pip install wheel auditwheel +source tools/build_prepare.sh -# This will fail if there is more than one file in libs -tar -C local/scipy_openblas64 --strip-components=2 -xf libs/openblas*.tar.gz +$PYTHON -m pip wheel -w dist -v . -# do not package the static libs and symlinks, only take the shared object -find local/scipy_openblas64/lib -maxdepth 1 -type l -delete -rm local/scipy_openblas64/lib/*.a -# Check that the pyproject.toml and the pkgconfig versions agree. -py_version=$(grep "^version" pyproject.toml | sed -e "s/version = \"//") -pkg_version=$(grep "version=" ./local/scipy_openblas64/lib/pkgconfig/scipy-openblas*.pc | sed -e "s/version=//" | sed -e "s/dev//") -if [[ -z "$pkg_version" ]]; then - echo Could not read version from pkgconfig file - exit 1 -fi -if [[ $py_version != $pkg_version* ]]; then - echo Version from pyproject.toml "$py_version" does not match version from build "pkg_version" - exit 1 -fi +echo "Repairing wheel with auditwheel" +auditwheel repair -w dist --lib-sdir /lib dist/*.whl +echo "Wheel repaired." -if [ $(uname) == "Darwin" ]; then - soname=$(cd local/scipy_openblas64/lib; ls libscipy_openblas*.dylib) - echo otool -D local/scipy_openblas64/lib/$soname - otool -D local/scipy_openblas64/lib/$soname - # issue 153: there is a ".0" in the install_name. Remove it - # also add a @rpath - install_name_tool -id @rpath/$soname local/scipy_openblas64/lib/$soname -fi +rm dist/*none-linux*.whl -rm -rf local/scipy_openblas64/lib/pkgconfig -echo "" >> LICENSE.txt -echo "----" >> LICENSE.txt -echo "" >> LICENSE.txt -if [ $(uname) == "Darwin" ]; then - cat tools/LICENSE_osx.txt >> LICENSE.txt +ls -l dist/ + +# Add an RPATH to libgfortran: +# https://github.com/pypa/auditwheel/issues/451 +if [ "$MB_ML_LIBC" == "musllinux" ]; then + apk add zip else - cat tools/LICENSE_linux.txt >> LICENSE.txt + yum install -y zip fi +unzip dist/*.whl "*libgfortran*" +patchelf --force-rpath --set-rpath '$ORIGIN' */lib/libgfortran* +zip dist/*.whl */lib/libgfortran* -if [ "${INTERFACE64}" != "1" ]; then - # rewrite the name of the project to scipy-openblas32 - # this is a hack, but apparently there is no other way to change the name - # of a pyproject.toml project - # - # use the BSD variant of sed -i and remove the backup - sed -e "s/openblas64/openblas32/" -i.bak pyproject.toml - rm *.bak - mv local/scipy_openblas64 local/scipy_openblas32 - sed -e "s/openblas_get_config64_/openblas_get_config/" -i.bak local/scipy_openblas32/__init__.py - sed -e "s/cflags =.*/cflags = '-DBLAS_SYMBOL_PREFIX=scipy_'/" -i.bak local/scipy_openblas32/__init__.py - sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__main__.py - sed -e "s/openblas64/openblas32/" -i.bak local/scipy_openblas32/__init__.py - rm local/scipy_openblas32/*.bak -fi +echo "Final wheel contents:" -rm -rf dist/* -$PYTHON -m pip wheel -w dist -v . +ls -l dist/ -if [ $(uname) == "Darwin" ]; then - $PYTHON -m pip install delocate - # move the mis-named scipy_openblas64-none-any.whl to a platform-specific name - if [ "${PLAT}" == "arm64" ]; then - for f in dist/*.whl; do mv $f "${f/%any.whl/macosx_11_0_$PLAT.whl}"; done - else - for f in dist/*.whl; do mv $f "${f/%any.whl/macosx_10_9_$PLAT.whl}"; done - fi - delocate-wheel -v dist/*.whl -else - auditwheel repair -w dist --lib-sdir /lib dist/*.whl - rm dist/scipy_openblas*-none-any.whl - # Add an RPATH to libgfortran: - # https://github.com/pypa/auditwheel/issues/451 - if [ "$MB_ML_LIBC" == "musllinux" ]; then - apk add zip - else - yum install -y zip - fi - unzip dist/*.whl "*libgfortran*" - patchelf --force-rpath --set-rpath '$ORIGIN' */lib/libgfortran* - zip dist/*.whl */lib/libgfortran* -fi +echo "Testing the wheel" -if [ "${PLAT}" == "arm64" ]; then - # Cannot test - exit 0 -fi # Test that the wheel works with a different python PYTHON=python3.11 -if [ "$(uname)" == "Darwin" -a "${PLAT}" == "x86_64" ]; then - which python3.11 - PYTHON="arch -x86_64 python3.11" -fi + if [ "${INTERFACE64}" != "1" ]; then $PYTHON -m pip install --no-index --find-links dist scipy_openblas32 $PYTHON -m scipy_openblas32 @@ -113,3 +50,5 @@ else $PYTHON -m pip install --no-index --find-links dist scipy_openblas64 $PYTHON -m scipy_openblas64 fi + +echo "Wheel test successful." \ No newline at end of file diff --git a/tools/docker_build_wrap.sh b/tools/docker_build_wrap.sh index e9f7b4b..20c68cd 100755 --- a/tools/docker_build_wrap.sh +++ b/tools/docker_build_wrap.sh @@ -5,6 +5,8 @@ set -e # Change into root directory of repo -cd /io +if [[ ! -e tools/build_steps.sh ]];then + cd /io +fi source tools/build_steps.sh do_build_lib "$PLAT" "" "$INTERFACE64" "$NIGHTLY" diff --git a/tools/gfortran_utils.sh b/tools/gfortran_utils.sh new file mode 100644 index 0000000..b7127f5 --- /dev/null +++ b/tools/gfortran_utils.sh @@ -0,0 +1,183 @@ +# this file come from https://github.com/MacPython/gfortran-install +# Follow the license below + + +# gfortran-install license +# Copyright 2016-2021 Matthew Brett, Isuru Fernando, Matti Picus + +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Bash utilities for use with gfortran + +ARCHIVE_SDIR="${ARCHIVE_SDIR:-archives}" + +GF_UTIL_DIR=$(dirname "${BASH_SOURCE[0]}") + +function get_distutils_platform { + # Report platform as in form of distutils get_platform. + # This is like the platform tag that pip will use. + # Modify fat architecture tags on macOS to reflect compiled architecture + + # Deprecate this function once get_distutils_platform_ex is used in all + # downstream projects + local plat=$1 + case $plat in + i686|x86_64|arm64|universal2|intel|aarch64|s390x|ppc64le) ;; + *) echo Did not recognize plat $plat; return 1 ;; + esac + local uname=${2:-$(uname)} + if [ "$uname" != "Darwin" ]; then + if [ "$plat" == "intel" ]; then + echo plat=intel not allowed for Manylinux + return 1 + fi + echo "manylinux1_$plat" + return + fi + # macOS 32-bit arch is i386 + [ "$plat" == "i686" ] && plat="i386" + local target=$(echo $MACOSX_DEPLOYMENT_TARGET | tr .- _) + echo "macosx_${target}_${plat}" +} + +function get_distutils_platform_ex { + # Report platform as in form of distutils get_platform. + # This is like the platform tag that pip will use. + # Modify fat architecture tags on macOS to reflect compiled architecture + # For non-darwin, report manylinux version + local plat=$1 + local mb_ml_ver=${MB_ML_VER:-1} + case $plat in + i686|x86_64|arm64|universal2|intel|aarch64|s390x|ppc64le) ;; + *) echo Did not recognize plat $plat; return 1 ;; + esac + local uname=${2:-$(uname)} + if [ "$uname" != "Darwin" ]; then + if [ "$plat" == "intel" ]; then + echo plat=intel not allowed for Manylinux + return 1 + fi + echo "manylinux${mb_ml_ver}_${plat}" + return + fi + # macOS 32-bit arch is i386 + [ "$plat" == "i686" ] && plat="i386" + local target=$(echo $MACOSX_DEPLOYMENT_TARGET | tr .- _) + echo "macosx_${target}_${plat}" +} + +function get_macosx_target { + # Report MACOSX_DEPLOYMENT_TARGET as given by distutils get_platform. + python3 -c "import sysconfig as s; print(s.get_config_vars()['MACOSX_DEPLOYMENT_TARGET'])" +} + +function check_gfortran { + # Check that gfortran exists on the path + if [ -z "$(which gfortran)" ]; then + echo Missing gfortran + exit 1 + fi +} + +function get_gf_lib_for_suf { + local suffix=$1 + local prefix=$2 + local plat=${3:-$PLAT} + local uname=${4:-$(uname)} + if [ -z "$prefix" ]; then echo Prefix not defined; exit 1; fi + local plat_tag=$(get_distutils_platform_ex $plat $uname) + if [ -n "$suffix" ]; then suffix="-$suffix"; fi + local fname="$prefix-${plat_tag}${suffix}.tar.gz" + local out_fname="${ARCHIVE_SDIR}/$fname" + [ -s $out_fname ] || (echo "$out_fname is empty"; exit 24) + echo "$out_fname" +} + +if [ "$(uname)" == "Darwin" ]; then + mac_target=${MACOSX_DEPLOYMENT_TARGET:-$(get_macosx_target)} + export MACOSX_DEPLOYMENT_TARGET=$mac_target + # Keep this for now as some builds might depend on this being + # available before install_gfortran is called + export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e + # Set SDKROOT env variable if not set + export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)} + + function download_and_unpack_gfortran { + local arch=$1 + local type=$2 + curl -L -O https://github.com/isuruf/gcc/releases/download/gcc-11.3.0-2/gfortran-darwin-${arch}-${type}.tar.gz + case ${arch}-${type} in + arm64-native) + export GFORTRAN_SHA=0d5c118e5966d0fb9e7ddb49321f63cac1397ce8 + ;; + arm64-cross) + export GFORTRAN_SHA=527232845abc5af21f21ceacc46fb19c190fe804 + ;; + x86_64-native) + export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e + ;; + x86_64-cross) + export GFORTRAN_SHA=107604e57db97a0ae3e7ca7f5dd722959752f0b3 + ;; + esac + if [[ "$(shasum gfortran-darwin-${arch}-${type}.tar.gz)" != "${GFORTRAN_SHA} gfortran-darwin-${arch}-${type}.tar.gz" ]]; then + echo "shasum mismatch for gfortran-darwin-${arch}-${type}" + exit 1 + fi + sudo mkdir -p /opt/ + sudo cp "gfortran-darwin-${arch}-${type}.tar.gz" /opt/gfortran-darwin-${arch}-${type}.tar.gz + pushd /opt + sudo tar -xvf gfortran-darwin-${arch}-${type}.tar.gz + sudo rm gfortran-darwin-${arch}-${type}.tar.gz + popd + if [[ "${type}" == "native" ]]; then + # Link these into /usr/local so that there's no need to add rpath or -L + for f in libgfortran.dylib libgfortran.5.dylib libgcc_s.1.dylib libgcc_s.1.1.dylib libquadmath.dylib libquadmath.0.dylib; do + ln -sf /opt/gfortran-darwin-${arch}-${type}/lib/$f /usr/local/lib/$f + done + # Add it to PATH + ln -sf /opt/gfortran-darwin-${arch}-${type}/bin/gfortran /usr/local/bin/gfortran + fi + } + + function install_arm64_cross_gfortran { + download_and_unpack_gfortran arm64 cross + export FC_ARM64="$(find /opt/gfortran-darwin-arm64-cross/bin -name "*-gfortran")" + local libgfortran="$(find /opt/gfortran-darwin-arm64-cross/lib -name libgfortran.dylib)" + local libdir=$(dirname $libgfortran) + + export FC_ARM64_LDFLAGS="-L$libdir -Wl,-rpath,$libdir" + if [[ "${PLAT:-}" == "arm64" ]]; then + export FC=$FC_ARM64 + fi + } + function install_gfortran { + download_and_unpack_gfortran $(uname -m) native + check_gfortran + if [[ "${PLAT:-}" == "universal2" || "${PLAT:-}" == "arm64" ]]; then + install_arm64_cross_gfortran + fi + } + + function get_gf_lib { + # Get lib with gfortran suffix + get_gf_lib_for_suf "gf_${GFORTRAN_SHA:0:7}" $@ + } +else + function install_gfortran { + # No-op - already installed on manylinux image + check_gfortran + } + + function get_gf_lib { + # Get library with no suffix + get_gf_lib_for_suf "" $@ + } +fi