Build platform-specific wheels containing libmagic

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"

.github/workflows/ci.yml

@@ -1,5 +1,8 @@
 name: ci
 on: [push, pull_request]
+concurrency:  # https://stackoverflow.com/questions/66335225#comment133398800_72408109
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 jobs:
   ci:
     strategy:
@@ -19,13 +22,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
+      - name: Copy libmagic into magic dir
+        run: ${{ (runner.os == 'Windows' && 'bash add_libmagic.sh')  || 'sudo -E bash add_libmagic.sh' }}
       - run: pip install --upgrade pip
       - run: pip install --upgrade pytest
       - run: pip install --editable .
-      - if: runner.os == 'macOS'
-        run: brew install libmagic
-      - if: runner.os == 'Windows'
-        run: pip install python-magic-bin
       - run: LC_ALL=en_US.UTF-8 pytest
         shell: bash
         timeout-minutes: 15 # Limit Windows infinite loop.

.github/workflows/wheels.yml

@@ -0,0 +1,180 @@
+name: wheels
+
+on:
+  pull_request:
+  push:
+    branches: master
+  release:
+    types: [released, prereleased]
+  workflow_dispatch:  # allows running workflow manually from the Actions tab
+
+concurrency:  # https://stackoverflow.com/questions/66335225#comment133398800_72408109
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  build-sdist:
+    runs-on: ubuntu-latest
+
+    env:
+      PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+
+    - run: sudo apt-get install -y libmagic1
+
+    - name: Build source distribution
+      run: |
+        pip install --upgrade setuptools wheel pip build
+        python -m build --sdist
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: dist
+        path: dist/*.tar.*
+
+
+  build-wheels-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      include: ${{ steps.set-matrix.outputs.include }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - run: pip install cibuildwheel==2.17.0  # sync version with pypa/cibuildwheel below
+      - id: set-matrix
+        env:
+          # only mention one (trivial) python version, as py2.py3 wheels only need to be build once per arch
+          CIBW_PROJECT_REQUIRES_PYTHON: '==3.12.*'
+          # skip PyPy wheels for now, and skip i686 wheels because pytest is failing
+          CIBW_SKIP: pp* *i686
+        run: |
+          MATRIX_INCLUDE=$(
+            {
+              cibuildwheel --print-build-identifiers --platform linux --arch all | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 |  jq -nRc '{"only": inputs, "os": "macos-13"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch arm64 |  jq -nRc '{"only": inputs, "os": "macos-14"}' \
+              && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 |  jq -nRc '{"only": inputs, "os": "windows-latest"}'
+            } | jq -sc
+          )
+          echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
+
+
+  build-wheels:
+    name: build ${{ matrix.only }}
+    needs: build-wheels-matrix
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
+
+    # For Windows, grabbing latest file from MSYS2 is easier than building from source
+    # It's generally up to date ref https://packages.msys2.org/base/mingw-w64-file
+    - name: Setup MSYS2 and install file
+      if: runner.os == 'Windows'
+      uses: msys2/[email protected]
+      with:
+        msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}
+        location: D:\
+        install: >-
+          ${{ endsWith(matrix.only, '32') && 'mingw-w64-i686-file' || 'mingw-w64-x86_64-file' }}
+
+    # The DLL dependency tree flattened out ref "Dependencies" https://packages.msys2.org/packages/mingw-w64-x86_64-file
+    - name: Copy Windows ddl and mgc
+      if: runner.os == 'Windows'
+      run: |
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/share/misc/magic.mgc" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libmagic-1.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libsystre-0.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libtre-5.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libasprintf-0.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libintl-8.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libatomic-1.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libgomp-1.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libquadmath-0.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libstdc++-6.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libcharset-1.dll" "magic"
+        cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libiconv-2.dll" "magic"
+
+    # These are needed additionally in the win32 wheel ref https://packages.msys2.org/packages/mingw-w64-i686-file
+    - name: Copy additional 32-bit runtime DLLs
+      if: runner.os == 'Windows' && endsWith(matrix.only, '32')
+      run: |
+        cp "/msys64/mingw32/bin/libgcc_s_dw2-1.dll" "magic"
+        cp "/msys64/mingw32/bin/libwinpthread-1.dll" "magic"
+
+    - uses: pypa/[email protected]  # sync version with pip install cibuildwheel above
+      timeout-minutes: 10
+      with:
+        only: ${{ matrix.only }}
+      env:
+        CIBW_BUILD_VERBOSITY: 1
+        # add compiled libmagic to the build directory (to include in the wheel)
+        CIBW_BEFORE_BUILD_MACOS: sudo -E bash add_libmagic.sh
+        CIBW_BEFORE_BUILD_LINUX: bash add_libmagic.sh
+        # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag)
+        MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }}
+        # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic
+        CIBW_TEST_COMMAND: python -c "import magic; assert magic.Magic(mime=True).from_buffer(b'\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00') == 'video/mp4'"
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: dist-${{ matrix.only }}
+        path: wheelhouse/*.whl
+
+
+  publish:
+    if: github.event_name == 'release'
+    needs: [build-sdist, build-wheels]
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # softprops/action-gh-release
+      id-token: write  # pypa/gh-action-pypi-publish
+
+    steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+
+    - uses: actions/download-artifact@v4
+      with:
+        path: dist/
+        pattern: dist-*
+        merge-multiple: true
+
+    - run: ls -ltra dist/
+
+    - run: pip install --upgrade python-magic --find-links ./dist
+
+    - name: Smoketest
+      run: python -c "import magic; magic.Magic()"
+
+    - name: Upload release assets
+      uses: softprops/[email protected]
+      with:
+        files: dist/*
+
+    - name: Publish package distributions to PyPI
+      uses: pypa/[email protected]

CHANGELOG

@@ -1,10 +1,14 @@
-Changes to 0.4.29:
+Changes to 0.4.28:
 
+- libmagic and magic.mgc now come bundled in the wheels on PyPI, and will be copied
+  into site-packages/magic along with the Python files of this library
+- magic.loader.load_lib now first searches for libmagic in the same directory as the
+  Python files, then in the current working directory, and only then in standard paths
+- magic.Magic(magic_file=...) and magic.compat.Magic.load(magic_file=...) will now
+  prefer "magic.mgc" in the same directory as the Python files, only if left
+  unspecified by the user (and the MAGIC env var is empty or not set)
 - support MAGIC_SYMLINK (via follow_symlink flag on Magic constructor)
 - correctly throw FileNotFoundException depending on flag
-
-Changes to 0.4.28:
-
 - support "magic-1.dll" on Windows, which is produced by vcpkg
 - add python 3.10 to tox config
 - update test for upstream gzip extensions

README.md

@@ -3,11 +3,11 @@
 [![ci](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml)
 [![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-python-magic is a Python interface to the libmagic file type
-identification library.  libmagic identifies file types by checking
+[python-magic](https://github.com/ahupp/python-magic) is a Python interface to the libmagic file type
+identification library. libmagic identifies file types by checking
 their headers according to a predefined list of file types. This
 functionality is exposed to the command line by the Unix command
-`file`.
+[`file`](https://www.darwinsys.com/file/).
 
 ## Usage
 
@@ -31,8 +31,7 @@ will fail throw if this is attempted.
 ```python
 >>> f = magic.Magic(uncompress=True)
 >>> f.from_file('testdata/test.gz')
-'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28
-21:32:52 2008, from Unix)'
+'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28 21:32:52 2008, from Unix)'
 ```
 
 You can also combine the flag options:
@@ -45,27 +44,53 @@ You can also combine the flag options:
 
 ## Installation
 
-The current stable version of python-magic is available on PyPI and
-can be installed by running `pip install python-magic`.
+This module is a simple [CDLL](https://docs.python.org/3/library/ctypes.html) wrapper around the libmagic C library.
+The current stable version of python-magic is available on [PyPI](http://pypi.python.org/pypi/python-magic/)
+and can be installed by running `pip install python-magic`.
 
-Other sources:
+Compiled libmagic and the magic database come bundled in the wheels on PyPI.
+You can use your own `magic.mgc` database by setting the `MAGIC`
+environment variable, or by using `magic.Magic(magic_file='path/to/magic.mgc')`.
+If you want to compile your own libmagic, circumvent the wheels
+by installing from source: `pip install python-magic --no-binary python-magic`.
 
-- PyPI: http://pypi.python.org/pypi/python-magic/
-- GitHub: https://github.com/ahupp/python-magic
+For systems not supported by the wheels, pip installs from source,
+requiring libmagic to be available before installing python-magic:
 
-This module is a simple wrapper around the libmagic C library, and
-that must be installed as well:
+### Linux
 
-### Debian/Ubuntu
+The Linux wheels should run on most systems out of the box.
 
+Depending on your system and CPU architecture, there might be no compatible wheel uploaded.
+However, precompiled libmagic might still be available for your system:
+
+```sh
+# Debian/Ubuntu
+apt-get update && apt-get install -y libmagic1
+# Alpine
+apk add --update libmagic
+# RHEL
+dnf install file-libs
 ```
-sudo apt-get install libmagic1
-```
+
+### Windows
+
+The DLLs that are bundled in the Windows wheels are compiled by @julian-r
+and are hosted at https://github.com/julian-r/file-windows/releases.
+
+For ARM64 Windows, you'll need to compile libmagic from source.
 
 ### OSX
 
-- When using Homebrew: `brew install libmagic`
-- When using macports: `port install file`
+The Mac wheels are compiled with maximum backward compatibility.
+For older Macs, you'll need to install libmagic from source:
+
+```sh
+# homebrew
+brew install libmagic
+# macports
+port install file
+```
 
 If python-magic fails to load the library it may be in a non-standard location, in which case you can set the environment variable `DYLD_LIBRARY_PATH` to point to it.
 
@@ -78,7 +103,7 @@ If python-magic fails to load the library it may be in a non-standard location,
 - 'MagicException: could not find any magic files!': some
   installations of libmagic do not correctly point to their magic
   database file.  Try specifying the path to the file explicitly in the
-  constructor: `magic.Magic(magic_file="path_to_magic_file")`.
+  constructor: `magic.Magic(magic_file='path/to/magic.mgc')`.
 
 - 'WindowsError: [Error 193] %1 is not a valid Win32 application':
   Attempting to run the 32-bit libmagic DLL in a 64-bit build of
@@ -88,7 +113,6 @@ If python-magic fails to load the library it may be in a non-standard location,
 - 'WindowsError: exception: access violation writing 0x00000000 ' This may indicate you are mixing
   Windows Python and Cygwin Python. Make sure your libmagic and python builds are consistent.
 
-
 ## Bug Reports
 
 python-magic is a thin layer over the libmagic C library.