Skip to content

Commit a98f13b

Browse files
ddelangeddelange
committed
Build platform-specific wheels containing libmagic
1 parent 2a01b18 commit a98f13b

File tree

4 files changed

+208
-34
lines changed

4 files changed

+208
-34
lines changed

‎.github/workflows/main.yml

+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: GH
2+
3+
permissions:
4+
contents: write
5+
6+
on:
7+
pull_request:
8+
push:
9+
branches: master
10+
release:
11+
types: [released, prereleased]
12+
workflow_dispatch: # allows running workflow manually from the Actions tab
13+
14+
jobs:
15+
16+
build-sdist:
17+
runs-on: ubuntu-latest
18+
19+
env:
20+
PIP_DISABLE_PIP_VERSION_CHECK: 1
21+
22+
steps:
23+
- uses: actions/checkout@v3
24+
with:
25+
fetch-depth: 0
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: '3.x'
31+
32+
- run: sudo apt-get install -y libmagic1
33+
34+
- name: Build source distribution
35+
run: |
36+
pip install -U setuptools wheel pip
37+
python setup.py sdist
38+
39+
- uses: actions/upload-artifact@v3
40+
with:
41+
name: dist
42+
path: dist/*.tar.*
43+
44+
45+
build-wheels-matrix:
46+
runs-on: ubuntu-latest
47+
outputs:
48+
include: ${{ steps.set-matrix.outputs.include }}
49+
steps:
50+
- uses: actions/checkout@v3
51+
- uses: actions/setup-python@v4
52+
with:
53+
python-version: '3.x'
54+
- run: pip install cibuildwheel==2.15.0
55+
- id: set-matrix
56+
env:
57+
CIBW_PROJECT_REQUIRES_PYTHON: '==3.8.*'
58+
run: |
59+
MATRIX_INCLUDE=$(
60+
{
61+
cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \
62+
&& cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-11"}' \
63+
&& cibuildwheel --print-build-identifiers --platform macos --arch arm64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-11"}' \
64+
&& cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp | jq -nRc '{"only": inputs, "os": "windows-latest"}'
65+
} | jq -sc
66+
)
67+
echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
68+
69+
70+
build-wheels:
71+
needs: build-wheels-matrix
72+
runs-on: ${{ matrix.os }}
73+
name: Build ${{ matrix.only }}
74+
75+
strategy:
76+
fail-fast: false
77+
matrix:
78+
include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}
79+
80+
steps:
81+
- uses: actions/checkout@v3
82+
with:
83+
fetch-depth: 0
84+
85+
- name: Set up QEMU
86+
if: runner.os == 'Linux'
87+
uses: docker/setup-qemu-action@v2
88+
89+
- uses: pypa/[email protected]
90+
timeout-minutes: 10
91+
with:
92+
only: ${{ matrix.only }}
93+
env:
94+
CIBW_BUILD_VERBOSITY: 1
95+
CIBW_BEFORE_BUILD: 'bash -c "make install_libmagic"'
96+
97+
- uses: actions/upload-artifact@v3
98+
with:
99+
name: dist
100+
path: wheelhouse/*.whl
101+
102+
103+
publish:
104+
needs: [build-sdist, build-wheels]
105+
if: github.event_name == 'release'
106+
runs-on: ubuntu-latest
107+
108+
steps:
109+
- uses: actions/download-artifact@v3
110+
with:
111+
name: dist
112+
path: dist/
113+
114+
- run: ls -ltra dist/
115+
116+
- name: Upload release assets
117+
uses: softprops/[email protected]
118+
with:
119+
files: dist/*
120+
121+
- name: Upload to PyPI
122+
uses: pypa/gh-action-pypi-publish@v1
123+
with:
124+
user: __token__
125+
password: ${{ secrets.PYPI_TOKEN }}

‎Makefile

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
SHELL := /bin/bash
2+
3+
.PHONY: install_libmagic
4+
## Install libmagic
5+
install_libmagic:
6+
# Debian https://packages.ubuntu.com/libmagic1
7+
# RHEL https://git.almalinux.org/rpms/file
8+
# Mac https://formulae.brew.sh/formula/libmagic
9+
# Windows https://github.com/julian-r/file-windows
10+
( ( ( brew install libmagic || ( apt-get update && apt-get install -y libmagic1 ) ) || apk add --update libmagic ) || yum install file-libs ) || ( python -c 'import platform, sysconfig, io, zipfile, urllib.request; assert platform.system() == "Windows"; machine = "x86" if sysconfig.get_platform() == "win32" else "x64"; print(machine); zipfile.ZipFile(io.BytesIO(urllib.request.urlopen(f"https://github.com/julian-r/file-windows/releases/download/v5.44/file_5.44-build104-vs2022-{machine}.zip").read())).extractall(".")' && ls )
11+
# on cibuildwheel, the lib needs to exist in the project before running setup.py
12+
python -c "import subprocess; from magic.loader import load_lib; lib = load_lib()._name; print(f'linking {lib}'); subprocess.check_call(['cp', lib, 'magic'])"
13+
ls magic
14+
15+
.DEFAULT_GOAL := help
16+
.PHONY: help
17+
## Print Makefile documentation
18+
help:
19+
@perl -0 -nle 'printf("\033[36m %-15s\033[0m %s\n", "$$2", "$$1") while m/^##\s*([^\r\n]+)\n^([\w.-]+):[^=]/gm' $(MAKEFILE_LIST) | sort

‎magic/loader.py

+36-31
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,55 @@
11
from ctypes.util import find_library
22
import ctypes
3-
import sys
43
import glob
54
import os.path
5+
import subprocess
6+
import sys
67

78
def _lib_candidates():
89

9-
yield find_library('magic')
10+
if sys.platform == 'darwin':
1011

11-
if sys.platform == 'darwin':
12+
paths = [
13+
'/opt/local/lib',
14+
'/usr/local/lib',
15+
'/opt/homebrew/lib',
16+
] + glob.glob('/usr/local/Cellar/libmagic/*/lib')
1217

13-
paths = [
14-
'/opt/local/lib',
15-
'/usr/local/lib',
16-
'/opt/homebrew/lib',
17-
] + glob.glob('/usr/local/Cellar/libmagic/*/lib')
18+
for i in paths:
19+
yield os.path.join(i, 'libmagic.dylib')
1820

19-
for i in paths:
20-
yield os.path.join(i, 'libmagic.dylib')
21+
elif sys.platform in ('win32', 'cygwin'):
2122

22-
elif sys.platform in ('win32', 'cygwin'):
23+
prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1']
2324

24-
prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1']
25+
for i in prefixes:
26+
# find_library searches in %PATH% but not the current directory,
27+
# so look for both
28+
yield os.path.join('.', '%s.dll' % i)
29+
yield find_library(i)
2530

26-
for i in prefixes:
27-
# find_library searches in %PATH% but not the current directory,
28-
# so look for both
29-
yield './%s.dll' % (i,)
30-
yield find_library(i)
31+
elif sys.platform == 'linux':
32+
# on some linux systems (musl/alpine), find_library('magic') returns None
33+
yield subprocess.check_output(
34+
"( ldconfig -p | grep 'libmagic.so.1' | grep -o '/.*' ) || echo '/usr/lib/libmagic.so.1'",
35+
shell=True,
36+
universal_newlines=True,
37+
).strip()
3138

32-
elif sys.platform == 'linux':
33-
# This is necessary because alpine is bad
34-
yield 'libmagic.so.1'
39+
yield find_library('magic')
3540

3641

3742
def load_lib():
3843

39-
for lib in _lib_candidates():
40-
# find_library returns None when lib not found
41-
if lib is None:
42-
continue
43-
try:
44-
return ctypes.CDLL(lib)
45-
except OSError:
46-
pass
47-
else:
48-
# It is better to raise an ImportError since we are importing magic module
49-
raise ImportError('failed to find libmagic. Check your installation')
44+
for lib in _lib_candidates():
45+
# find_library returns None when lib not found
46+
if lib is None:
47+
continue
48+
try:
49+
return ctypes.CDLL(lib)
50+
except OSError as exc:
51+
pass
52+
else:
53+
# It is better to raise an ImportError since we are importing magic module
54+
raise ImportError('failed to find libmagic. Check your installation')
5055

‎setup.py

+28-3
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,40 @@
44
import setuptools
55
import io
66
import os
7+
import sys
78

9+
# python packages should not install succesfully if libraries are missing
10+
from magic.loader import load_lib
11+
lib = load_lib()._name
812

913
def read(file_name):
1014
"""Read a text file and return the content as a string."""
1115
with io.open(os.path.join(os.path.dirname(__file__), file_name),
1216
encoding='utf-8') as f:
1317
return f.read()
1418

19+
def get_cmdclass():
20+
"""Build a forward compatible ABI3 wheel when `setup.py bdist_wheel` is called."""
21+
if sys.version_info[0] == 2:
22+
return {}
23+
24+
try:
25+
from wheel.bdist_wheel import bdist_wheel
26+
except ImportError:
27+
return {}
28+
29+
class bdist_wheel_abi3(bdist_wheel):
30+
def get_tag(self):
31+
python, abi, _ = super().get_tag()
32+
# get the platform tag based on libmagic included in this wheel
33+
self.root_is_pure = False
34+
_, _, plat = super().get_tag()
35+
return python, abi, plat
36+
37+
return {"bdist_wheel": bdist_wheel_abi3}
38+
39+
cmdclass = get_cmdclass()
40+
1541
setuptools.setup(
1642
name='python-magic',
1743
description='File type identification using libmagic',
@@ -22,9 +48,8 @@ def read(file_name):
2248
long_description=read('README.md'),
2349
long_description_content_type='text/markdown',
2450
packages=['magic'],
25-
package_data={
26-
'magic': ['py.typed', '*.pyi', '**/*.pyi'],
27-
},
51+
package_data={'magic': ['py.typed', '*.pyi', '*.dylib*', '*.dll', '*.so*']},
52+
cmdclass=cmdclass,
2853
keywords="mime magic file",
2954
license="MIT",
3055
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',

0 commit comments

Comments
 (0)