-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Showing
11 changed files
with
323 additions
and
300 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,175 @@ | ||
# SPDX-FileCopyrightText: 2024 geisserml <[email protected]> | ||
# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause | ||
|
||
import sys | ||
import argparse | ||
from pathlib import Path | ||
from functools import partial | ||
|
||
sys.path.insert(0, str(Path(__file__).parents[1]/"setupsrc")) | ||
from pypdfium2_setup.packaging_base import * | ||
from pypdfium2_setup.emplace import prepare_setup | ||
|
||
CondaDir = ProjectDir / "conda" | ||
CondaRaw_BuildNumF = CondaDir / "raw" / "build_num.txt" | ||
|
||
T_RAW = "raw" | ||
T_HELPERS = "helpers" | ||
|
||
|
||
def main(): | ||
parser = argparse.ArgumentParser( | ||
description = "Craft conda packages for pypdfium2" | ||
) | ||
parser.add_argument( | ||
"type", | ||
choices = (T_RAW, T_HELPERS), | ||
help = "The package type to build (raw or helpers)", | ||
) | ||
parser.add_argument("--pdfium-ver", default=None) | ||
parser.add_argument("--new-only", action="store_true") | ||
|
||
args = parser.parse_args() | ||
if args.type == T_RAW: | ||
main_conda_raw(args) | ||
elif args.type == T_HELPERS: | ||
assert not args.new_only, "--new-only / buildnum handling not implemented for helpers package" | ||
main_conda_helpers(args) | ||
else: | ||
assert False # unreached, handled by argparse | ||
|
||
|
||
def _handle_ver(args, get_latest): | ||
if not args.pdfium_ver or args.pdfium_ver == "latest": | ||
args.pdfium_ver = get_latest() | ||
else: | ||
args.pdfium_ver = int(args.pdfium_ver) | ||
|
||
|
||
def main_conda_raw(args): | ||
|
||
_handle_ver(args, CondaPkgVer.get_latest_pdfium) | ||
os.environ["PDFIUM_SHORT"] = str(args.pdfium_ver) | ||
os.environ["PDFIUM_FULL"] = ".".join([str(v) for v in PdfiumVer.to_full(args.pdfium_ver)]) | ||
os.environ["BUILD_NUM"] = str(_get_build_num(args)) | ||
|
||
emplace_func = partial(prepare_setup, ExtPlats.system, args.pdfium_ver, use_v8=None) | ||
with CondaExtPlatfiles(emplace_func): | ||
run_conda_build(CondaDir/"raw", CondaDir/"raw"/"out", args=["--override-channels", "-c", "bblanchon", "-c", "defaults"]) | ||
|
||
|
||
def main_conda_helpers(args): | ||
|
||
_handle_ver(args, CondaPkgVer.get_latest_bindings) | ||
helpers_info = parse_git_tag() | ||
os.environ["M_HELPERS_VER"] = merge_tag(helpers_info, "py") | ||
|
||
# Set the current pdfium version as upper boundary, for inherent API safety. | ||
# pdfium does not do semantic versioning, so upward flexibility is difficult. | ||
os.environ["PDFIUM_MAX"] = str(args.pdfium_ver) | ||
|
||
# NOTE To build with a local pypdfium2_raw, add the args below for the source dir, and remove the pypdfium2-team prefix from the helpers recipe's run requirements | ||
# args=["-c", CondaDir/"raw"/"out"] | ||
run_conda_build(CondaDir/"helpers", CondaDir/"helpers"/"out", args=["--override-channels", "-c", "pypdfium2-team", "-c", "bblanchon", "-c", "defaults"]) | ||
|
||
|
||
def run_conda_build(recipe_dir, out_dir, args=()): | ||
with TmpCommitCtx(): | ||
run_cmd(["conda", "build", recipe_dir, "--output-folder", out_dir, *args], cwd=ProjectDir, env=os.environ) | ||
|
||
|
||
@functools.lru_cache(maxsize=2) | ||
def run_conda_search(package, channel): | ||
output = run_cmd(["conda", "search", "--json", package, "--override-channels", "-c", channel], cwd=None, capture=True) | ||
return json.loads(output)[package] | ||
|
||
|
||
class CondaPkgVer: | ||
|
||
@staticmethod | ||
@functools.lru_cache(maxsize=2) | ||
def _get_latest_for(package, channel, v_func): | ||
search = run_conda_search(package, channel) | ||
search = sorted(search, key=lambda d: v_func(d["version"]), reverse=True) | ||
result = v_func(search[0]["version"]) | ||
print(f"Resolved latest {channel}::{package} to {result}", file=sys.stderr) | ||
return result | ||
|
||
@staticmethod | ||
def get_latest_pdfium(): | ||
return CondaPkgVer._get_latest_for( | ||
"pdfium-binaries", "bblanchon", lambda v: int(v.split(".")[2]) | ||
) | ||
|
||
@staticmethod | ||
def get_latest_bindings(): | ||
return CondaPkgVer._get_latest_for( | ||
"pypdfium2_raw", "pypdfium2-team", lambda v: int(v) | ||
) | ||
|
||
|
||
def _get_build_num(args): | ||
|
||
# parse existing releases to automatically handle arbitrary version builds | ||
# TODO expand to pypdfium2_helpers as well, so we could rebuild with different pdfium bounds in a workflow | ||
|
||
search = reversed(run_conda_search("pypdfium2_raw", "pypdfium2-team")) | ||
|
||
if args.new_only: | ||
# or `args.pdfium_ver not in {...}` to allow new builds of older versions | ||
assert args.pdfium_ver > max(int(d["version"]) for d in search), f"--new-only given, but {args.pdfium_ver} already has a build" | ||
|
||
# determine build number | ||
build_num = max((d["build_number"] for d in search if int(d["version"]) == args.pdfium_ver), default=None) | ||
build_num = 0 if build_num is None else build_num+1 | ||
|
||
return build_num | ||
|
||
|
||
class TmpCommitCtx: | ||
|
||
# Work around local conda `git_url` not including uncommitted changes | ||
# In particular, this is used to transfer data files, so we can generate them externally and don't have to conda package ctypesgen. | ||
|
||
# use a tmp control file so we can also undo the commit in conda's isolated clone | ||
FILE = CondaDir / "with_tmp_commit.txt" | ||
|
||
def __enter__(self): | ||
# determine if there are any modified or new files | ||
out = run_cmd(["git", "status", "--porcelain"], capture=True, cwd=ProjectDir) | ||
self.have_mods = bool(out) | ||
if self.have_mods: # make tmp commit | ||
self.FILE.touch() | ||
run_cmd(["git", "add", "."], cwd=ProjectDir) | ||
run_cmd(["git", "commit", "-m", "!!! tmp commit for conda-build", "-m", "make conda-build include uncommitted changes"], cwd=ProjectDir) | ||
|
||
@classmethod | ||
def undo(cls): | ||
# assuming FILE exists (promised by callers) | ||
run_cmd(["git", "reset", "--soft", "HEAD^"], cwd=ProjectDir) | ||
run_cmd(["git", "reset", cls.FILE], cwd=ProjectDir) | ||
cls.FILE.unlink() | ||
|
||
def __exit__(self, *_): | ||
if self.have_mods: # pop tmp commit, if any | ||
self.undo() | ||
|
||
|
||
class CondaExtPlatfiles: | ||
|
||
def __init__(self, emplace_func): | ||
self.emplace_func = emplace_func | ||
|
||
def __enter__(self): | ||
self.platfiles = self.emplace_func() | ||
self.platfiles = [ModuleDir_Raw/f for f in self.platfiles] | ||
run_cmd(["git", "add", "-f"] + [str(f) for f in self.platfiles], cwd=ProjectDir) | ||
|
||
def __exit__(self, *_): | ||
run_cmd(["git", "reset"] + [str(f) for f in self.platfiles], cwd=ProjectDir) | ||
for fp in self.platfiles: | ||
fp.unlink() | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,131 @@ | ||
# SPDX-FileCopyrightText: 2024 geisserml <[email protected]> | ||
# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause | ||
|
||
import os | ||
import sys | ||
import json | ||
import shutil | ||
import argparse | ||
import tempfile | ||
import contextlib | ||
import urllib.request as url_request | ||
from pathlib import Path | ||
|
||
sys.path.insert(0, str(Path(__file__).parents[1])) | ||
from pypdfium2_setup.packaging_base import * | ||
|
||
try: | ||
import build.__main__ as build_module | ||
except ImportError: | ||
build_module = None | ||
|
||
|
||
def main(): | ||
|
||
parser = argparse.ArgumentParser( | ||
description = "Craft PyPI packages for pypdfium2" | ||
) | ||
parser.add_argument("--pdfium-ver", default=None) | ||
parser.add_argument("--use-v8", action="store_true") | ||
parser.add_argument("--wheels", action="store_true") | ||
parser.add_argument("--sdist", action="store_true") | ||
|
||
args = parser.parse_args() | ||
if not (args.wheels or args.sdist): | ||
args.wheels, args.sdist = True, True | ||
if not args.pdfium_ver or args.pdfium_ver == "latest": | ||
args.pdfium_ver = PdfiumVer.get_latest() | ||
else: | ||
args.pdfium_ver = int(args.pdfium_ver) | ||
|
||
with ArtifactStash(): | ||
main_pypi(args) | ||
|
||
|
||
def main_pypi(args): | ||
|
||
assert args.sdist or args.wheels | ||
|
||
if args.sdist: | ||
os.environ[PlatSpec_EnvVar] = ExtPlats.sdist | ||
helpers_info = get_helpers_info() | ||
with tmp_ctypesgen_pin(): | ||
if not helpers_info["dirty"]: | ||
os.environ["SDIST_IGNORE_DIRTY"] = "1" | ||
_run_pypi_build(["--sdist"]) | ||
|
||
if args.wheels: | ||
suffix = build_pl_suffix(args.pdfium_ver, args.use_v8) | ||
for plat in WheelPlatforms: | ||
os.environ[PlatSpec_EnvVar] = plat + suffix | ||
_run_pypi_build(["--wheel"]) | ||
clean_platfiles() | ||
|
||
|
||
def _run_pypi_build(caller_args): | ||
# -nx: --no-isolation --skip-dependency-check | ||
assert build_module, "Module 'build' is not importable. Cannot craft PyPI packages." | ||
with tmp_cwd_context(ProjectDir): | ||
build_module.main([str(ProjectDir), "-nx", *caller_args]) | ||
|
||
|
||
class ArtifactStash: | ||
|
||
# Preserve in-tree artifacts from editable install | ||
|
||
def __enter__(self): | ||
|
||
self.tmpdir = None | ||
|
||
file_names = [VersionFN, BindingsFN, LibnameForSystem[Host.system]] | ||
self.files = [fp for fp in [ModuleDir_Raw / fn for fn in file_names] if fp.exists()] | ||
if len(self.files) == 0: | ||
return | ||
|
||
self.tmpdir = tempfile.TemporaryDirectory(prefix="pypdfium2_artifact_stash_") | ||
self.tmpdir_path = Path(self.tmpdir.name) | ||
for fp in self.files: | ||
shutil.move(fp, self.tmpdir_path) | ||
|
||
def __exit__(self, *_): | ||
if self.tmpdir is None: | ||
return | ||
for fp in self.files: | ||
shutil.move(self.tmpdir_path / fp.name, ModuleDir_Raw) | ||
self.tmpdir.cleanup() | ||
|
||
|
||
@contextlib.contextmanager | ||
def tmp_replace_ctx(fp, orig, tmp): | ||
orig_txt = fp.read_text() | ||
assert orig_txt.count(orig) == 1 | ||
tmp_txt = orig_txt.replace(orig, tmp) | ||
fp.write_text(tmp_txt) | ||
try: | ||
yield | ||
finally: | ||
fp.write_text(orig_txt) | ||
|
||
|
||
@contextlib.contextmanager | ||
def tmp_ctypesgen_pin(): | ||
|
||
pin = os.environ.get("CTYPESGEN_PIN", None) | ||
if not pin: | ||
head_url = "https://api.github.com/repos/pypdfium2-team/ctypesgen/git/refs/heads/pypdfium2" | ||
with url_request.urlopen(head_url) as rq: | ||
content = rq.read().decode() | ||
content = json.loads(content) | ||
pin = content["object"]["sha"] | ||
print(f"Resolved pypdfium2 ctypesgen HEAD to SHA {pin}", file=sys.stderr) | ||
|
||
base_txt = "ctypesgen @ git+https://github.com/pypdfium2-team/ctypesgen@" | ||
ctx = tmp_replace_ctx(ProjectDir/"pyproject.toml", base_txt+"pypdfium2", base_txt+pin) | ||
with ctx: | ||
print(f"Wrote temporary pyproject.toml with ctypesgen pin", file=sys.stderr) | ||
yield | ||
print(f"Reset pyproject.toml", file=sys.stderr) | ||
|
||
|
||
if __name__ == '__main__': | ||
main() |
Oops, something went wrong.