Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
(Experimental:) Isolate conda
Browse files Browse the repository at this point in the history
mara004 committed Jan 23, 2025

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 3f7a9c0 commit 613cafb
Showing 11 changed files with 323 additions and 300 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/conda.yaml
Original file line number Diff line number Diff line change
@@ -62,8 +62,9 @@ jobs:
git config --global user.name "geisserml"
python -m pip install -U -r req/setup.txt
# TODO make --new-only optional
- name: Build package
run: ./run craft --pdfium-ver "${{ inputs.pdfium_ver }}" conda_${{ inputs.package }}
run: ./run craft-conda ${{ inputs.package }} --pdfium-ver "${{ inputs.pdfium_ver }}" --new-only

- name: Upload artifact
uses: actions/upload-artifact@v4
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -789,7 +789,7 @@ You may also trigger the workflow manually using the GitHub Actions panel or the

Python release scripts are located in the folder `setupsrc/pypdfium2_setup`, along with custom setup code:
* `update_pdfium.py` downloads binaries.
* `craft_packages.py pypi` builds platform-specific wheel packages and a source distribution suitable for PyPI upload.
* `craft.py` builds platform-specific wheel packages and a source distribution suitable for PyPI upload.
* `autorelease.py` takes care of versioning, changelog, release note generation and VCS checkin.

The autorelease script has some peculiarities maintainers should know about:
@@ -815,7 +815,7 @@ In case of necessity, you may also forego autorelease/CI and do the release manu
* Build the packages
```bash
python setupsrc/pypdfium2_setup/update_pdfium.py
python setupsrc/pypdfium2_setup/craft_packages.py pypi
python setupsrc/pypdfium2_setup/craft.py
```
* Upload to PyPI
```bash
175 changes: 175 additions & 0 deletions conda/craft_conda_pkgs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
# SPDX-FileCopyrightText: 2024 geisserml <[email protected]>
# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause

import sys
import argparse
from pathlib import Path
from functools import partial

sys.path.insert(0, str(Path(__file__).parents[1]/"setupsrc"))
from pypdfium2_setup.packaging_base import *
from pypdfium2_setup.emplace import prepare_setup

CondaDir = ProjectDir / "conda"
CondaRaw_BuildNumF = CondaDir / "raw" / "build_num.txt"

T_RAW = "raw"
T_HELPERS = "helpers"


def main():
parser = argparse.ArgumentParser(
description = "Craft conda packages for pypdfium2"
)
parser.add_argument(
"type",
choices = (T_RAW, T_HELPERS),
help = "The package type to build (raw or helpers)",
)
parser.add_argument("--pdfium-ver", default=None)
parser.add_argument("--new-only", action="store_true")

args = parser.parse_args()
if args.type == T_RAW:
main_conda_raw(args)
elif args.type == T_HELPERS:
assert not args.new_only, "--new-only / buildnum handling not implemented for helpers package"
main_conda_helpers(args)
else:
assert False # unreached, handled by argparse


def _handle_ver(args, get_latest):
if not args.pdfium_ver or args.pdfium_ver == "latest":
args.pdfium_ver = get_latest()
else:
args.pdfium_ver = int(args.pdfium_ver)


def main_conda_raw(args):

_handle_ver(args, CondaPkgVer.get_latest_pdfium)
os.environ["PDFIUM_SHORT"] = str(args.pdfium_ver)
os.environ["PDFIUM_FULL"] = ".".join([str(v) for v in PdfiumVer.to_full(args.pdfium_ver)])
os.environ["BUILD_NUM"] = str(_get_build_num(args))

emplace_func = partial(prepare_setup, ExtPlats.system, args.pdfium_ver, use_v8=None)
with CondaExtPlatfiles(emplace_func):
run_conda_build(CondaDir/"raw", CondaDir/"raw"/"out", args=["--override-channels", "-c", "bblanchon", "-c", "defaults"])


def main_conda_helpers(args):

_handle_ver(args, CondaPkgVer.get_latest_bindings)
helpers_info = parse_git_tag()
os.environ["M_HELPERS_VER"] = merge_tag(helpers_info, "py")

# Set the current pdfium version as upper boundary, for inherent API safety.
# pdfium does not do semantic versioning, so upward flexibility is difficult.
os.environ["PDFIUM_MAX"] = str(args.pdfium_ver)

# NOTE To build with a local pypdfium2_raw, add the args below for the source dir, and remove the pypdfium2-team prefix from the helpers recipe's run requirements
# args=["-c", CondaDir/"raw"/"out"]
run_conda_build(CondaDir/"helpers", CondaDir/"helpers"/"out", args=["--override-channels", "-c", "pypdfium2-team", "-c", "bblanchon", "-c", "defaults"])


def run_conda_build(recipe_dir, out_dir, args=()):
with TmpCommitCtx():
run_cmd(["conda", "build", recipe_dir, "--output-folder", out_dir, *args], cwd=ProjectDir, env=os.environ)


@functools.lru_cache(maxsize=2)
def run_conda_search(package, channel):
output = run_cmd(["conda", "search", "--json", package, "--override-channels", "-c", channel], cwd=None, capture=True)
return json.loads(output)[package]


class CondaPkgVer:

@staticmethod
@functools.lru_cache(maxsize=2)
def _get_latest_for(package, channel, v_func):
search = run_conda_search(package, channel)
search = sorted(search, key=lambda d: v_func(d["version"]), reverse=True)
result = v_func(search[0]["version"])
print(f"Resolved latest {channel}::{package} to {result}", file=sys.stderr)
return result

@staticmethod
def get_latest_pdfium():
return CondaPkgVer._get_latest_for(
"pdfium-binaries", "bblanchon", lambda v: int(v.split(".")[2])
)

@staticmethod
def get_latest_bindings():
return CondaPkgVer._get_latest_for(
"pypdfium2_raw", "pypdfium2-team", lambda v: int(v)
)


def _get_build_num(args):

# parse existing releases to automatically handle arbitrary version builds
# TODO expand to pypdfium2_helpers as well, so we could rebuild with different pdfium bounds in a workflow

search = reversed(run_conda_search("pypdfium2_raw", "pypdfium2-team"))

if args.new_only:
# or `args.pdfium_ver not in {...}` to allow new builds of older versions
assert args.pdfium_ver > max(int(d["version"]) for d in search), f"--new-only given, but {args.pdfium_ver} already has a build"

# determine build number
build_num = max((d["build_number"] for d in search if int(d["version"]) == args.pdfium_ver), default=None)
build_num = 0 if build_num is None else build_num+1

return build_num


class TmpCommitCtx:

# Work around local conda `git_url` not including uncommitted changes
# In particular, this is used to transfer data files, so we can generate them externally and don't have to conda package ctypesgen.

# use a tmp control file so we can also undo the commit in conda's isolated clone
FILE = CondaDir / "with_tmp_commit.txt"

def __enter__(self):
# determine if there are any modified or new files
out = run_cmd(["git", "status", "--porcelain"], capture=True, cwd=ProjectDir)
self.have_mods = bool(out)
if self.have_mods: # make tmp commit
self.FILE.touch()
run_cmd(["git", "add", "."], cwd=ProjectDir)
run_cmd(["git", "commit", "-m", "!!! tmp commit for conda-build", "-m", "make conda-build include uncommitted changes"], cwd=ProjectDir)

@classmethod
def undo(cls):
# assuming FILE exists (promised by callers)
run_cmd(["git", "reset", "--soft", "HEAD^"], cwd=ProjectDir)
run_cmd(["git", "reset", cls.FILE], cwd=ProjectDir)
cls.FILE.unlink()

def __exit__(self, *_):
if self.have_mods: # pop tmp commit, if any
self.undo()


class CondaExtPlatfiles:

def __init__(self, emplace_func):
self.emplace_func = emplace_func

def __enter__(self):
self.platfiles = self.emplace_func()
self.platfiles = [ModuleDir_Raw/f for f in self.platfiles]
run_cmd(["git", "add", "-f"] + [str(f) for f in self.platfiles], cwd=ProjectDir)

def __exit__(self, *_):
run_cmd(["git", "reset"] + [str(f) for f in self.platfiles], cwd=ProjectDir)
for fp in self.platfiles:
fp.unlink()


if __name__ == "__main__":
main()
4 changes: 2 additions & 2 deletions conda/prepare_script.py
Original file line number Diff line number Diff line change
@@ -4,8 +4,8 @@
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parents[1] / "setupsrc"))
from pypdfium2_setup.craft_packages import TmpCommitCtx
sys.path.insert(0, str(Path(__file__).parents[1] / "conda"))
from craft_conda_pkgs import TmpCommitCtx

def main():
if TmpCommitCtx.FILE.exists():
1 change: 1 addition & 0 deletions docs/devel/changelog_staging.md
Original file line number Diff line number Diff line change
@@ -40,6 +40,7 @@
*Project*
- Merged `tests_old/` back into `tests/`.
- Docs: Improved logic when to include the unreleased version warning and upcoming changelog.
- Cleanly split out conda packaging into an own file, and confined it to the `conda/` directory, to avoid polluting the main setup code.

<!-- TODO
See https://github.com/pypdfium2-team/pypdfium2/blob/devel_old/docs/devel/changelog_staging.md
7 changes: 5 additions & 2 deletions run
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ function packaging_pypi() {

# calling update_pdfium is not strictly necessary, but may improve performance because downloads are done in parallel, rather than linear with each package
python3 setupsrc/pypdfium2_setup/update_pdfium.py
python3 setupsrc/pypdfium2_setup/craft_packages.py pypi
python3 setupsrc/pypdfium2_setup/craft.py

twine check dist/*
# ignore W002: erroneous detection of __init__.py files as duplicates
@@ -60,7 +60,10 @@ update)
python3 setupsrc/pypdfium2_setup/update_pdfium.py $args;;

craft)
python3 setupsrc/pypdfium2_setup/craft_packages.py $args;;
python3 setupsrc/pypdfium2_setup/craft.py $args;;

craft-conda)
python3 conda/craft_conda_pkgs.py $args;;

build)
python3 setupsrc/pypdfium2_setup/build_pdfium.py $args;;
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -104,7 +104,7 @@ def run_setup(modnames, pl_name, pdfium_ver):
helpers_info = get_helpers_info()
if pl_name == ExtPlats.sdist:
if helpers_info["dirty"]:
# ignore dirty state due to craft_packages::tmp_ctypesgen_pin()
# ignore dirty state due to craft.py::tmp_ctypesgen_pin()
if int(os.environ.get("SDIST_IGNORE_DIRTY", 0)):
helpers_info["dirty"] = False
else:
131 changes: 131 additions & 0 deletions setupsrc/pypdfium2_setup/craft.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# SPDX-FileCopyrightText: 2024 geisserml <[email protected]>
# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause

import os
import sys
import json
import shutil
import argparse
import tempfile
import contextlib
import urllib.request as url_request
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parents[1]))
from pypdfium2_setup.packaging_base import *

try:
import build.__main__ as build_module
except ImportError:
build_module = None


def main():

parser = argparse.ArgumentParser(
description = "Craft PyPI packages for pypdfium2"
)
parser.add_argument("--pdfium-ver", default=None)
parser.add_argument("--use-v8", action="store_true")
parser.add_argument("--wheels", action="store_true")
parser.add_argument("--sdist", action="store_true")

args = parser.parse_args()
if not (args.wheels or args.sdist):
args.wheels, args.sdist = True, True
if not args.pdfium_ver or args.pdfium_ver == "latest":
args.pdfium_ver = PdfiumVer.get_latest()
else:
args.pdfium_ver = int(args.pdfium_ver)

with ArtifactStash():
main_pypi(args)


def main_pypi(args):

assert args.sdist or args.wheels

if args.sdist:
os.environ[PlatSpec_EnvVar] = ExtPlats.sdist
helpers_info = get_helpers_info()
with tmp_ctypesgen_pin():
if not helpers_info["dirty"]:
os.environ["SDIST_IGNORE_DIRTY"] = "1"
_run_pypi_build(["--sdist"])

if args.wheels:
suffix = build_pl_suffix(args.pdfium_ver, args.use_v8)
for plat in WheelPlatforms:
os.environ[PlatSpec_EnvVar] = plat + suffix
_run_pypi_build(["--wheel"])
clean_platfiles()


def _run_pypi_build(caller_args):
# -nx: --no-isolation --skip-dependency-check
assert build_module, "Module 'build' is not importable. Cannot craft PyPI packages."
with tmp_cwd_context(ProjectDir):
build_module.main([str(ProjectDir), "-nx", *caller_args])


class ArtifactStash:

# Preserve in-tree artifacts from editable install

def __enter__(self):

self.tmpdir = None

file_names = [VersionFN, BindingsFN, LibnameForSystem[Host.system]]
self.files = [fp for fp in [ModuleDir_Raw / fn for fn in file_names] if fp.exists()]
if len(self.files) == 0:
return

self.tmpdir = tempfile.TemporaryDirectory(prefix="pypdfium2_artifact_stash_")
self.tmpdir_path = Path(self.tmpdir.name)
for fp in self.files:
shutil.move(fp, self.tmpdir_path)

def __exit__(self, *_):
if self.tmpdir is None:
return
for fp in self.files:
shutil.move(self.tmpdir_path / fp.name, ModuleDir_Raw)
self.tmpdir.cleanup()


@contextlib.contextmanager
def tmp_replace_ctx(fp, orig, tmp):
orig_txt = fp.read_text()
assert orig_txt.count(orig) == 1
tmp_txt = orig_txt.replace(orig, tmp)
fp.write_text(tmp_txt)
try:
yield
finally:
fp.write_text(orig_txt)


@contextlib.contextmanager
def tmp_ctypesgen_pin():

pin = os.environ.get("CTYPESGEN_PIN", None)
if not pin:
head_url = "https://api.github.com/repos/pypdfium2-team/ctypesgen/git/refs/heads/pypdfium2"
with url_request.urlopen(head_url) as rq:
content = rq.read().decode()
content = json.loads(content)
pin = content["object"]["sha"]
print(f"Resolved pypdfium2 ctypesgen HEAD to SHA {pin}", file=sys.stderr)

base_txt = "ctypesgen @ git+https://github.com/pypdfium2-team/ctypesgen@"
ctx = tmp_replace_ctx(ProjectDir/"pyproject.toml", base_txt+"pypdfium2", base_txt+pin)
with ctx:
print(f"Wrote temporary pyproject.toml with ctypesgen pin", file=sys.stderr)
yield
print(f"Reset pyproject.toml", file=sys.stderr)


if __name__ == '__main__':
main()
Loading

0 comments on commit 613cafb

Please sign in to comment.