diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 769a83560e9..de094472229 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -16,15 +16,6 @@ on: - develop workflow_dispatch: # Allow to run manually - inputs: - platform: - description: 'Platform' - required: true - default: 'ubuntu-noble-standard' - docker_tag: - description: 'Docker tag' - required: true - default: 'dev' concurrency: # Cancel previous runs of this workflow for the same branch @@ -32,101 +23,60 @@ concurrency: cancel-in-progress: true env: - # Same as in build.yml - TOX_ENV: "docker-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-incremental" - BUILD_IMAGE: "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-with-targets:ci" - FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/" - FROM_DOCKER_TARGET: "with-targets" - FROM_DOCKER_TAG: ${{ github.event.inputs.docker_tag || 'dev'}} - EXTRA_CONFIGURE_ARGS: --enable-fat-binary + PYTHON_VERSION: 3.11 jobs: build-doc: runs-on: ubuntu-latest - services: - # https://docs.docker.com/build/ci/github-actions/local-registry/ - registry: - image: registry:2 - ports: - - 5000:5000 steps: - - name: Maximize build disk space - uses: easimon/maximize-build-space@v10 - with: - # need space in /var for Docker images - root-reserve-mb: 30000 - remove-dotnet: true - remove-android: true - remove-haskell: true - remove-codeql: true - remove-docker-images: true - name: Checkout uses: actions/checkout@v4 - - name: Install test prerequisites - # From docker.yml - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install tox - sudo apt-get clean - df -h + - name: Merge CI fixes from sagemath/sage run: | - mkdir -p upstream - .github/workflows/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log + .github/workflows/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} - SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - - # Building - - - name: Generate Dockerfile - # From docker.yml - run: | - tox -e ${{ env.TOX_ENV }} - cp .tox/${{ env.TOX_ENV }}/Dockerfile . - env: - # Only generate the Dockerfile, do not run 'docker build' here - DOCKER_TARGETS: "" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Cache conda packages + uses: actions/cache@v4 with: - driver-opts: network=host - - - name: Build Docker image - id: image - uses: docker/build-push-action@v6 + path: ~/conda_pkgs_dir + key: + ${{ runner.os }}-conda-${{ hashFiles('environment-${{ env.PYTHON_VERSION }}-linux.yml') }} + + - name: Compiler cache + uses: hendrikmuhs/ccache-action@v1.2 with: - # push and load may not be set together at the moment - push: true - load: false - context: . - tags: ${{ env.BUILD_IMAGE }} - target: with-targets - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - NUMPROC=6 - USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse - TARGETS_PRE=build/make/Makefile - TARGETS=ci-build-with-fallback + key: ${{ runner.os }}-meson-${{ env.PYTHON_VERSION }} - - name: Start container - id: container - # Try to continue when "exporting to GitHub Actions Cache" failed with timeout + - name: Setup Conda environment + uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ env.PYTHON_VERSION }} + # Disabled for now due to + # https://github.com/conda-incubator/setup-miniconda/issues/379 + # miniforge-version: latest + use-mamba: true + channels: conda-forge + channel-priority: true + activate-environment: sage-dev + environment-file: environment-${{ env.PYTHON_VERSION }}-linux.yml + + - name: Build Sage + shell: bash -l {0} run: | - docker run --name BUILD -dit \ - --mount type=bind,src=$(pwd),dst=$(pwd) \ - --workdir $(pwd) \ - ${{ env.BUILD_IMAGE }} /bin/sh + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export CC="ccache $CC" + export CXX="ccache $CXX" + pip install --no-build-isolation --config-settings=builddir=builddir --editable . -v # - # On pull request and push to develop events + # For pull requests # - - name: Get workflow run-id id: get_run_id - if: steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/') && github.event_name == 'pull_request' + if: github.event_name == 'pull_request' run: | RESPONSE=$(curl -s -L \ -H "Accept: application/vnd.github+json" \ @@ -149,6 +99,7 @@ jobs: - name: Store old doc id: worktree if: steps.download-doc.outcome == 'success' + shell: bash -l {0} run: | git config --global --add safe.directory $(pwd) git config --global user.email "ci-sage@example.com" @@ -164,8 +115,8 @@ jobs: # mathjax path in old doc (regex) mathjax_path_from="[-./A-Za-z_]*/tex-chtml[.]js?v=[0-9a-f]*" # mathjax path in new doc - mathjax_path_to=$(docker exec -e SAGE_USE_CDNS=yes BUILD /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)") - new_version=$(docker exec BUILD cat src/VERSION.txt) + mathjax_path_to=$(SAGE_USE_CDNS=yes python -c "from src.sage_docbuild.conf import mathjax_path; print(mathjax_path)") + new_version=$(cat src/VERSION.txt) # Wipe out chronic diffs between old doc and new doc (cd doc && \ find . -name "*.html" | xargs sed -i -e '/class="sidebar-brand-text"/ s/Sage [0-9a-z.]* /Sage '"$new_version"' /' \ @@ -185,20 +136,15 @@ jobs: git add -A && git commit --quiet -m 'old') fi - - name: Build doc + - name: Build documentation id: docbuild - if: steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/') - # Always non-incremental because of the concern that - # incremental docbuild may introduce broken links (inter-file references) though build succeeds + if: steps.worktree.outcome == 'success' + shell: bash -l {0} run: | - export GITHUB_REF=${{ github.ref }} - export PR_SHA=${{ github.event.pull_request.head.sha }} - export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5 - make doc-clean doc-uninstall - export SAGE_USE_CDNS=yes - export SAGE_DOCBUILD_OPTS="--include-tests-blocks" - ./config.status && make sagemath_doc_html-no-deps - shell: sh .github/workflows/docker-exec-script.sh BUILD /sage {0} + meson compile -C builddir doc-html + env: + SAGE_USE_CDNS: yes + SAGE_DOCBUILD_OPTS: "--include-tests-blocks" - name: Copy doc id: copy @@ -209,12 +155,7 @@ jobs: if [ -d "doc/html" ]; then rm -rf doc/html fi - # Simpler "docker cp --follow-link ... doc" does not work - mkdir -p doc - mkdir -p temp - docker cp --follow-link BUILD:/sage/local/share/doc/sage/html temp - docker cp --follow-link BUILD:/sage/local/share/doc/sage/index.html temp - cp -r -L temp/* doc/ + cp -r builddir/src/doc/* doc/ # Check if we are on pull request event PR_NUMBER="" if [[ -n "$GITHUB_REF" ]]; then @@ -275,20 +216,14 @@ jobs: - name: Build live doc id: buildlivedoc if: startsWith(github.ref, 'refs/tags/') + shell: bash -l {0} run: | - # Avoid running out of disk space - rm -rf upstream - export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5 - export PATH="build/bin:$PATH" - eval $(sage-print-system-package-command auto update) - eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) - eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive texlive_luatex free_fonts xindy) - export SAGE_USE_CDNS=yes - export SAGE_LIVE_DOC=yes - export SAGE_JUPYTER_SERVER=binder:sagemath/sage-binder-env/dev - make doc-clean doc-uninstall - ./config.status && make sagemath_doc_html-no-deps sagemath_doc_pdf-no-deps - shell: sh .github/workflows/docker-exec-script.sh BUILD /sage {0} + meson compile -C builddir doc-html + env: + SAGE_USE_CDNS: yes + SAGE_LIVE_DOC: yes + SAGE_JUPYTER_SERVER: binder:sagemath/sage-binder-env/dev + SAGE_DOCBUILD_OPTS: "--include-tests-blocks" - name: Copy live doc id: copylivedoc @@ -296,9 +231,9 @@ jobs: run: | mkdir -p ./livedoc # We copy everything to a local folder - docker cp --follow-link BUILD:/sage/local/share/doc/sage/html livedoc - docker cp --follow-link BUILD:/sage/local/share/doc/sage/pdf livedoc - docker cp --follow-link BUILD:/sage/local/share/doc/sage/index.html livedoc + cp -r builddir/src/doc/html livedoc/ + cp -r builddir/src/doc/pdf livedoc/ + cp builddir/src/doc/index.html livedoc/ zip -r livedoc.zip livedoc - name: Upload live doc diff --git a/environment-3.11-linux.yml b/environment-3.11-linux.yml index 0fde543be6f..29da5832337 100644 --- a/environment-3.11-linux.yml +++ b/environment-3.11-linux.yml @@ -336,6 +336,7 @@ dependencies: - soupsieve=2.5=pyhd8ed1ab_1 - sphinx=8.2.3=pyhd8ed1ab_0 - sphinx-basic-ng=1.0.0b2=pyhd8ed1ab_3 + - sphinx-copybutton=0.5.2=pyhd8ed1ab_1 - sphinx-inline-tabs=2023.4.21=pyhd8ed1ab_1 - sphinxcontrib-applehelp=2.0.0=pyhd8ed1ab_1 - sphinxcontrib-devhelp=2.0.0=pyhd8ed1ab_1 diff --git a/meson.build b/meson.build old mode 100644 new mode 100755 index b257ac0f944..13a445376c6 --- a/meson.build +++ b/meson.build @@ -4,6 +4,7 @@ project( version: files('src/VERSION.txt'), license: 'GPL v3', default_options: ['c_std=c17', 'cpp_std=c++17', 'python.install_env=auto'], + meson_version: '>=1.2', ) # Python module @@ -215,4 +216,6 @@ for path in file_paths: ] run_command(create_files_command, check: true) +root = meson.current_source_dir() + subdir('src') diff --git a/pyproject.toml b/pyproject.toml index 87e730a2104..7d626b54be7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,7 +184,8 @@ test = [ ] docs = [ "sphinx", - "sphinx-inline-tabs", + "sphinx-copybutton", + "sphinx-inline-tabs", "furo", "python-dateutil", ] @@ -201,6 +202,7 @@ dev = [ "conda-lock", "grayskull", "toml", + "uv", ] [tool.ruff] diff --git a/src/build-docs.py b/src/build-docs.py new file mode 100644 index 00000000000..2fa1c4f718e --- /dev/null +++ b/src/build-docs.py @@ -0,0 +1,4 @@ +from sage_docbuild.__main__ import main + +if __name__ == "__main__": + main() diff --git a/src/doc/Makefile b/src/doc/Makefile index 9c03292b070..47a843cb591 100644 --- a/src/doc/Makefile +++ b/src/doc/Makefile @@ -40,7 +40,7 @@ doc-inventory-reference: doc-src $(eval DOCS = $(shell sage --docbuild --all-documents reference)) @if [ -z "$(DOCS)" ]; then echo "Error: 'sage --docbuild --all-documents' failed"; exit 1; fi $(eval BIBLIO = $(firstword $(DOCS))) - $(eval OTHER_DOCS = $(wordlist 2, 100, $(DOCS))) + $(eval OTHER_DOCS = $(filter-out reference_top, $(wordlist 2, 100, $(DOCS)))) $(MAKE) doc-inventory--$(subst /,-,$(BIBLIO)) $(MAKE) $(foreach doc, $(OTHER_DOCS), doc-inventory--$(subst /,-,$(doc))) $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" doc-inventory--reference_top @@ -51,7 +51,7 @@ doc-html-reference-sub: doc-inventory-reference $(eval DOCS = $(shell sage --docbuild --all-documents reference)) @if [ -z "$(DOCS)" ]; then echo "Error: 'sage --docbuild --all-documents' failed"; exit 1; fi $(eval BIBLIO = $(firstword $(DOCS))) - $(eval OTHER_DOCS = $(wordlist 2, 100, $(DOCS))) + $(eval OTHER_DOCS = $(filter-out reference_top, $(wordlist 2, 100, $(DOCS)))) $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" doc-html--$(subst /,-,$(BIBLIO)) $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" $(foreach doc, $(OTHER_DOCS), doc-html--$(subst /,-,$(doc))) @@ -63,7 +63,7 @@ doc-html-reference: doc-html-reference-sub doc-html-other: doc-html-reference $(eval DOCS = $(shell sage --docbuild --all-documents all)) @if [ -z "$(DOCS)" ]; then echo "Error: 'sage --docbuild --all-documents' failed"; exit 1; fi - $(MAKE) $(foreach doc, $(wordlist 2, 100, $(DOCS)), doc-html--$(subst /,-,$(doc))) + $(MAKE) $(foreach doc, $(DOCS), doc-html--$(subst /,-,$(doc))) doc-html: doc-html-reference doc-html-other SAGE_DOC=$$(sage --python -c "from sage.env import SAGE_DOC; print(SAGE_DOC)") @@ -78,7 +78,7 @@ doc-pdf-reference: doc-inventory-reference $(eval DOCS = $(shell sage --docbuild --all-documents reference)) @if [ -z "$(DOCS)" ]; then echo "Error: 'sage --docbuild --all-documents' failed"; exit 1; fi $(eval BIBLIO = $(firstword $(DOCS))) - $(eval OTHER_DOCS = $(wordlist 2, 100, $(DOCS))) + $(eval OTHER_DOCS = $(filter-out reference_top, $(wordlist 2, 100, $(DOCS)))) $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" doc-pdf--$(subst /,-,$(BIBLIO)) $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" $(foreach doc, $(OTHER_DOCS), doc-pdf--$(subst /,-,$(doc))) $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" doc-pdf--reference_top @@ -87,7 +87,7 @@ doc-pdf-reference: doc-inventory-reference doc-pdf-other: doc-pdf-reference $(eval DOCS = $(shell sage --docbuild --all-documents all)) @if [ -z "$(DOCS)" ]; then echo "Error: 'sage --docbuild --all-documents' failed"; exit 1; fi - $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" $(foreach doc, $(wordlist 2, 100, $(DOCS)), doc-pdf--$(subst /,-,$(doc))) + $(MAKE) SAGE_DOCBUILD_OPTS="$(SAGE_DOCBUILD_OPTS) --no-prune-empty-dirs" $(foreach doc, $(DOCS), doc-pdf--$(subst /,-,$(doc))) doc-pdf: doc-pdf-reference doc-pdf-other SAGE_DOC=$$(sage --python -c "from sage.env import SAGE_DOC; print(SAGE_DOC)") diff --git a/src/doc/bootstrap b/src/doc/bootstrap index e99ecd52c28..9d59aa1755e 100755 --- a/src/doc/bootstrap +++ b/src/doc/bootstrap @@ -18,6 +18,7 @@ if [ -z "$SAGE_ROOT" ]; then fi cd "$SAGE_ROOT" +export PATH=build/bin:$PATH OUTPUT_DIR="src/doc/en/installation" mkdir -p "$OUTPUT_DIR" diff --git a/src/doc/en/installation/meson.rst b/src/doc/en/installation/meson.rst index cf5e7656312..7f3abcfc4cf 100644 --- a/src/doc/en/installation/meson.rst +++ b/src/doc/en/installation/meson.rst @@ -172,7 +172,7 @@ To compile and install Sage in editable install, then just use: memory_allocator \ "numpy >=1.25" \ jinja2 \ - setuptool + setuptools $ uv sync --frozen --inexact --no-build-isolation You can then start Sage from the command line with ``./sage`` diff --git a/src/doc/en/reference/conf_sub.py b/src/doc/en/reference/conf_sub.py old mode 100644 new mode 100755 index 479e9c306d2..380e86e0fdc --- a/src/doc/en/reference/conf_sub.py +++ b/src/doc/en/reference/conf_sub.py @@ -11,9 +11,10 @@ # serve to show the default. import os -from sage.env import SAGE_DOC_SRC, SAGE_DOC -from sage_docbuild.conf import release, exclude_patterns + +from sage.env import SAGE_DOC, SAGE_DOC_SRC from sage_docbuild.conf import * +from sage_docbuild.conf import exclude_patterns # Add any paths that contain custom static files (such as style sheets), # relative to this directory to html_static_path. They are copied after the @@ -26,7 +27,7 @@ ref_out = os.path.join(SAGE_DOC, 'html', 'en', 'reference') # We use the main document's title, if we can find it. -rst_file = open('index.rst', 'r') +rst_file = open('index.rst', 'r', encoding='utf-8') rst_lines = rst_file.read().splitlines() rst_file.close() diff --git a/src/doc/en/reference/repl/meson.build b/src/doc/en/reference/repl/meson.build new file mode 100644 index 00000000000..381f31d0f00 --- /dev/null +++ b/src/doc/en/reference/repl/meson.build @@ -0,0 +1,6 @@ +doc_src += custom_target( + 'doc-src', + output: ['options.txt'], + command: [py, src / 'sage' / 'cli', '--help'], + capture: true, +) diff --git a/src/doc/meson.build b/src/doc/meson.build new file mode 100644 index 00000000000..dedbe0b60c0 --- /dev/null +++ b/src/doc/meson.build @@ -0,0 +1,121 @@ +doc_src = [] +subdir('en/reference/repl') +# TODO: Migrate this completely to meson +doc_src += custom_target( + 'doc-src', + output: ['autogen'], + command: [files('bootstrap')], + env: {'SAGE_ROOT': root}, +) + +references = run_command( + py, + [ + src / 'build-docs.py', + '--all-documents', + 'reference', + '--source', + meson.current_source_dir(), + ], + check: true, +).stdout().strip() + +reference_inventory = [] +reference_html = [] +reference_pdf = [] +bibliography = [] +foreach type : ['inventory', 'html', 'pdf'] + foreach ref : references.splitlines() + if '/' in ref + short_ref = ref.split('/')[1] + else + short_ref = ref + endif + deps = [] + deps += doc_src + if type == 'html' or type == 'pdf' + deps += reference_inventory + endif + if short_ref != 'references' + deps += bibliography + endif + if short_ref == 'reference_top' + deps += reference_inventory + if type == 'html' + deps += reference_html + elif type == 'pdf' + deps += reference_pdf + endif + endif + target = custom_target( + 'doc-' + type + '-reference-' + short_ref, + output: [type + short_ref], + command: [ + py, + src / 'build-docs.py', + '--no-pdf-links', + ref, + type, + '-o', + '@OUTDIR@', + '--source', + meson.current_source_dir(), + ], + depends: deps, + ) + if short_ref == 'references' + bibliography += target + endif + if type == 'inventory' + reference_inventory += target + elif type == 'html' + reference_html += target + elif type == 'pdf' + reference_pdf += target + endif + endforeach +endforeach + +other_documents = run_command( + py, + [ + src / 'build-docs.py', + '--all-documents', + 'all', + '--source', + meson.current_source_dir(), + ], + check: true, +).stdout().strip() +other_documents_html = [] +other_documents_pdf = [] +foreach type : ['html', 'pdf'] + foreach doc : other_documents.splitlines() + short_doc = doc.replace('/', '-') + target = custom_target( + 'doc-' + type + '-other-' + short_doc, + output: [type + short_doc], + command: [ + py, + src / 'build-docs.py', + '--no-pdf-links', + doc, + type, + '-o', + '@OUTDIR@', + '--source', + meson.current_source_dir(), + ], + depends: reference_inventory, + ) + if type == 'html' + other_documents_html += target + elif type == 'pdf' + other_documents_pdf += target + endif + endforeach +endforeach + +# Custom target for building the complete documentation +alias_target('doc-html', [reference_html, other_documents_html]) +alias_target('doc-pdf', [reference_pdf, other_documents_pdf]) diff --git a/src/meson.build b/src/meson.build index fe627fc760f..335f35d5823 100644 --- a/src/meson.build +++ b/src/meson.build @@ -234,5 +234,8 @@ inc_ext = include_directories('sage/ext') inc_partn_ref2 = include_directories('sage/groups/perm_gps/partn_ref2') inc_src = include_directories('.') +src = meson.current_source_dir() + # Submodules subdir('sage') +subdir('doc') diff --git a/src/sage/repl/ipython_kernel/all_jupyter.py b/src/sage/repl/ipython_kernel/all_jupyter.py index 2d2677da27d..03c7af72c96 100644 --- a/src/sage/repl/ipython_kernel/all_jupyter.py +++ b/src/sage/repl/ipython_kernel/all_jupyter.py @@ -8,3 +8,5 @@ from sage.repl.ipython_kernel.widgets_sagenb import (input_box, text_control, slider, range_slider, checkbox, selector, input_grid, color_selector) from sage.repl.ipython_kernel.interact import interact + +from pathlib import Path \ No newline at end of file diff --git a/src/sage_docbuild/__main__.py b/src/sage_docbuild/__main__.py index a7b7b39880b..6459596a28b 100644 --- a/src/sage_docbuild/__main__.py +++ b/src/sage_docbuild/__main__.py @@ -72,15 +72,22 @@ en/reference. If ARG is 'all', list all main documents """ -import logging import argparse +import logging import os -import shlex import sys +from pathlib import Path + import sphinx.ext.intersphinx -from sage.env import SAGE_DOC_SRC -from .builders import DocBuilder, ReferenceBuilder, get_builder, get_documents + from . import build_options +from .build_options import BuildOptions +from .builders import ( + DocBuilder, + get_all_documents, + get_all_reference_documents, + get_builder, +) logger = logging.getLogger(__name__) @@ -161,7 +168,7 @@ def help_documents(): s += "\n" if 'reference' in docs: s += "Other valid document names take the form 'reference/DIR', where\n" - s += "DIR is a subdirectory of SAGE_DOC_SRC/en/reference/.\n" + s += "DIR is a subdirectory of src/doc/en/reference/.\n" s += "This builds just the specified part of the reference manual.\n" s += "DOCUMENT may also have the form 'file=/path/to/FILE', which builds\n" s += "the documentation for the specified file.\n" @@ -173,7 +180,7 @@ def get_formats(): Return a list of output formats the Sage documentation builder will accept on the command-line. """ - tut_b = DocBuilder('en/tutorial') + tut_b = DocBuilder('en/tutorial', BuildOptions()) formats = tut_b._output_formats() formats.remove('html') return ['html', 'pdf'] + formats @@ -251,21 +258,6 @@ def __call__(self, parser, namespace, values, option_string=None): print(help_formats(), end="") if self.dest == 'commands': print(help_commands(values), end="") - if self.dest == 'all_documents': - if values == 'reference': - b = ReferenceBuilder('reference') - refdir = os.path.join(os.environ['SAGE_DOC_SRC'], 'en', b.name) - s = b.get_all_documents(refdir) - # Put the bibliography first, because it needs to be built first: - s.remove('reference/references') - s.insert(0, 'reference/references') - elif values == 'all': - s = get_documents() - # Put the reference manual first, because it needs to be built first: - s.remove('reference') - s.insert(0, 'reference') - for d in s: - print(d) setattr(namespace, 'printed_list', 1) sys.exit(0) @@ -339,7 +331,11 @@ def setup_parser(): type=int, default=1, metavar="LEVEL", action="store", help="report progress at LEVEL=0 (quiet), 1 (normal), 2 (info), or 3 (debug); does not affect children") + standard.add_argument("-s", "--source", dest="source_dir", type=Path, + default=None, metavar="DIR", action="store", + help="directory containing the documentation source files") standard.add_argument("-o", "--output", dest="output_dir", default=None, + type=Path, metavar="DIR", action="store", help="if DOCUMENT is a single file ('file=...'), write output to this directory") @@ -359,7 +355,6 @@ def setup_parser(): advanced.add_argument("--all-documents", dest="all_documents", type=str, metavar="ARG", choices=['all', 'reference'], - action=help_wrapper, help="if ARG is 'reference', list all subdocuments" " of en/reference. If ARG is 'all', list all main" " documents") @@ -456,8 +451,35 @@ def fetch_inventory(self, app, uri, inv): def main(): # Parse the command-line. parser = setup_parser() - args = parser.parse_args() - DocBuilder._options = args + args: BuildOptions = parser.parse_args() # type: ignore + + # Check that the docs source directory exists + if args.source_dir is None: + args.source_dir = Path(os.environ.get('SAGE_DOC_SRC', 'src/doc')) + args.source_dir = args.source_dir.absolute() + if not args.source_dir.is_dir(): + parser.error(f"Source directory {args.source_dir} does not exist.") + + if args.all_documents: + if args.all_documents == 'reference': + docs = get_all_reference_documents(args.source_dir / 'en') + elif args.all_documents == 'all': + docs = get_all_documents(args.source_dir) + else: + parser.error(f"Unknown argument {args.all_documents} for --all-documents.") + for d in docs: + print(d.as_posix()) + sys.exit(0) + + # Check that the docs output directory exists + if args.output_dir is None: + args.output_dir = Path(os.environ.get('SAGE_DOC', 'src/doc')) + args.output_dir = args.output_dir.absolute() + if not args.output_dir.exists(): + try: + args.output_dir.mkdir(parents=True) + except Exception as e: + parser.error(f"Failed to create output directory {args.output_dir}: {e}") # Get the name and type (target format) of the document we are # trying to build. @@ -465,30 +487,24 @@ def main(): if not name or not typ: parser.print_help() sys.exit(1) - elif name == 'all': - sys.exit(os.system(f'cd {shlex.quote(SAGE_DOC_SRC)} ' - f'&& ${{MAKE:-make}} -j${{SAGE_NUM_THREADS_PARALLEL:-1}} doc-{typ}')) # Set up module-wide logging. setup_logger(args.verbose, args.color) def excepthook(*exc_info): logger.error('Error building the documentation.', exc_info=exc_info) - if build_options.INCREMENTAL_BUILD: - logger.error(''' - Note: incremental documentation builds sometimes cause spurious - error messages. To be certain that these are real errors, run - "make doc-clean doc-uninstall" first and try again.''') + logger.info(''' +Note: incremental documentation builds sometimes cause spurious +error messages. To be certain that these are real errors, run +"make doc-clean doc-uninstall" first and try again.''') sys.excepthook = excepthook - # Process selected options. + # Set up the environment based on the command-line options if args.check_nested: os.environ['SAGE_CHECK_NESTED'] = 'True' - if args.underscore: os.environ['SAGE_DOC_UNDERSCORE'] = "True" - if args.sphinx_opts: build_options.ALLSPHINXOPTS += args.sphinx_opts.replace(',', ' ') + " " if args.no_pdf_links: @@ -505,13 +521,15 @@ def excepthook(*exc_info): os.environ['SAGE_SKIP_TESTS_BLOCKS'] = 'True' if args.use_cdns: os.environ['SAGE_USE_CDNS'] = 'yes' + os.environ['SAGE_DOC_SRC'] = str(args.source_dir) + os.environ['SAGE_DOC'] = str(args.output_dir) build_options.ABORT_ON_ERROR = not args.keep_going # Set up Intersphinx cache _ = IntersphinxCache() - builder = get_builder(name) + builder = get_builder(name, args) if not args.no_prune_empty_dirs: # Delete empty directories. This is needed in particular for empty @@ -519,11 +537,13 @@ def excepthook(*exc_info): # directories it leaves behind. See Issue #20010. # Issue #31948: This is not parallelization-safe; use the option # --no-prune-empty-dirs to turn it off - for dirpath, dirnames, filenames in os.walk(builder.dir, topdown=False): + for dirpath, dirnames, filenames in os.walk(args.source_dir, topdown=False): if not dirnames + filenames: logger.warning('Deleting empty directory {0}'.format(dirpath)) os.rmdir(dirpath) + import sage.all # TODO: Remove once all modules can be imported independently # noqa: F401 + build = getattr(builder, typ) build() diff --git a/src/sage_docbuild/build_options.py b/src/sage_docbuild/build_options.py index 4857c1ed125..7be118e3c12 100644 --- a/src/sage_docbuild/build_options.py +++ b/src/sage_docbuild/build_options.py @@ -4,12 +4,10 @@ This module defines options for building Sage documentation. """ +import argparse import os -import re +from pathlib import Path -from sage.env import SAGE_DOC_SRC, SAGE_DOC - -LANGUAGES = [d for d in os.listdir(SAGE_DOC_SRC) if re.match('^[a-z][a-z]$', d)] SPHINXOPTS = "" PAPER = "" OMIT = ["introspect"] # docs/dirs to omit when listing and building 'all' @@ -26,7 +24,9 @@ # Number of threads to use for parallel-building the documentation. NUM_THREADS = int(os.environ.get('SAGE_NUM_THREADS', 1)) -INCREMENTAL_BUILD = os.path.isdir(SAGE_DOC) - # Error out on errors ABORT_ON_ERROR = True + +class BuildOptions(argparse.Namespace): + source_dir: Path + output_dir: Path diff --git a/src/sage_docbuild/builders.py b/src/sage_docbuild/builders.py index dc93754ec98..91035a01f1c 100644 --- a/src/sage_docbuild/builders.py +++ b/src/sage_docbuild/builders.py @@ -70,14 +70,12 @@ import subprocess import sys import time -import types import warnings +from pathlib import Path +from typing import Generator, Literal -import sage.all -from sage.misc.cachefunc import cached_method -# Do not import SAGE_DOC globally as it interferes with doctesting with a random replacement -from sage.env import SAGE_DOC_SRC, SAGE_SRC, DOT_SAGE from . import build_options +from .build_options import BuildOptions from .utils import build_many as _build_many logger = logging.getLogger(__name__) @@ -106,31 +104,6 @@ def builder_helper(type): """ Return a function which builds the documentation for output type ``type``. - - TESTS: - - Check that :issue:`25161` has been resolved:: - - sage: from sage_docbuild.builders import DocBuilder - sage: from sage_docbuild.__main__ import setup_parser - sage: DocBuilder._options = setup_parser().parse_args([]) # builder_helper needs _options to be set - - sage: import sage_docbuild.sphinxbuild - sage: def raiseBaseException(): - ....: raise BaseException("abort pool operation") - sage: original_runsphinx, sage_docbuild.sphinxbuild.runsphinx = sage_docbuild.sphinxbuild.runsphinx, raiseBaseException - - sage: from sage.misc.temporary_file import tmp_dir - sage: os.environ['SAGE_DOC'] = tmp_dir() - sage: sage.env.var('SAGE_DOC') # random - sage: from sage_docbuild.builders import builder_helper, build_ref_doc - sage: from sage_docbuild.builders import _build_many as build_many - sage: helper = builder_helper("html") - sage: try: # optional - sagemath_doc_html - ....: build_many(build_ref_doc, [("docname", "en", "html", {})]) - ....: except Exception as E: - ....: "Non-exception during docbuild: abort pool operation" in str(E) - True """ def f(self, *args, **kwds): output_dir = self._output_dir(type) @@ -181,24 +154,16 @@ def f(self, *args, **kwds): class DocBuilder(): - def __init__(self, name, lang='en'): + def __init__(self, name: str, options: BuildOptions): """ INPUT: - - ``name`` -- the name of a subdirectory in SAGE_DOC_SRC, such as - 'tutorial' or 'bordeaux_2008' - - - ``lang`` -- (default "en") the language of the document. + - ``name`` -- the name of a subdirectory in ``doc/``, such as + 'tutorial' or 'installation' """ - doc = name.split(os.path.sep) - - if doc[0] in build_options.LANGUAGES: - lang = doc[0] - doc.pop(0) - - self.name = os.path.join(*doc) - self.lang = lang - self.dir = os.path.join(SAGE_DOC_SRC, self.lang, self.name) + self.name = name + self.dir = options.source_dir / self.name + self._options = options def _output_dir(self, type): """ @@ -210,16 +175,19 @@ def _output_dir(self, type): EXAMPLES:: sage: from sage_docbuild.builders import DocBuilder - sage: b = DocBuilder('tutorial') - sage: b._output_dir('html') # optional - sagemath_doc_html - '.../html/en/tutorial' + sage: from sage_docbuild.build_options import BuildOptions + sage: import tempfile + sage: with tempfile.TemporaryDirectory() as directory: + ....: options = BuildOptions(output_dir=Path(directory), source_dir=Path('src/doc')) + ....: builder = DocBuilder('en/tutorial', options) + ....: builder._output_dir('html') + ...Path('.../html/en/tutorial') """ - from sage.env import SAGE_DOC - d = os.path.join(SAGE_DOC, type, self.lang, self.name) - os.makedirs(d, exist_ok=True) - return d + dir = self._options.output_dir / type / self.name + dir.mkdir(parents=True, exist_ok=True) + return dir - def _doctrees_dir(self): + def _doctrees_dir(self) -> Path: """ Return the directory where the doctrees are stored. @@ -229,14 +197,17 @@ def _doctrees_dir(self): EXAMPLES:: sage: from sage_docbuild.builders import DocBuilder - sage: b = DocBuilder('tutorial') - sage: b._doctrees_dir() # optional - sagemath_doc_html - '.../doctrees/en/tutorial' - """ - from sage.env import SAGE_DOC - d = os.path.join(SAGE_DOC, 'doctrees', self.lang, self.name) - os.makedirs(d, exist_ok=True) - return d + sage: from sage_docbuild.build_options import BuildOptions + sage: import tempfile + sage: with tempfile.TemporaryDirectory() as directory: + ....: options = BuildOptions(output_dir=Path(directory), source_dir=Path('src/doc')) + ....: builder = DocBuilder('en/tutorial', options) + ....: builder._doctrees_dir() + ...Path('.../doctrees/en/tutorial') + """ + dir = self._options.output_dir / 'doctrees' / self.name + dir.mkdir(parents=True, exist_ok=True) + return dir def _output_formats(self): """ @@ -245,8 +216,10 @@ def _output_formats(self): EXAMPLES:: sage: from sage_docbuild.builders import DocBuilder - sage: b = DocBuilder('tutorial') - sage: b._output_formats() + sage: from sage_docbuild.build_options import BuildOptions + sage: options = BuildOptions(source_dir=Path('src/doc')) + sage: builder = DocBuilder('tutorial', options) + sage: builder._output_formats() ['changes', 'html', 'htmlhelp', 'inventory', 'json', 'latex', 'linkcheck', 'pickle', 'web'] """ # Go through all the attributes of self and check to @@ -269,8 +242,10 @@ def pdf(self): EXAMPLES:: sage: from sage_docbuild.builders import DocBuilder - sage: b = DocBuilder('tutorial') - sage: b.pdf() #not tested + sage: from sage_docbuild.build_options import BuildOptions + sage: options = BuildOptions(source_dir = Path('src/doc')) + sage: builder = DocBuilder('tutorial', options) + sage: builder.pdf() #not tested """ self.latex() tex_dir = self._output_dir('latex') @@ -278,7 +253,7 @@ def pdf(self): if self.name == 'reference': # recover maths in tex, undoing what Sphinx did (trac #29993) - tex_file = os.path.join(tex_dir, 'reference.tex') + tex_file = tex_dir / 'reference.tex' with open(tex_file) as f: ref = f.read() ref = re.sub(r'\\textbackslash{}', r'\\', ref) @@ -333,99 +308,6 @@ def build_many(target, args, processes=None): ########################################## # Parallel Building Ref Manual # ########################################## - -def build_other_doc(args): - document = args[0] - name = args[1] - kwds = args[2] - args = args[3:] - logger.warning("\nBuilding %s.\n" % document) - getattr(get_builder(document), name)(*args, **kwds) - - -class AllBuilder(): - """ - A class used to build all of the documentation. - """ - def __getattr__(self, attr): - """ - For any attributes not explicitly defined, we just go through - all of the documents and call their attr. For example, - 'AllBuilder().json()' will go through all of the documents - and call the json() method on their builders. - """ - from functools import partial - return partial(self._wrapper, attr) - - def _wrapper(self, name, *args, **kwds): - """ - This is the function which goes through all of the documents - and does the actual building. - """ - start = time.time() - docs = self.get_all_documents() - refs = [x for x in docs if x.endswith('reference')] - others = [x for x in docs if not x.endswith('reference')] - - # Build the reference manual twice to resolve references. That is, - # build once with the inventory builder to construct the intersphinx - # inventory files, and then build the second time for real. So the - # first build should be as fast as possible; - logger.warning("\nBuilding reference manual, first pass.\n") - for document in refs: - getattr(get_builder(document), 'inventory')(*args, **kwds) - - from sage.env import SAGE_DOC - logger.warning("Building reference manual, second pass.\n") - os.makedirs(os.path.join(SAGE_DOC, "html", "en", "reference", "_static"), exist_ok=True) - for document in refs: - getattr(get_builder(document), name)(*args, **kwds) - - # build the other documents in parallel - L = [(doc, name, kwds) + args for doc in others] - - # Issue #31344: Work around crashes from multiprocessing - if sys.platform == 'darwin': - for target in L: - build_other_doc(target) - else: - build_many(build_other_doc, L) - logger.warning("Elapsed time: %.1f seconds." % (time.time() - start)) - logger.warning("Done building the documentation!") - - def get_all_documents(self): - """ - Return a list of all of the documents. - - A document is a directory within one of the language - subdirectories of SAGE_DOC_SRC specified by the global - LANGUAGES variable. - - EXAMPLES:: - - sage: from sage_docbuild.builders import AllBuilder - sage: documents = AllBuilder().get_all_documents() - sage: 'en/tutorial' in documents # optional - sage_spkg - True - sage: documents[0] == 'en/reference' - True - """ - documents = [] - for lang in build_options.LANGUAGES: - for document in os.listdir(os.path.join(SAGE_DOC_SRC, lang)): - if (document not in build_options.OMIT - and os.path.isdir(os.path.join(SAGE_DOC_SRC, lang, document))): - documents.append(os.path.join(lang, document)) - - # Ensure that the reference guide is compiled first so that links from - # the other documents to it are correctly resolved. - if 'en/reference' in documents: - documents.remove('en/reference') - documents.insert(0, 'en/reference') - - return documents - - class WebsiteBuilder(DocBuilder): def html(self): """ @@ -502,28 +384,21 @@ def clean(self): DocBuilder.clean(self) -class ReferenceBuilder(AllBuilder): +class ReferenceBuilder(): """ - This class builds the reference manual. It uses DocBuilder to + This class builds the reference manual. It uses DocBuilder to build the top-level page and ReferenceSubBuilder for each sub-component. """ - def __init__(self, name, lang='en'): + def __init__(self, name:str, options: BuildOptions): """ Record the reference manual's name, in case it's not identical to 'reference'. """ - AllBuilder.__init__(self) - doc = name.split(os.path.sep) - - if doc[0] in build_options.LANGUAGES: - lang = doc[0] - doc.pop(0) + self.name = name + self.options = options - self.name = doc[0] - self.lang = lang - - def _output_dir(self, type, lang=None): + def _output_dir(self, type: Literal['html', 'latex', 'pdf']) -> Path: """ Return the directory where the output of type ``type`` is stored. @@ -533,19 +408,20 @@ def _output_dir(self, type, lang=None): EXAMPLES:: sage: from sage_docbuild.builders import ReferenceBuilder - sage: b = ReferenceBuilder('reference') - sage: b._output_dir('html') # optional - sagemath_doc_html - '.../html/en/reference' + sage: from sage_docbuild.build_options import BuildOptions + sage: import tempfile + sage: with tempfile.TemporaryDirectory() as directory: + ....: options = BuildOptions(output_dir = Path(directory)) + ....: builder = ReferenceBuilder('reference', options) + ....: builder._output_dir('html') + ...Path('.../html/reference') """ - from sage.env import SAGE_DOC - if lang is None: - lang = self.lang - d = os.path.join(SAGE_DOC, type, lang, self.name) - os.makedirs(d, exist_ok=True) - return d + dir = self.options.output_dir / type / self.name + dir.mkdir(parents=True, exist_ok=True) + return dir - def _refdir(self): - return os.path.join(SAGE_DOC_SRC, self.lang, self.name) + def _source_dir(self) -> Path: + return self.options.source_dir / self.name def _build_bibliography(self, format, *args, **kwds): """ @@ -554,9 +430,8 @@ def _build_bibliography(self, format, *args, **kwds): The bibliography references.aux is referenced by the other manuals and needs to be built first. """ - refdir = self._refdir() references = [ - (doc, self.lang, format, kwds) + args for doc in self.get_all_documents(refdir) + (doc, 'en', format, kwds) + args for doc in get_all_documents(self._source_dir()) if doc == 'reference/references' ] build_many(build_ref_doc, references) @@ -565,10 +440,9 @@ def _build_everything_except_bibliography(self, format, *args, **kwds): """ Build the entire reference manual except the bibliography """ - refdir = self._refdir() non_references = [ - (doc, self.lang, format, kwds) + args for doc in self.get_all_documents(refdir) - if doc != 'reference/references' + (doc, 'en', format, kwds) + args for doc in get_all_documents(self._source_dir()) + if doc != Path('reference/references') ] build_many(build_ref_doc, non_references) @@ -576,18 +450,17 @@ def _build_top_level(self, format, *args, **kwds): """ Build top-level document. """ - getattr(ReferenceTopBuilder('reference'), format)(*args, **kwds) + getattr(ReferenceTopBuilder('reference', self.options), format)(*args, **kwds) def _wrapper(self, format, *args, **kwds): """ - Build reference manuals: build the - top-level document and its components. + Build reference manuals: build the top-level document and its components. """ logger.info('Building bibliography') self._build_bibliography(format, *args, **kwds) logger.info('Bibliography finished, building dependent manuals') self._build_everything_except_bibliography(format, *args, **kwds) - # The html refman must be build at the end to ensure correct + # The html refman must be built at the end to ensure correct # merging of indexes and inventories. # Sphinx is run here in the current process (not in a # subprocess) and the IntersphinxCache gets populated to be @@ -595,67 +468,12 @@ def _wrapper(self, format, *args, **kwds): # the other documents. self._build_top_level(format, *args, **kwds) - def get_all_documents(self, refdir): - """ - Return a list of all reference manual components to build. - - We add a component name if it's a subdirectory of the manual's - directory and contains a file named 'index.rst'. - - We return the largest component (most subdirectory entries) - first since they will take the longest to build. - - EXAMPLES:: - - sage: from sage_docbuild.builders import ReferenceBuilder - sage: b = ReferenceBuilder('reference') - sage: refdir = os.path.join(os.environ['SAGE_DOC_SRC'], 'en', b.name) # optional - sage_spkg - sage: sorted(b.get_all_documents(refdir)) # optional - sage_spkg - ['reference/algebras', - 'reference/arithgroup', - ..., - 'reference/valuations'] - """ - documents = [] - - for doc in os.listdir(refdir): - directory = os.path.join(refdir, doc) - if os.path.exists(os.path.join(directory, 'index.rst')): - n = len(os.listdir(directory)) - documents.append((-n, os.path.join(self.name, doc))) - - return [doc[1] for doc in sorted(documents)] - - class ReferenceTopBuilder(DocBuilder): """ This class builds the top-level page of the reference manual. """ - def __init__(self, *args, **kwds): - DocBuilder.__init__(self, *args, **kwds) - self.name = 'reference' - self.lang = 'en' - - def _output_dir(self, type, lang=None): - """ - Return the directory where the output of type ``type`` is stored. - - If the directory does not exist, then it will automatically be - created. - - EXAMPLES:: - - sage: from sage_docbuild.builders import ReferenceTopBuilder - sage: b = ReferenceTopBuilder('reference') - sage: b._output_dir('html') # optional - sagemath_doc_html - '.../html/en/reference' - """ - from sage.env import SAGE_DOC - if lang is None: - lang = self.lang - d = os.path.join(SAGE_DOC, type, lang, self.name) - os.makedirs(d, exist_ok=True) - return d + def __init__(self, name: str, options: BuildOptions): + DocBuilder.__init__(self, 'en/reference', options) def html(self): """ @@ -666,25 +484,19 @@ def html(self): # We want to build master index file which lists all of the PDF file. # We modify the file index.html from the "reference_top" target, if it # exists. Otherwise, we are done. - - from sage.env import SAGE_DOC - reference_dir = os.path.join(SAGE_DOC, 'html', 'en', 'reference') output_dir = self._output_dir('html') - with open(os.path.join(reference_dir, 'index.html')) as f: - html = f.read() - # Install in output_dir a symlink to the directory containing static files. # Prefer relative path for symlinks. - relpath = os.path.relpath(reference_dir, output_dir) + relpath = output_dir.relative_to(self._options.output_dir) try: - os.symlink(os.path.join(relpath, '_static'), os.path.join(output_dir, '_static')) + (output_dir / '_static').symlink_to(relpath / '_static') except FileExistsError: pass # Now modify top reference index.html page and write it to output_dir. - html_output_dir = os.path.dirname(reference_dir) - + with open(output_dir / 'index.html') as f: + html = f.read() # Fix links in navigation bar html = re.sub(r'Sage(.*)Documentation', r'Sage\2Documentation', @@ -700,7 +512,7 @@ def html(self): # For the content, we modify doc/en/reference/index.rst, which # has two parts: the body and the table of contents. - with open(os.path.join(SAGE_DOC_SRC, self.lang, 'reference', 'index.rst')) as f: + with open(self.dir / 'index.rst') as f: rst = f.read() # Get rid of todolist and miscellaneous rst markup. rst = rst.replace('.. _reference-manual:\n\n', '') @@ -744,7 +556,7 @@ def html(self): rst_toc = re.sub(r'\n([A-Z][a-zA-Z, ]*)\n[-]*\n', r'\n\n\n

\1

\n\n