diff --git a/build-pelican/action.yml b/build-pelican/action.yml index 65f1021..c00a3bb 100644 --- a/build-pelican/action.yml +++ b/build-pelican/action.yml @@ -1,48 +1,72 @@ name: Build a Pelican Website description: "Generate a Pelican website from Markdown" inputs: - token: - description: 'Set to secrets.GITHUB_TOKEN in the workflow' - required: true destination: description: "Pelican Output branch" required: true - default: "asf-site" - gfm: - description: "Uses GitHub Flavored Markdown" + default: 'asf-site' + output: + description: "Output directory in output branch" + required: false + default: 'output' + image: + description: "Docker Image" + required: false + default: 'ghcr.io/apache/infrastructure-pelican:latest' + tempdir: + description: "Name of temporary directory. Must not exist in source or output branches." required: false - default: 'false' + default: 'output.tmp' runs: using: "composite" steps: - - name: Install Pelican - run: pip3 install pelican markdown ghp-import bs4 ezt + - uses: actions/checkout@v4 + with: + filter: tree:0 # This reduces download time whilst still giving access to other branches + fetch-depth: 0 # Required for multi-branch checkouts + show-progress: false # does not yet work for checkout unfortunately + - name: Identify Committer shell: bash - - # If the site uses Github Flavored Markdown, use this build branch - - name: fetch libcmark-gfm.so buildscript - run: wget https://raw.githubusercontent.com/apache/infrastructure-pelican/master/bin/build-cmark.sh -O ./build-cmark.sh + run: | + git config --global user.email "users@infra.apache.org" + git config --global user.name "Build Pelican (action)" + - name: Run Docker Pelican build => ${{ inputs.tempdir }} shell: bash - if: ${{ inputs.gfm == 'true' }} - - - name: build libcmark-gfm.so - run: /bin/bash ./build-cmark.sh + run: | + docker run --quiet -v $PWD:/site --entrypoint bash ${{ inputs.image }} \ + -c "source /tmp/pelican-asf/LIBCMARKDIR.sh && python3 -m pelican content -o ${{ inputs.tempdir }}" + - name: Checkout site deploy branch shell: bash - if: ${{ inputs.gfm == 'true' }} - - - name: Generate website from markdown - run: /usr/bin/env python3 -m pelican content -o output -D + run: | + if git checkout ${{ inputs.destination }} + then + git pull origin ${{ inputs.destination }} + else + echo "branch ${{ inputs.destination }} is new; create basic site" + git config --global --add --bool push.autoSetupRemote true + git checkout --orphan ${{ inputs.destination }} -f + git rm -rf . + # TODO: does it make sense to copy the source version of the file? + # What is actually needed by the site branch? + # assume we have an asf.yaml file + git checkout origin/${{ github.ref_name }} -- .asf.yaml + git add .asf.yaml -f + fi + - name: Commit changes from ${{ inputs.tempdir }} shell: bash - env: # in case we are using GFM (otherwise ignored) - LIBCMARKDIR: "cmark-gfm-0.28.3.gfm.12/lib" - - - name: Open a PR against the staging branch - uses: peter-evans/create-pull-request@v6.0.5 - with: - token: ${{ inputs.token }} - commit-message: GitHub Actions Generated Pelican Build - title: Generated Pelican Output - body: output generated - add-paths: | - output/ - base: ${{ inputs.destination }} + run: | + # Remove all existing output so deletions will be captured + rm -rf ${{ inputs.output }} + git rm --quiet -r --ignore-unmatch --cached ${{ inputs.output }}/* + # replace with generated output + mv ${{ inputs.tempdir }} ${{ inputs.output }} + git diff # Show changes + git add ${{ inputs.output }} + git status + if git commit -m "Commit build products" + then + git push + else + echo "No change" + true # ensure step is successful + fi diff --git a/pelican/migration/build-pelican.ezt b/pelican/migration/build-pelican.ezt new file mode 100644 index 0000000..e611a74 --- /dev/null +++ b/pelican/migration/build-pelican.ezt @@ -0,0 +1,18 @@ +[# TEMPLATE for the workflow file build-pelican.yml. Must be generated with compress-whitespace off.] +name: Build a Pelican Website +on: + workflow_dispatch: + push: + paths: + - 'content/**' # can this vary? + +permissions: + contents: write # needed to commit updates to the output branch + +jobs: + build-pelican: + runs-on: ubuntu-latest + steps: + - uses: 'apache/infrastructure-actions/build-pelican@main' + with: + destination: '[destination]' diff --git a/pelican/migration/build-pelican.yml b/pelican/migration/build-pelican.yml deleted file mode 100644 index 5ff47a2..0000000 --- a/pelican/migration/build-pelican.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Build a Pelican Website -on: - push: - branches: [ "dfoulks/pelican_gha" ] - workflow_dispatch: -jobs: - build-pelican: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - with: - ref: 'dfoulks/pelican_gha' - - uses: apache/infrastructure-actions/build-pelican@main - with: - destination: 'dfoulks/gha-site' - gfm: 'true' - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/pelican/migration/generate_settings.py b/pelican/migration/generate_settings.py index 74f5c5a..5a5eb5a 100644 --- a/pelican/migration/generate_settings.py +++ b/pelican/migration/generate_settings.py @@ -1,30 +1,32 @@ import argparse import os -import shutil -import glob +import fnmatch import datetime import yaml import ezt -# Command definitions - put into a conf later on? -GIT = "/usr/bin/git" -SVN = "/usr/bin/svn" -BASH = "/bin/bash" SCRATCH_DIR = "/tmp" -PLUGINS = "/opt/infrastructure-pelican/plugins" THIS_DIR = os.path.abspath(os.path.dirname(__file__)) # Automatic settings filenames AUTO_SETTINGS_YAML = "pelicanconf.yaml" AUTO_SETTINGS_TEMPLATE = "pelican.auto.ezt" AUTO_SETTINGS = "pelicanconf.py" +BUILD_PELICAN_TEMPLATE = "build-pelican.ezt" class _helper: def __init__(self, **kw): vars(self).update(kw) +def find(pattern, path): + for _root, _dirs, files in os.walk(path): + for name in files: + if fnmatch.fnmatch(name, pattern): + return True + return False + def generate_settings(source_yaml, settings_path, builtin_p_paths=None, sourcepath="."): """Generate the Pelican settings file @@ -34,10 +36,10 @@ def generate_settings(source_yaml, settings_path, builtin_p_paths=None, sourcepa :param sourcepath: path to source (defaults to '.') """ - print(f"Reading {source_yaml}") - ydata = yaml.safe_load(open(source_yaml)) + print(f"Reading {source_yaml} in {sourcepath}") + ydata = yaml.safe_load(open(source_yaml, encoding='utf-8')) - print(f"converting to pelican.auto.py...") + print(f"Converting to {settings_path}...") tdata = ydata["site"] # Easy to copy these simple values. tdata.update( { @@ -58,43 +60,38 @@ def generate_settings(source_yaml, settings_path, builtin_p_paths=None, sourcepa if builtin_p_paths is None: builtin_p_paths = [] - tdata["p_paths"] = builtin_p_paths - tdata["use"] = ["gfm"] + tdata["p_paths"] = builtin_p_paths + tdata["use"] = ["gfm"] tdata["uses_sitemap"] = None if "plugins" in ydata: if "paths" in ydata["plugins"]: - tdata["p_paths"] = [] for p in ydata["plugins"]["paths"]: tdata["p_paths"].append(os.path.join(p)) - else: - tdata["p_paths"] = ["plugins"] - - if "use" in ydata["plugins"]: - tdata["use"] = ydata["plugins"]["use"] - else: - tdata["use"] = [] - - if "sitemap" in ydata["plugins"]: - sm = ydata["plugins"]["sitemap"] - sitemap_params = _helper( - exclude=str(sm["exclude"]), - format=sm["format"], - priorities=_helper( - articles=sm["priorities"]["articles"], - indexes=sm["priorities"]["indexes"], - pages=sm["priorities"]["pages"], - ), - changefreqs=_helper( - articles=sm["changefreqs"]["articles"], - indexes=sm["changefreqs"]["indexes"], - pages=sm["changefreqs"]["pages"], - ), - ) - tdata["uses_sitemap"] = "yes" # ezt.boolean - tdata["sitemap"] = sitemap_params - tdata["use"].append("sitemap") # add the plugin + if "use" in ydata["plugins"]: + tdata["use"] = ydata["plugins"]["use"] + + if "sitemap" in ydata["plugins"]: + sm = ydata["plugins"]["sitemap"] + sitemap_params = _helper( + exclude=str(sm["exclude"]), + format=sm["format"], + priorities=_helper( + articles=sm["priorities"]["articles"], + indexes=sm["priorities"]["indexes"], + pages=sm["priorities"]["pages"], + ), + changefreqs=_helper( + articles=sm["changefreqs"]["articles"], + indexes=sm["changefreqs"]["indexes"], + pages=sm["changefreqs"]["pages"], + ), + ) + + tdata["uses_sitemap"] = "yes" # ezt.boolean + tdata["sitemap"] = sitemap_params + tdata["use"].append("sitemap") # add the plugin tdata["uses_index"] = None if "index" in tdata: @@ -117,85 +114,82 @@ def generate_settings(source_yaml, settings_path, builtin_p_paths=None, sourcepa else: tdata["uses_genid"] = None - tdata["uses_data"] = None - tdata["uses_run"] = None - tdata["uses_postrun"] = None - tdata["uses_ignore"] = None - tdata["uses_copy"] = None + tdata["uses_data"] = None + tdata["uses_run"] = None + tdata["uses_postrun"] = None + tdata["uses_ignore"] = None + tdata["uses_copy"] = None if "setup" in ydata: sdata = ydata["setup"] - else: - sdata = {} - - # Load data structures into the pelican METADATA. - if "data" in sdata: - tdata["uses_data"] = "yes" # ezt.boolean() - tdata["asfdata"] = sdata["data"] - tdata["use"].append("asfdata") # add the plugin - # Run the included scripts with the asfrun plugin during initialize - if "run" in sdata: - tdata["uses_run"] = "yes" # ezt.boolean - tdata["run"] = sdata["run"] - tdata["use"].append("asfrun") # add the plugin - # Run the included scripts with the asfrun plugin during finalize - if "postrun" in sdata: - tdata["uses_postrun"] = "yes" # ezt.boolean - tdata["postrun"] = sdata["postrun"] - if not "run" in sdata: - tdata["use"].append("asfrun") # add the plugin (if not already added) - # Ignore files avoids copying these files to output. - if "ignore" in sdata: - tdata["uses_ignore"] = "yes" # ezt.boolean - tdata["ignore"] = sdata["ignore"] - # No plugin needed. - # Copy directories to output. - if "copy" in sdata: - tdata["uses_copy"] = "yes" # ezt.boolean - tdata["copy"] = sdata["copy"] - tdata["use"].append("asfcopy") # add the plugin + # Load data structures into the pelican METADATA. + if "data" in sdata: + tdata["uses_data"] = "yes" # ezt.boolean() + tdata["asfdata"] = sdata["data"] + tdata["use"].append("asfdata") # add the plugin + # Run the included scripts with the asfrun plugin during initialize + if "run" in sdata: + tdata["uses_run"] = "yes" # ezt.boolean + tdata["run"] = sdata["run"] + tdata["use"].append("asfrun") # add the plugin + # Run the included scripts with the asfrun plugin during finalize + if "postrun" in sdata: + tdata["uses_postrun"] = "yes" # ezt.boolean + tdata["postrun"] = sdata["postrun"] + if not "run" in sdata: + tdata["use"].append("asfrun") # add the plugin (if not already added) + # Ignore files avoids copying these files to output. + if "ignore" in sdata: + tdata["uses_ignore"] = "yes" # ezt.boolean + tdata["ignore"] = sdata["ignore"] + # No plugin needed. + # Copy directories to output. + if "copy" in sdata: + tdata["uses_copy"] = "yes" # ezt.boolean + tdata["copy"] = sdata["copy"] + tdata["use"].append("asfcopy") # add the plugin # if ezmd files are present then use the asfreader plugin - ezmd_count = len(glob.glob(f"{sourcepath}/**/*.ezmd", recursive=True)) - if ezmd_count > 0: + if find('*.ezmd', sourcepath): tdata["use"].append("asfreader") # add the plugin - print(f"Writing converted settings to {os.path.join(THIS_DIR, AUTO_SETTINGS)})") - if len(tdata["use"]) > 0: - if not os.path.isdir(tdata["p_paths"][0]): - os.mkdir(tdata["p_paths"][0]) - else: - print("Plugins directory found!") - - for plugin in tdata["use"]: - src = os.path.join(os.path.abspath(os.path.join(THIS_DIR, os.pardir)), f"plugins/{plugin}.py") - dest = tdata["p_paths"][0] - shutil.copy(src, dest) - if not os.path.isdir(".github/workflows"): - if not os.path.isdir(".github"): - os.mkdir(".github") - os.mkdir("./.github/workflows") - shutil.copy(os.path.join(THIS_DIR, "build-pelican.yml"), "./.github/workflows/") - + # We assume that pelicanconf.yaml is at the top level of the repo + # so .asf.yaml amd .github/workflow are located under sourcepath + workflows = os.path.join(sourcepath, ".github/workflows") + if not os.path.isdir(workflows): + print(f"Creating directory {workflows}") + os.makedirs(workflows) + workfile = f"{workflows}/build-pelican.yml" + print(f"Creating workfile {workfile} from .asf.yaml") + workfiletemplate = ezt.Template(os.path.join(THIS_DIR, BUILD_PELICAN_TEMPLATE), 0) + asfyaml = os.path.join(sourcepath,'.asf.yaml') + with open(workfile, "w", encoding='utf-8') as w: + workfiletemplate.generate(w, { + 'destination': yaml.safe_load(open(asfyaml, encoding='utf-8'))['pelican']['target'] + }) + + print(f"Writing converted settings to {settings_path}") t = ezt.Template(os.path.join(THIS_DIR, AUTO_SETTINGS_TEMPLATE)) - t.generate(open(os.path.join(".", AUTO_SETTINGS), "w+"), tdata) + with open(settings_path, "w", encoding='utf-8') as w: + t.generate(w, tdata) -if __name__ == "__main__": +def main(): parser = argparse.ArgumentParser(description="Convert pelicanconf.yaml to pelicanconf.py") - parser.add_argument('-p', '--project', required=True, help="Owning Project") + parser.add_argument('-p', '--project', required=False, help="Owning Project") # ignored,can be deleted parser.add_argument('-y', '--yaml', required=True, help="Pelicanconf YAML file") args = parser.parse_args() - sourcepath = THIS_DIR - tool_dir = THIS_DIR - - path = os.path.join(SCRATCH_DIR, args.project) - content_dir = os.path.join(sourcepath, "content") - settings_dir = sourcepath pelconf_yaml = args.yaml - #pelconf_yaml = os.path.join(sourcepath, AUTO_SETTINGS_YAML) + sourcepath = os.path.dirname(pelconf_yaml) + if sourcepath == '': + sourcepath = '.' # Needed for find if os.path.exists(pelconf_yaml): print(f"found {pelconf_yaml}") - settings_path = os.path.join(path, AUTO_SETTINGS) - builtin_plugins = os.path.join(tool_dir, os.pardir, "plugins") - generate_settings(pelconf_yaml, settings_path, [builtin_plugins], settings_dir) + settings_path = os.path.join(sourcepath, AUTO_SETTINGS) + builtin_plugins = '/tmp/pelican-asf/plugins' # Where the Docker plugins are currently + generate_settings(pelconf_yaml, settings_path, [builtin_plugins], sourcepath) + else: + print(f"Unable to find {pelconf_yaml}") + +if __name__ == "__main__": + main()