Merge asfpostrun into asfrun; add unit tests

.github/workflows/postrun.yml

@@ -0,0 +1,35 @@
+name: Unit test postrun update
+on:
+  push:
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        # Not sure it's worth testing on multiple Pythons
+        python-version: [3.8]
+    steps:
+    - uses: actions/checkout@master
+      with:
+        persist-credentials: false
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pelican[Markdown]==4.5.4
+        pip install -r requirements.txt
+        pip install 'MarkupSafe<2.1.0' # needed for Pelican 4.5.4
+    - name: Run test
+      run: |
+        cd tests/postrun
+        bash runtest.sh

bin/buildsite.py

@@ -381,11 +381,12 @@ def generate_settings(source_yaml, settings_path, builtin_p_paths=None, sourcepa
             tdata['uses_run'] = 'yes'  # ezt.boolean
             tdata['run'] = sdata['run']
             tdata['use'].append('asfrun')  # add the plugin
-        # Run the included scripts with the asfpostrun plugin during finalize
+        # Run the included scripts with the asfrun plugin during finalize
         if 'postrun' in sdata:
             tdata['uses_postrun'] = 'yes'  # ezt.boolean
             tdata['postrun'] = sdata['postrun']
-            tdata['use'].append('asfpostrun')  # add the plugin
+            if not 'run' in sdata:
+                tdata['use'].append('asfrun')  # add the plugin (if not already added)
         # Ignore files avoids copying these files to output.
         if 'ignore' in sdata:
             tdata['uses_ignore'] = 'yes'  # ezt.boolean

bin/pelican.auto.ezt

@@ -106,11 +106,11 @@ ASF_DATA = {
 }
 [end]
 [if-any uses_run]
-# Configure the asfrun plugin
+# Configure the asfrun plugin (initialization)
 ASF_RUN = [ [for run]'[run]', [end] ]
 [end]
 [if-any uses_postrun]
-# Configure the asfpostrun plugin
+# Configure the asfrun plugin (finalization)
 ASF_POSTRUN = [ [for postrun]'[postrun]', [end] ]
 [end]
 [if-any uses_ignore]

plugins/PLUGIN_ARCH.md

@@ -71,12 +71,12 @@ At a high level consider the following sequence of events:
    ]
    ```
 
-   - The [asfpostrun plugin](./asfpostrun.py) runs shell scripts during finalization
+   - The [asfrun plugin](./asfrun.py) also runs shell scripts during finalization
 
    ```yaml
    setup:
      postrun:
-       - /bin/bash shell.sh
+       - /bin/bash postshell.sh
    ```
 
    ```python

plugins/asfrun.py

@@ -17,9 +17,10 @@
 # under the License.
 #
 #
-# asfrun.py - Pelican plugin that runs shell scripts during initialization
+# asfrun.py - Pelican plugin that runs shell scripts during initialization or finalization
 #
 
+import os
 import sys
 import subprocess
 import shlex
@@ -30,20 +31,27 @@
 
 
 # open a subprocess
-def os_run(args):
-    return subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True)
-
+def os_run(args, env=None):
+    return subprocess.Popen(args, env=env, stdout=subprocess.PIPE, universal_newlines=True)
 
 # run shell
-def run_script(pel_ob):
-    asf_run = pel_ob.settings.get('ASF_RUN')
-    if asf_run:
-        print('-----\nasfrun')
-        for command in asf_run:
+def run_script(pel_ob, command_source, env=False):
+    commands = pel_ob.settings.get(command_source)
+    if commands:
+        print(f'-----\nasfrun {command_source}')
+        if env:
+            # copy the pelican environment into the OS env
+            my_env = os.environ.copy()
+            for k, v in sorted(pel_ob.settings.items()):
+                if k != 'ASF_DATA': # rather large; not needed
+                    my_env['PELICAN_'+k] = str(v)
+        else:
+            my_env = None
+        for command in commands:
             print(f'-----\n{command}')
             args = shlex.split(command)
             print(args)
-            with os_run(args) as s:
+            with os_run(args, my_env) as s:
                 for line in s.stdout:
                     line = line.strip()
                     print(f'{line}')
@@ -52,7 +60,17 @@ def run_script(pel_ob):
 def tb_initialized(pel_ob):
     """ Print any exception, before Pelican chews it into nothingness."""
     try:
-        run_script(pel_ob)
+        run_script(pel_ob, 'ASF_RUN')
+    except Exception:
+        print('-----', file=sys.stderr)
+        traceback.print_exc()
+        # exceptions here stop the build
+        raise
+
+def tb_finalized(pel_ob):
+    """ Print any exception, before Pelican chews it into nothingness."""
+    try:
+        run_script(pel_ob, 'ASF_POSTRUN', env=True)
     except Exception:
         print('-----', file=sys.stderr)
         traceback.print_exc()
@@ -62,3 +80,4 @@ def tb_initialized(pel_ob):
 
 def register():
     pelican.plugins.signals.initialized.connect(tb_initialized)
+    pelican.plugins.signals.finalized.connect(tb_finalized)

tests/postrun/content/empty.md

@@ -0,0 +1,3 @@
+title: Dummy
+
+Dummy file

tests/postrun/pelican.auto.py

@@ -0,0 +1,52 @@
+
+# Basic information about the site.
+SITENAME = 'Apache Software Foundation'
+SITEDESC = 'Test run command'
+SITEDOMAIN = 'test.apache.org'
+SITEURL = 'https://test.apache.org'
+SITELOGO = 'https://test.apache.org/None'
+SITEREPOSITORY = 'None'
+CURRENTYEAR = 2023
+TRADEMARKS = 'Apache, the Apache feather logo are trademarks'
+TIMEZONE = 'UTC'
+# Theme includes templates and possibly static files
+THEME = './theme/apache'
+# Specify location of plugins, and which to use
+PLUGIN_PATHS = [ '/private/var/sebb/git/infra/pelican/bin/../plugins',  ]
+PLUGINS = [ 'gfm', 'asfrun',  ]
+# All content is located at '.' (aka content/ )
+PAGE_PATHS = [ '.' ]
+STATIC_PATHS = [ '.',  ]
+# Where to place/link generated pages
+
+PATH_METADATA = '(?P<path_no_ext>.*)\\..*'
+
+PAGE_SAVE_AS = '{path_no_ext}.html'
+# Don't try to translate
+PAGE_TRANSLATION_ID = None
+# Disable unused Pelican features
+# N.B. These features are currently unsupported, see https://github.com/apache/infrastructure-pelican/issues/49
+FEED_ALL_ATOM = None
+INDEX_SAVE_AS = ''
+TAGS_SAVE_AS = ''
+CATEGORIES_SAVE_AS = ''
+AUTHORS_SAVE_AS = ''
+ARCHIVES_SAVE_AS = ''
+# Disable articles by pointing to a (should-be-absent) subdir
+ARTICLE_PATHS = [ 'blog' ]
+# needed to create blogs page
+ARTICLE_URL = 'blog/{slug}.html'
+ARTICLE_SAVE_AS = 'blog/{slug}.html'
+# Disable all processing of .html files
+READERS = { 'html': None, }
+
+
+
+
+
+# Configure the asfrun plugin (finalization)
+ASF_POSTRUN = [ '/bin/bash postrun.sh',  ]
+
+
+
+

tests/postrun/pelicanconf.yaml

@@ -0,0 +1,15 @@
+site:
+  name: Apache Software Foundation
+  domain: test.apache.org
+  logo: None
+  repository: None
+  description: Test run command
+  trademarks: Apache, the Apache feather logo are trademarks
+
+theme: theme/apache
+
+plugins: []
+
+setup:
+  postrun:
+    - /bin/bash postrun.sh

tests/postrun/pelicanconf1.yaml

@@ -0,0 +1,15 @@
+site:
+  name: Apache Software Foundation
+  domain: test.apache.org
+  logo: None
+  repository: None
+  description: Test run command
+  trademarks: Apache, the Apache feather logo are trademarks
+
+theme: theme/apache
+
+plugins: []
+
+setup:
+  run:
+    - /bin/bash run.sh

tests/postrun/pelicanconf2.yaml

@@ -0,0 +1,15 @@
+site:
+  name: Apache Software Foundation
+  domain: test.apache.org
+  logo: None
+  repository: None
+  description: Test run command
+  trademarks: Apache, the Apache feather logo are trademarks
+
+theme: theme/apache
+
+plugins: []
+
+setup:
+  postrun:
+    - /bin/bash postrun.sh

tests/postrun/pelicanconf3.yaml

@@ -0,0 +1,17 @@
+site:
+  name: Apache Software Foundation
+  domain: test.apache.org
+  logo: None
+  repository: None
+  description: Test run command
+  trademarks: Apache, the Apache feather logo are trademarks
+
+theme: theme/apache
+
+plugins: []
+
+setup:
+  run: 
+    - /bin/bash run.sh
+  postrun:
+    - /bin/bash postrun.sh

tests/postrun/postrun.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Script to test postrun command
+
+cd temp
+cp date2.txt date2.tmp
+
+if [ -z "$PELICAN_OUTPUT_PATH" ]
+then
+    echo "PELICAN_OUTPUT_PATH is not defined!"
+    exit 1
+fi

tests/postrun/run.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Script to test run command
+
+cd temp
+cp date1.txt date1.tmp