Mac CI followup: docs & naming

mac_pw_pool/AllocateTestDH.sh

@@ -90,10 +90,10 @@ modargs=(
     "DH_PFX              $INST_NAME     $DH_PFX"
     # The default launch template name includes $DH_PFX, ensure the production template name is used.
     # N/B: The old/unmodified pw_lib.sh is still loaded for the running script
-    "TEMPLATE_NAME       $TEMPLATE_NAME Cirrus${DH_PFX}PWinstance"
+    "TEMPLATE_NAME       $TEMPLATE_NAME GHA${DH_PFX}Runner"
     # Permit developer to use instance for up to 3 days max (orphan vm cleaning process will nail it after that).
     "PW_MAX_HOURS        72             $PW_MAX_HOURS"
-    # Permit developer to execute as many Cirrus-CI tasks as they want w/o automatic shutdown.
+    # Permit developer to execute as many GHA jobs as they want w/o automatic shutdown.
     "PW_MAX_TASKS        9999           $PW_MAX_TASKS"
 )
 
@@ -170,7 +170,7 @@ done
 # At this point the script could call SetupInstances.sh in another loop
 # but it takes about 20-minutes to complete.  Also, the developer may
 # not need it, they may simply want to ssh into the instance to poke
-# around.  i.e. they don't need to run any Cirrus-CI jobs on the test
+# around.  i.e. they don't need to run any GHA jobs on the test
 # instance.
 warn "---"
 warn "NOT copying/running setup.sh to new instance (in case manual activities are desired)."
@@ -185,12 +185,13 @@ msg "Dropping you into a shell inside a temp. repo clone:
 ($TMP_CLONE_DIRPATH/$LIB_DIRNAME)"
 msg "---"
 msg "Once it finishes booting (5m), you may use './InstanceSSH.sh ${INST_NAME}-0'
-to access it.  Otherwise to fully setup the instance for Cirrus-CI, you need
+to access it.  Otherwise to fully setup the instance for GHA, you need
 to execute './SetupInstances.sh' repeatedly until the ${INST_NAME}-0 line in
 'pw_status.txt' includes the text 'complete alive'.  That process can take 20+
-minutes.  Once alive, you may then use Cirrus-CI to test against this specific
+minutes.  Once alive, you may then use GHA to test against this specific
 instance with any 'persistent_worker' task having a label of
-'$DH_REQ_TAG=$DH_REQ_VAL' set."
+'$DH_REQ_TAG=$DH_REQ_VAL' set. You can target this instance by opening a PR
+with a workflow that contains a job that runs-on: group: $DH_REQ_VAL"
 msg "---"
 warn "$w"
 

mac_pw_pool/LaunchInstances.sh

@@ -4,7 +4,7 @@ set -eo pipefail
 
 # Script intended to be executed by humans (and eventually automation) to
 # ensure instances are launched from the current template version, on all
-# available Cirrus-CI Persistent Worker M1 Mac dedicated hosts.  These
+# available GitHub Actions self hosted runners on M1 Mac dedicated hosts.  These
 # dedicated host (slots) are selected at runtime based their possessing a
 # 'true' value for their `PWPoolReady` tag.  The script assumes:
 #

mac_pw_pool/README.md

@@ -1,4 +1,4 @@
-# Cirrus-CI persistent worker maintenance
+# GitHub Actions self-hosted runners
 
 These scripts are intended to be used from a repository clone,
 by cron, on an always-on cloud machine.  They make a lot of
@@ -10,14 +10,15 @@ detailed/specific information.
 
 * The `aws` binary present somewhere on `$PATH`.
 * Standard AWS `credentials` and `config` files exist under `~/.aws`
-  and set the region to `us-east-1`.
-* A copy of the ssh-key referenced by `CirrusMacM1PWinstance` launch template
+  and set the region to `us-east-1`. If you are debugging on your local machine,
+  and not running in prod, you can also use `aws login` with your own creds.
+* A copy of the ssh-key referenced by `GHAMacM1Runner` launch template
   under "Assumptions" below.
 * The ssh-key has been added to a running ssh-agent.
 * The running ssh-agent sh-compatible env. vars. are stored in
   `/run/user/$UID/ssh-agent.env`
-* The env. var. `POOLTOKEN` is set to the Cirrus-CI persistent worker pool
-  token value.
+* The env. var. `GITHUB_TOKEN` is set. This token should have read and write access
+to organization `podman-container-tools` self hosted runners.
 
 ## Assumptions
 
@@ -34,16 +35,47 @@ detailed/specific information.
   * Disabled "Instance auto-placement", "Host recovery", and "Host maintenance"
   * Quantity: 1
   * Tags: `automation=false`, `purpose=prod`, and `PWPoolReady=true`
-* The EC2 `CirrusMacM1PWinstance` instance-template exists and sets:
+* The EC2 `GHAMacM1Runner` instance-template exists and sets:
   * Shutdown-behavior: terminate
   * Same "key pair" referenced under `Prerequisites`
   * All other required instance parameters complete
   * A user-data script that shuts down the instance after 2 days.
 
+## Glossary
+
+### GitHub Actions and Cirrus Terms
+These scripts were originally written for Cirrus CI using Cirrus-specific terminology, and have since been migrated to GitHub Actions.
+Within the codebase, some variables and file names may still use Cirrus terms.
+
+| GitHub Actions     | Cirrus            | Description     |
+|--------------------|-------------------|-----------------|
+| Runner             | Worker            | A system used to execute a CI run                         |
+| Self-hosted Runner | Persistent Worker | A self-deployed and managed system used to execute CI runs|
+| Runner Group       | Worker Pool       | The set of runners available to run a CI job. <br> Note that in GHA, runners can exist without being part of a group. In this document, we mostly use "pool" to refer to the set of runners/instances that will be registered under our `mac-pool` runner group. |
+| Job                | Task              | A CI run                                                  |
+
+### AWS Terms
+* **Dedicated host**: A physical server with EC2 instance capacity
+* **Instance**: A bare metal system running on a dedicated host
+
+### Relationship Between Terms
+A dedicated host is provisioned on AWS, and an AWS instance is brought up on the dedicated host.
+The instance is set up with dependencies, including the GitHub Actions runner application. 
+The GitHub Actions runner is configured and added to a runner group. In this state, the instance is considered a "runner". 
+The runner listener is started, which waits for a job.
+A workflow within the repo's `.github/workflows` directory is triggered by an event such as a PR or a tag push.
+This workflow has a job that `runs-on` a specific runner group.
+GHA selects one of the registered runners within that runner group and sends the job to the runner.
+
+The **job** is run on the **instance** (configured as a **self-hosted runner** within a **runner group**) on the **dedicated host**.
+
+**Note**: In this document, we often use "runner" and "instance" interchangeably, as they most often mean a system that is configured to run a CI job.
+Similarly, we may use "pool" to refer to the set of instances that are properly set up to run jobs and are registered under the production (`mac-pool`) runner group.
+
 ## Operation (Theory)
 
 The goal is to maintain sufficient alive/running/working instances
-to service most Cirrus-CI tasks pointing at the pool.  This is
+to service most GitHub Actions jobs pointing at the pool.  This is
 best achieved with slower maintenance of hosts compared to setup
 of ready instances.  This is because hosts can be inaccessible for
 up to 2 hours, but instances come up in ~10-20m, ready to run tasks.
@@ -53,11 +85,11 @@ setting "false" or removing their `PWPoolReady=true` tag.  Otherwise,
 the pool should be maintained by installing the crontab lines
 indicated in the `Cron.sh` script.
 
-Cirrus-CI will assign tasks (specially) targeted at the pool, to an
-instance with a running listener (`cirrus worker run` process).  If
+GH actions will assign tasks that `runs-on: group: mac-pool`, to an
+instance with a running listener (`Runner.Listener` process).  If
 there are none, the task will queue forever (there might be a 24-hour
 timeout, I can't remember). From a PR perspective, there is little
-control over which instance you get.  It could easily be one where
+control over which instance you get. It could easily be one where
 a previous task barfed all over and rendered unusable.
 
 ## Initialization
@@ -83,12 +115,15 @@ Now the `Cron.sh` cron-job may be installed, enabled and started.
 
 Verifying changes to these scripts / cron-job must be done manually.
 To support this, every dedicated host and instance has a `purpose`
-tag, which must correspond to the value indicated in `pw_lib.sh`
-and in the target repo `.cirrus.yml`.  To test script and/or
-CI changes:
+tag, which must correspond to the value indicated in `pw_lib.sh`.
+This `purpose` tag will also be the name of the runner group in GitHub actions
+the runner is registered under. 
+To test script and/or CI changes:
+
 
 1. Make sure you have locally met all requirements spelled out in the
    header-comment of `AllocateTestDH.sh`.
+1. Set `DH_REQ_VAL` to a test tag.
 1. Execute `AllocateTestDH.sh`.  It will operate out of a temporary
    clone of the repository to prevent pushing required test-modifications
    upstream.
@@ -100,13 +135,16 @@ CI changes:
    instance should have a `setup.log` file in the `ec2-user` homedir.  There
    should also be `/private/tmp/<name>-worker.log` with entries from the
    pool listener process.
-1. To test CI changes against the test instance(s), push a PR that includes
-   `.cirrus.yml` changes to the task's `persistent_worker` dictionary's
-   `purpose` attribute.  Set the value the same as the tag in step 1.
+1. To test CI changes against the test instance(s), push a PR that sets a job's
+   `runs-on: group: ` field to the value of `DH_REQ_VAL` tag that you set. 
+   Example: `DH_REQ_VAL="testing"` -> `runs-on: group: testing`
 1. When you're done with all testing, terminate the instance.  Then wait
    a full 24-hours before "releasing" the dedicated host.  Both operations
    can be performed using the AWS EC2 WebUI.  Please remember to do the
    release step, as the $-clock continues to run while it's allocated.
+1. Remove the test runner group from the podman-container-tools org. This
+   The name of the runner group should be the value of `DH_REQ_VAL` that 
+   you set previously.
 
 Note: Instances are set to auto-terminate on shutdown.  They should
 self shutdown after 24-hours automatically.  After termination for
@@ -120,7 +158,7 @@ properly.
 * On each MacOS instance:
   * The pool listener process (running as the worker user) keeps a log under `/private/tmp`.  The
     file includes the registered name of the worker.  For example, on MacM1-7 you would find `/private/tmp/MacM1-7-worker.log`.
-    This log shows tasks taken on, completed, and any errors reported back from Cirrus-CI internals.
+    This log shows tasks taken on, completed, and any errors reported back from GitHub Actions internals.
   * In the ec2-user's home directory is a `setup.log` file.  This stores the output from executing
     `setup.sh`.  It also contains any warnings/errors from the (very important) `service_pool.sh` script - which should
     _always_ be running in the background.

mac_pw_pool/SetupInstances.sh

@@ -330,12 +330,7 @@ for _dhentry in "${_dhstate[@]}"; do
                 continue  # try again next loop
             fi
 
-            if ! $SCP $CIENV_SCRIPT ec2-user@$pub_dns:./; then
-                pwst_warn "Could not scp CI Env. script to instance $(ctx 0)."
-                continue  # try again next loop
-            fi
-
-            if ! $SSH ec2-user@$pub_dns chmod +x /var/tmp/*.sh ./ci_env.sh; then
+            if ! $SSH ec2-user@$pub_dns chmod +x /var/tmp/*.sh; then
                 pwst_warn "Could not chmod scripts $(ctx 0)."
                 continue  # try again next loop
             fi

mac_pw_pool/Utilization.gnuplot

@@ -8,25 +8,21 @@
 set terminal png enhanced rounded size 1400,800 nocrop
 set output 'html/utilization.png'
 
-set title "Persistent Workers & Utilization"
+set title "Runners Online & Active"
 
 set xdata time
 set timefmt "%Y-%m-%dT%H:%M:%S+00:00"
 set xtics nomirror rotate timedate
 set xlabel "time/date"
 set xrange [(system("date -u -Iseconds -d '26 hours ago'")):(system("date -u -Iseconds"))]
 
-set ylabel "Workers Online"
+set ylabel "Number of Runners"
 set ytics border nomirror numeric
 # Not practical to lookup $DH_PFX from pw_lib.sh
-set yrange [0:(system("grep -E '^[a-zA-Z0-9]+-[0-9]' dh_status.txt | wc -l") * 1.5)]
-
-set y2label "Worker Utilization"
-set y2tics border nomirror numeric
-set y2range [0:100]
+set yrange [0:(system("grep -E '^[a-zA-Z0-9]+-[0-9]' dh_status.txt | wc -l") + 1)]
 
 set datafile separator comma
 set grid
 
-plot 'utilization.csv' using 1:2                  axis x1y1 title "Workers"     pt 7 ps 2, \
-                    '' using 1:((($3-$4)/$2)*100) axis x1y2 title "Utilization" with lines lw 2
+plot 'utilization.csv' using 1:2         title "Online" pt 7 ps 2, \
+                    '' using 1:(($3-$4)) title "Active" with lines lw 2

mac_pw_pool/html/index.html

@@ -2,11 +2,11 @@
 <html lang="en">
 <head>
   <meta charset="utf-8">
-  <title>Cirrus-CI Persistent Workers</title>
+  <title>GitHub Actions Self-Hosted Runners</title>
 </head>
 <body>
   <center>
-    <a href="https://cirrus-ci.com/pool/1cf8c7f7d7db0b56aecd89759721d2e710778c523a8c91c7c3aaee5b15b48d05">
+    <a href="https://github.com/organizations/podman-container-tools/settings/actions/runner-groups/5">
       <img src="utilization.png">
     </a>
     <p>

mac_pw_pool/pw_lib.sh

@@ -57,13 +57,12 @@ SETUP_MAX_SECONDS=2400  # Typical time ~10 minutes, use 2x safety-factor.
 
 # Name of launch template. Current/default version will be used.
 # https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#LaunchTemplates:
-TEMPLATE_NAME="${TEMPLATE_NAME:-Cirrus${DH_PFX}PWinstance}"
+TEMPLATE_NAME="${TEMPLATE_NAME:-GHA${DH_PFX}Runner}"
 
 # Path to scripts to copy/execute on Darwin instances
 SETUP_SCRIPT="$LIB_DIRPATH/setup.sh"
 SPOOL_SCRIPT="$LIB_DIRPATH/service_pool.sh"
 SHDWN_SCRIPT="$LIB_DIRPATH/shutdown.sh"
-CIENV_SCRIPT="$LIB_DIRPATH/ci_env.sh"
 CLEANUP_HOOK_SCRIPT="$LIB_DIRPATH/cleanup_hook.sh"
 
 # Set to 1 to enable debugging

mac_pw_pool/setup.sh

@@ -239,7 +239,7 @@ RUNNER_CONFIG_FILE="$RUNNER_DIR/.runner"
 if [[ ! -r "$RUNNER_CONFIG_FILE" ]]; then
     msg "Registering runner '$PWNAME'"
     # Configure the runner (but don't start it - service_pool.sh will do that)
-    # Work directory matches Cirrus pattern: /Users/$PWUSER/ci
+    # Work directory matches previous Cirrus pattern: /Users/$PWUSER/ci
     # --replace automatically removes any existing runner with the same name
     sudo -u $PWUSER bash -c "cd $RUNNER_DIR && ./config.sh \
         --unattended \