diff --git a/mac_pw_pool/AllocateTestDH.sh b/mac_pw_pool/AllocateTestDH.sh index 3cfb4ee..2ff05f4 100755 --- a/mac_pw_pool/AllocateTestDH.sh +++ b/mac_pw_pool/AllocateTestDH.sh @@ -90,10 +90,10 @@ modargs=( "DH_PFX $INST_NAME $DH_PFX" # The default launch template name includes $DH_PFX, ensure the production template name is used. # N/B: The old/unmodified pw_lib.sh is still loaded for the running script - "TEMPLATE_NAME $TEMPLATE_NAME Cirrus${DH_PFX}PWinstance" + "TEMPLATE_NAME $TEMPLATE_NAME GHA${DH_PFX}Runner" # Permit developer to use instance for up to 3 days max (orphan vm cleaning process will nail it after that). "PW_MAX_HOURS 72 $PW_MAX_HOURS" - # Permit developer to execute as many Cirrus-CI tasks as they want w/o automatic shutdown. + # Permit developer to execute as many GHA jobs as they want w/o automatic shutdown. "PW_MAX_TASKS 9999 $PW_MAX_TASKS" ) @@ -170,7 +170,7 @@ done # At this point the script could call SetupInstances.sh in another loop # but it takes about 20-minutes to complete. Also, the developer may # not need it, they may simply want to ssh into the instance to poke -# around. i.e. they don't need to run any Cirrus-CI jobs on the test +# around. i.e. they don't need to run any GHA jobs on the test # instance. warn "---" warn "NOT copying/running setup.sh to new instance (in case manual activities are desired)." @@ -185,12 +185,13 @@ msg "Dropping you into a shell inside a temp. repo clone: ($TMP_CLONE_DIRPATH/$LIB_DIRNAME)" msg "---" msg "Once it finishes booting (5m), you may use './InstanceSSH.sh ${INST_NAME}-0' -to access it. Otherwise to fully setup the instance for Cirrus-CI, you need +to access it. Otherwise to fully setup the instance for GHA, you need to execute './SetupInstances.sh' repeatedly until the ${INST_NAME}-0 line in 'pw_status.txt' includes the text 'complete alive'. That process can take 20+ -minutes. Once alive, you may then use Cirrus-CI to test against this specific +minutes. Once alive, you may then use GHA to test against this specific instance with any 'persistent_worker' task having a label of -'$DH_REQ_TAG=$DH_REQ_VAL' set." +'$DH_REQ_TAG=$DH_REQ_VAL' set. You can target this instance by opening a PR +with a workflow that contains a job that runs-on: group: $DH_REQ_VAL" msg "---" warn "$w" diff --git a/mac_pw_pool/LaunchInstances.sh b/mac_pw_pool/LaunchInstances.sh index ce9e3f6..146c774 100755 --- a/mac_pw_pool/LaunchInstances.sh +++ b/mac_pw_pool/LaunchInstances.sh @@ -4,7 +4,7 @@ set -eo pipefail # Script intended to be executed by humans (and eventually automation) to # ensure instances are launched from the current template version, on all -# available Cirrus-CI Persistent Worker M1 Mac dedicated hosts. These +# available GitHub Actions self hosted runners on M1 Mac dedicated hosts. These # dedicated host (slots) are selected at runtime based their possessing a # 'true' value for their `PWPoolReady` tag. The script assumes: # diff --git a/mac_pw_pool/README.md b/mac_pw_pool/README.md index ebf480f..0358e9c 100644 --- a/mac_pw_pool/README.md +++ b/mac_pw_pool/README.md @@ -1,4 +1,4 @@ -# Cirrus-CI persistent worker maintenance +# GitHub Actions self-hosted runners These scripts are intended to be used from a repository clone, by cron, on an always-on cloud machine. They make a lot of @@ -10,14 +10,15 @@ detailed/specific information. * The `aws` binary present somewhere on `$PATH`. * Standard AWS `credentials` and `config` files exist under `~/.aws` - and set the region to `us-east-1`. -* A copy of the ssh-key referenced by `CirrusMacM1PWinstance` launch template + and set the region to `us-east-1`. If you are debugging on your local machine, + and not running in prod, you can also use `aws login` with your own creds. +* A copy of the ssh-key referenced by `GHAMacM1Runner` launch template under "Assumptions" below. * The ssh-key has been added to a running ssh-agent. * The running ssh-agent sh-compatible env. vars. are stored in `/run/user/$UID/ssh-agent.env` -* The env. var. `POOLTOKEN` is set to the Cirrus-CI persistent worker pool - token value. +* The env. var. `GITHUB_TOKEN` is set. This token should have read and write access +to organization `podman-container-tools` self hosted runners. ## Assumptions @@ -34,16 +35,47 @@ detailed/specific information. * Disabled "Instance auto-placement", "Host recovery", and "Host maintenance" * Quantity: 1 * Tags: `automation=false`, `purpose=prod`, and `PWPoolReady=true` -* The EC2 `CirrusMacM1PWinstance` instance-template exists and sets: +* The EC2 `GHAMacM1Runner` instance-template exists and sets: * Shutdown-behavior: terminate * Same "key pair" referenced under `Prerequisites` * All other required instance parameters complete * A user-data script that shuts down the instance after 2 days. +## Glossary + +### GitHub Actions and Cirrus Terms +These scripts were originally written for Cirrus CI using Cirrus-specific terminology, and have since been migrated to GitHub Actions. +Within the codebase, some variables and file names may still use Cirrus terms. + +| GitHub Actions | Cirrus | Description | +|--------------------|-------------------|-----------------| +| Runner | Worker | A system used to execute a CI run | +| Self-hosted Runner | Persistent Worker | A self-deployed and managed system used to execute CI runs| +| Runner Group | Worker Pool | The set of runners available to run a CI job.
Note that in GHA, runners can exist without being part of a group. In this document, we mostly use "pool" to refer to the set of runners/instances that will be registered under our `mac-pool` runner group. | +| Job | Task | A CI run | + +### AWS Terms +* **Dedicated host**: A physical server with EC2 instance capacity +* **Instance**: A bare metal system running on a dedicated host + +### Relationship Between Terms +A dedicated host is provisioned on AWS, and an AWS instance is brought up on the dedicated host. +The instance is set up with dependencies, including the GitHub Actions runner application. +The GitHub Actions runner is configured and added to a runner group. In this state, the instance is considered a "runner". +The runner listener is started, which waits for a job. +A workflow within the repo's `.github/workflows` directory is triggered by an event such as a PR or a tag push. +This workflow has a job that `runs-on` a specific runner group. +GHA selects one of the registered runners within that runner group and sends the job to the runner. + +The **job** is run on the **instance** (configured as a **self-hosted runner** within a **runner group**) on the **dedicated host**. + +**Note**: In this document, we often use "runner" and "instance" interchangeably, as they most often mean a system that is configured to run a CI job. +Similarly, we may use "pool" to refer to the set of instances that are properly set up to run jobs and are registered under the production (`mac-pool`) runner group. + ## Operation (Theory) The goal is to maintain sufficient alive/running/working instances -to service most Cirrus-CI tasks pointing at the pool. This is +to service most GitHub Actions jobs pointing at the pool. This is best achieved with slower maintenance of hosts compared to setup of ready instances. This is because hosts can be inaccessible for up to 2 hours, but instances come up in ~10-20m, ready to run tasks. @@ -53,11 +85,11 @@ setting "false" or removing their `PWPoolReady=true` tag. Otherwise, the pool should be maintained by installing the crontab lines indicated in the `Cron.sh` script. -Cirrus-CI will assign tasks (specially) targeted at the pool, to an -instance with a running listener (`cirrus worker run` process). If +GH actions will assign tasks that `runs-on: group: mac-pool`, to an +instance with a running listener (`Runner.Listener` process). If there are none, the task will queue forever (there might be a 24-hour timeout, I can't remember). From a PR perspective, there is little -control over which instance you get. It could easily be one where +control over which instance you get. It could easily be one where a previous task barfed all over and rendered unusable. ## Initialization @@ -83,12 +115,15 @@ Now the `Cron.sh` cron-job may be installed, enabled and started. Verifying changes to these scripts / cron-job must be done manually. To support this, every dedicated host and instance has a `purpose` -tag, which must correspond to the value indicated in `pw_lib.sh` -and in the target repo `.cirrus.yml`. To test script and/or -CI changes: +tag, which must correspond to the value indicated in `pw_lib.sh`. +This `purpose` tag will also be the name of the runner group in GitHub actions +the runner is registered under. +To test script and/or CI changes: + 1. Make sure you have locally met all requirements spelled out in the header-comment of `AllocateTestDH.sh`. +1. Set `DH_REQ_VAL` to a test tag. 1. Execute `AllocateTestDH.sh`. It will operate out of a temporary clone of the repository to prevent pushing required test-modifications upstream. @@ -100,13 +135,16 @@ CI changes: instance should have a `setup.log` file in the `ec2-user` homedir. There should also be `/private/tmp/-worker.log` with entries from the pool listener process. -1. To test CI changes against the test instance(s), push a PR that includes - `.cirrus.yml` changes to the task's `persistent_worker` dictionary's - `purpose` attribute. Set the value the same as the tag in step 1. +1. To test CI changes against the test instance(s), push a PR that sets a job's + `runs-on: group: ` field to the value of `DH_REQ_VAL` tag that you set. + Example: `DH_REQ_VAL="testing"` -> `runs-on: group: testing` 1. When you're done with all testing, terminate the instance. Then wait a full 24-hours before "releasing" the dedicated host. Both operations can be performed using the AWS EC2 WebUI. Please remember to do the release step, as the $-clock continues to run while it's allocated. +1. Remove the test runner group from the podman-container-tools org. This + The name of the runner group should be the value of `DH_REQ_VAL` that + you set previously. Note: Instances are set to auto-terminate on shutdown. They should self shutdown after 24-hours automatically. After termination for @@ -120,7 +158,7 @@ properly. * On each MacOS instance: * The pool listener process (running as the worker user) keeps a log under `/private/tmp`. The file includes the registered name of the worker. For example, on MacM1-7 you would find `/private/tmp/MacM1-7-worker.log`. - This log shows tasks taken on, completed, and any errors reported back from Cirrus-CI internals. + This log shows tasks taken on, completed, and any errors reported back from GitHub Actions internals. * In the ec2-user's home directory is a `setup.log` file. This stores the output from executing `setup.sh`. It also contains any warnings/errors from the (very important) `service_pool.sh` script - which should _always_ be running in the background. diff --git a/mac_pw_pool/SetupInstances.sh b/mac_pw_pool/SetupInstances.sh index 3518cd6..c500fdf 100755 --- a/mac_pw_pool/SetupInstances.sh +++ b/mac_pw_pool/SetupInstances.sh @@ -330,12 +330,7 @@ for _dhentry in "${_dhstate[@]}"; do continue # try again next loop fi - if ! $SCP $CIENV_SCRIPT ec2-user@$pub_dns:./; then - pwst_warn "Could not scp CI Env. script to instance $(ctx 0)." - continue # try again next loop - fi - - if ! $SSH ec2-user@$pub_dns chmod +x /var/tmp/*.sh ./ci_env.sh; then + if ! $SSH ec2-user@$pub_dns chmod +x /var/tmp/*.sh; then pwst_warn "Could not chmod scripts $(ctx 0)." continue # try again next loop fi diff --git a/mac_pw_pool/Utilization.gnuplot b/mac_pw_pool/Utilization.gnuplot index 2e2fd9c..563d4b8 100644 --- a/mac_pw_pool/Utilization.gnuplot +++ b/mac_pw_pool/Utilization.gnuplot @@ -8,7 +8,7 @@ set terminal png enhanced rounded size 1400,800 nocrop set output 'html/utilization.png' -set title "Persistent Workers & Utilization" +set title "Runners Online & Active" set xdata time set timefmt "%Y-%m-%dT%H:%M:%S+00:00" @@ -16,17 +16,13 @@ set xtics nomirror rotate timedate set xlabel "time/date" set xrange [(system("date -u -Iseconds -d '26 hours ago'")):(system("date -u -Iseconds"))] -set ylabel "Workers Online" +set ylabel "Number of Runners" set ytics border nomirror numeric # Not practical to lookup $DH_PFX from pw_lib.sh -set yrange [0:(system("grep -E '^[a-zA-Z0-9]+-[0-9]' dh_status.txt | wc -l") * 1.5)] - -set y2label "Worker Utilization" -set y2tics border nomirror numeric -set y2range [0:100] +set yrange [0:(system("grep -E '^[a-zA-Z0-9]+-[0-9]' dh_status.txt | wc -l") + 1)] set datafile separator comma set grid -plot 'utilization.csv' using 1:2 axis x1y1 title "Workers" pt 7 ps 2, \ - '' using 1:((($3-$4)/$2)*100) axis x1y2 title "Utilization" with lines lw 2 +plot 'utilization.csv' using 1:2 title "Online" pt 7 ps 2, \ + '' using 1:(($3-$4)) title "Active" with lines lw 2 diff --git a/mac_pw_pool/ci_env.sh b/mac_pw_pool/ci_env.sh deleted file mode 100755 index b2e0c64..0000000 --- a/mac_pw_pool/ci_env.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# This script drops the caller into a bash shell inside an environment -# substantially similar to a Cirrus-CI task running on this host. -# The envars below may require adjustment to better fit them to -# current/ongoing development in podman's .cirrus.yml - -set -eo pipefail - -# Not running as the pool worker user -if [[ "$USER" == "ec2-user" ]]; then - PWINST=$(curl -sSLf http://instance-data/latest/meta-data/tags/instance/Name) - PWUSER=$PWINST-worker - - if [[ ! -d "/Users/$PWUSER" ]]; then - echo "Warnin: Instance hasn't been setup. Assuming caller will tend to this." - sudo sysadminctl -addUser $PWUSER - fi - - sudo install -o $PWUSER "${BASH_SOURCE[0]}" "/Users/$PWUSER/" - exec sudo su -c "/Users/$PWUSER/$(basename ${BASH_SOURCE[0]})" - $PWUSER -fi - -# Export all CI-critical envars defined below -set -a - -CIRRUS_SHELL="/bin/bash" -CIRRUS_TASK_ID="0123456789" -CIRRUS_WORKING_DIR="$HOME/ci/task-${CIRRUS_TASK_ID}" - -GOPATH="$CIRRUS_WORKING_DIR/.go" -GOCACHE="$CIRRUS_WORKING_DIR/.go/cache" -GOENV="$CIRRUS_WORKING_DIR/.go/support" - -CONTAINERS_MACHINE_PROVIDER="applehv" - -MACHINE_IMAGE="https://fedorapeople.org/groups/podman/testing/applehv/arm64/fedora-coreos-38.20230925.dev.0-applehv.aarch64.raw.gz" - -GINKGO_TAGS="remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp remote" - -DEBUG_MACHINE="1" - -ORIGINAL_HOME="$HOME" -HOME="$HOME/ci" -TMPDIR="/private/tmp/ci" -mkdir -p "$TMPDIR" "$CIRRUS_WORKING_DIR" - -# Drop caller into the CI-like environment -cd "$CIRRUS_WORKING_DIR" -bash -il diff --git a/mac_pw_pool/html/index.html b/mac_pw_pool/html/index.html index c263540..470c857 100644 --- a/mac_pw_pool/html/index.html +++ b/mac_pw_pool/html/index.html @@ -2,11 +2,11 @@ - Cirrus-CI Persistent Workers + GitHub Actions Self-Hosted Runners
- +

diff --git a/mac_pw_pool/pw_lib.sh b/mac_pw_pool/pw_lib.sh index 933070e..ad71132 100644 --- a/mac_pw_pool/pw_lib.sh +++ b/mac_pw_pool/pw_lib.sh @@ -57,13 +57,12 @@ SETUP_MAX_SECONDS=2400 # Typical time ~10 minutes, use 2x safety-factor. # Name of launch template. Current/default version will be used. # https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#LaunchTemplates: -TEMPLATE_NAME="${TEMPLATE_NAME:-Cirrus${DH_PFX}PWinstance}" +TEMPLATE_NAME="${TEMPLATE_NAME:-GHA${DH_PFX}Runner}" # Path to scripts to copy/execute on Darwin instances SETUP_SCRIPT="$LIB_DIRPATH/setup.sh" SPOOL_SCRIPT="$LIB_DIRPATH/service_pool.sh" SHDWN_SCRIPT="$LIB_DIRPATH/shutdown.sh" -CIENV_SCRIPT="$LIB_DIRPATH/ci_env.sh" CLEANUP_HOOK_SCRIPT="$LIB_DIRPATH/cleanup_hook.sh" # Set to 1 to enable debugging diff --git a/mac_pw_pool/setup.sh b/mac_pw_pool/setup.sh index 1d5f1df..4017735 100644 --- a/mac_pw_pool/setup.sh +++ b/mac_pw_pool/setup.sh @@ -239,7 +239,7 @@ RUNNER_CONFIG_FILE="$RUNNER_DIR/.runner" if [[ ! -r "$RUNNER_CONFIG_FILE" ]]; then msg "Registering runner '$PWNAME'" # Configure the runner (but don't start it - service_pool.sh will do that) - # Work directory matches Cirrus pattern: /Users/$PWUSER/ci + # Work directory matches previous Cirrus pattern: /Users/$PWUSER/ci # --replace automatically removes any existing runner with the same name sudo -u $PWUSER bash -c "cd $RUNNER_DIR && ./config.sh \ --unattended \