Many improvements

.gitignore

@@ -18,4 +18,4 @@ challenges/*/terraform/versions.tf
 *.egg-info
 
 .vscode/
-
+.idea

README.md

@@ -1,8 +1,47 @@
 # CTF Script
 
-## Usage
+Opinionated command line interface (CLI) tool to manage Capture The Flag (CTF) challenges.
+It uses:
+- YAML files to describe a challenge and forum posts
+- OpenTofu (terraform fork) to describe the infrastructure
+- Incus (LXD fork) to run the challenges in containers
+- Ansible to configure the challenges
 
-Setup a `CTF_ROOT_DIR` environment variable to make the script execute in the right folder or execute the script from that folder.
+This tool is used by the NorthSec CTF team to manage their challenges since 2025.
+[NorthSec](https://nsec.io/) is one of the largest on-site cybersecurity CTF in the world, held annually in Montreal, Canada, 
+where 700+ participants compete in a 48-hour long CTF competition.
+
+## Features and Usage
+
+- `ctf init` to initialize a new ctf
+- `ctf new` to create a new challenge. Supports templates for common challenge types.
+- `ctf deploy` deploys the challenges to a local Incus instance
+- `ctf validate` runs lots of static checks (including JSON Schemas) on the challenges to ensure quality
+- `ctf stats` provide lots of helpful statistics about the CTF
+- and many more. See `ctf --help` for the full list of commands.
+
+To run `ctf` from any directory, set up the `CTF_ROOT_DIR` environment variable to make the script
+execute in the right directory or execute the script from that directory.
+
+## Structure of a CTF repository
+
+```
+my-ctf/
+├── challenges/                   # Directory containing all the challenges
+│   ├── track1/                   # Directory for a specific track that contains N flags. 
+│   │   ├── track.yaml            # Main file that describes the track
+│   │   ├── files/                # Directory that contains all the files available for download in the track
+│   │   │   ├── somefile.zip
+│   │   ├── ansible/              # Directory containing Ansible playbooks to configure the challenge
+│   │   │   ├── deploy.yaml       # Main playbook to deploy the challenge
+│   │   │   └── inventory         # Inventory file for Ansible
+│   │   ├── terraform/            # Directory containing OpenTofu (terraform fork) files to describe the infrastructure
+│   │   │   └── main.tf           # Main OpenTofu file to deploy the challenge
+│   │   ├── posts/                # Directory containing forum posts related to the challenge
+│   │   │   ├── track1.yaml       # Initial post for the track
+│   │   │   └── track1_flag1.yaml # Inventory file for Ansible
+
+```
 
 ## Installation
 
@@ -18,6 +57,12 @@ Install with pipx:
 pipx install git+https://github.com/nsec/ctf-script.git
 ```
 
+Install with pip:
+
+```bash
+pip install git+https://github.com/nsec/ctf-script.git
+```
+
 ### Add Bash/Zsh autocompletion to .bashrc
 
 ```bash

ctf/__main__.py

@@ -93,10 +93,14 @@ def terraform_binary() -> str:
 
 
 def init(args: argparse.Namespace) -> None:
-    if os.path.isdir(os.path.join(args.path, "challenges")) or os.path.isdir(
-        os.path.join(args.path, ".deploy")
-    ):
-        LOG.error(f"Directory {args.path} is already initialized.")
+    if (
+        os.path.isdir(os.path.join(args.path, "challenges"))
+        or os.path.isdir(os.path.join(args.path, ".deploy"))
+    ) and not args.force:
+        LOG.error(
+            f"Directory {args.path} is already initialized. Use --force to overwrite."
+        )
+        LOG.error(args.force)
         exit(code=1)
 
     created_assets: list[str] = []
@@ -105,7 +109,7 @@ def init(args: argparse.Namespace) -> None:
         for asset in os.listdir(p := os.path.join(TEMPLATES_ROOT_DIRECTORY, "init")):
             dst_asset = os.path.join(args.path, asset)
             if os.path.isdir(src_asset := os.path.join(p, asset)):
-                shutil.copytree(src_asset, dst_asset)
+                shutil.copytree(src_asset, dst_asset, dirs_exist_ok=True)
                 LOG.info(f"Created {dst_asset} folder")
             else:
                 shutil.copy(src_asset, dst_asset)
@@ -1356,6 +1360,13 @@ def main():
         default=CTF_ROOT_DIRECTORY,
         help="Initialize the folder at the given path.",
     )
+    parser_init.add_argument(
+        "--force",
+        "-f",
+        action="store_true",
+        default=False,
+        help="Overwrite the directory if it's already initialized.",
+    )
 
     parser_flags = subparsers.add_parser(
         "flags",

ctf/templates/init/.devcontainer/devcontainer.json

@@ -0,0 +1,18 @@
+{
+  "image": "mcr.microsoft.com/devcontainers/base:ubuntu",
+  "features": {
+    "ghcr.io/robbert229/devcontainer-features/opentofu:1": {
+      "version": "1.9.0"
+    },
+    "ghcr.io/devcontainers-extra/features/ansible:2": {}
+  },
+  "runArgs": [
+    "--privileged",
+    "--cap-add=SYS_PTRACE",
+    "--security-opt", "seccomp=unconfined",
+    "--cgroupns=host",
+    "--pid=host",
+    "--volume", "/dev:/dev",
+    "--volume", "/lib/modules:/lib/modules:ro"
+  ]
+}

ctf/templates/init/.github/workflows/tests.yml

@@ -0,0 +1,218 @@
+name: Tests
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check_changes:
+    name: Check changes
+    outputs:
+      run_job: ${{ steps.check_files.outputs.run_job }}
+      tracks: ${{ steps.check_files.outputs.tracks }}
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check modified files
+        id: check_files
+        run: |
+          tracks=()
+          echo "run_job=false" > "$GITHUB_OUTPUT"
+          while IFS= read -r file
+          do
+              if [[ $file == challenges/*/ansible/** || $file == challenges/*/terraform/** || $file == scripts/*.py || $file == .deploy/* ]]; then
+                  echo "[+] Detected required deployment because of file: ${file}"
+                  echo "run_job=true" > "$GITHUB_OUTPUT"
+                  if [[ $file == scripts/*.py || $file == .deploy/* ]]; then
+                      echo "[!] Running full deployment to properly test the scripts or .deploy changes."
+                      tracks=()
+                      break
+                  else
+                      track=$(sed -E 's/challenges\/(.+)\/(ansible|terraform)\/.*/\1/g' <<< "$file")
+                      if [[ ! " ${tracks[*]} " =~ [[:space:]]${track}[[:space:]] ]]; then
+                          tracks+=("$track")
+                      fi
+                  fi
+              fi
+          done < <(git diff --name-only origin/main..)
+          echo "tracks=${tracks[*]}" >> "$GITHUB_OUTPUT"
+
+  static_validations:
+    name: Static validations
+    timeout-minutes: 5
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup OpenTofu
+        run: |
+          curl -sL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh
+          chmod +x install-opentofu.sh
+          ./install-opentofu.sh --install-method deb
+          rm -f install-opentofu.sh
+
+      - name: Install python dependencies
+        run: |
+          pip install git+https://github.com/nsec/ctf-script.git
+
+      - name: Run ctf validate
+        run: |
+          ctf validate
+
+  deploy:
+    name: Full deployment test
+    needs: check_changes
+    if: needs.check_changes.outputs.run_job == 'true'
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Git LFS Pull for deployment
+        run: |
+          tracks="${{needs.check_changes.outputs.tracks}}"
+
+          if [ -z "$tracks" ]; then
+              echo "Pulling all Git LFS"
+              git lfs pull -I **/challenges/**/ansible/**/*
+          else
+              IFS=' ' read -ra tracks <<< "$tracks"
+
+              for track in "${tracks[@]}"
+              do
+                  echo "Pulling Git LFS for ${track}, if any..."
+                  git lfs pull -I **/challenges/"$track"/ansible/**/*
+              done
+          fi
+
+          echo "Pulled files:"
+          { git lfs ls-files | grep -E '[a-f0-9]{10}\s\*'; } || true
+
+      - name: Remove docker
+        run: |
+          sudo apt-get autopurge -y moby-containerd docker uidmap
+          sudo ip link delete docker0
+          sudo nft flush ruleset
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get install --no-install-recommends --yes zfsutils-linux
+
+      - name: Setup squid
+        run: |
+          sudo apt-get install --no-install-recommends --yes squid
+
+          (
+          cat << EOF
+          # No logging
+          cache_access_log /dev/null
+          cache_store_log none
+          cache_log /dev/null
+
+          # Caching
+          maximum_object_size 200 MB
+          cache_mem 1024 MB
+
+          # Port and mode configuration
+          acl local_subnet src 9000::/16
+          http_access allow local_subnet
+          http_access deny all
+          http_port [2602:fc62:ef:11::2]:3128
+
+          # Hide our traces
+          forwarded_for transparent
+          via off
+          reply_header_access X-Cache deny all
+          reply_header_access X-Cache-Lookup deny all
+
+          EOF
+          ) | sudo tee /etc/squid/conf.d/ctf.conf
+
+          sudo systemctl restart squid --no-block
+          sudo ip -6 a add dev lo 2602:fc62:ef:11::2/128
+
+      - name: Setup Incus
+        run: |
+          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh
+          sudo chmod 666 /var/lib/incus/unix.socket
+
+          incus network create incusbr0
+          incus profile device add default eth0 nic network=incusbr0 name=eth0
+
+          incus storage create default zfs size=100GiB
+          incus profile device add default root disk pool=default path=/
+
+          sudo zfs set sync=disabled default
+
+          sudo ip6tables -I FORWARD -j REJECT
+
+      - name: Setup Ansible
+        run: |
+          pipx install --force --include-deps ansible
+          pipx inject ansible passlib
+
+      - name: Setup OpenTofu
+        run: |
+          curl -sL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh
+          chmod +x install-opentofu.sh
+          ./install-opentofu.sh --install-method deb
+          rm -f install-opentofu.sh
+
+      - name: Install python dependencies
+        run: |
+          pip install -e .
+
+      - name: Deployment check
+        run: |
+          tracks="${{needs.check_changes.outputs.tracks}}"
+
+          if [ -z "$tracks" ]; then
+            ctf check
+          else
+            ctf check --tracks $tracks
+          fi
+
+      - name: File generation
+        run: |
+          tracks="${{needs.check_changes.outputs.tracks}}"
+
+          if [ -z "$tracks" ]; then
+            ctf generate
+          else
+            ctf generate --tracks $tracks
+          fi
+
+      - name: Test deployment
+        run: |
+          tracks="${{needs.check_changes.outputs.tracks}}"
+
+          if [ -z "$tracks" ]; then
+              tracks="$(python3 -c 'from scripts.utils import get_all_available_tracks,validate_track_can_be_deployed;print(str([t for t in get_all_available_tracks() if validate_track_can_be_deployed(t)]).strip("[]\x27").replace("\x27, \x27"," "))')"
+          fi
+
+          IFS=" " read -r -a tracks <<< "$tracks"
+
+          for track in "${tracks[@]}"
+          do
+              ctf deploy --production --tracks "$track"
+          done
+
+      - name: Check deployment results
+        run: |
+          incus project list
+          incus network zone record list ctf
+          incus network list --all-projects
+          incus list --all-projects

ctf/templates/init/.gitignore

@@ -18,4 +18,6 @@ challenges/*/terraform/versions.tf
 *.egg-info
 
 .vscode/
+!.vscode/settings.json
+!.vscode/extensions.json
 

ctf/templates/init/.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+  "recommendations": [
+    "redhat.vscode-yaml",
+    "hashicorp.terraform",
+    "github.codespaces",
+    "redhat.ansible"
+  ]
+}

ctf/templates/init/.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+    "[yaml]": {
+        "editor.tabSize": 2
+    },
+    "yaml.schemas": {
+        "scripts/schemas/track.yaml.json": "challenges/**/track.yaml",
+        "scripts/schemas/post.json": "challenges/*/posts/*.yaml"
+    }
+}