Raspberry Pi Ansible Script

.github/scripts/get_projects_matrix.py

@@ -0,0 +1,102 @@
+# This file is used to generate a matrix of projects and platforms for GitHub Actions
+# The matrix is used to build all projects its respective platforms
+# The output is two JSON files: all_projects_matrix.json and modified_projects_matrix.json
+# The all_projects_matrix.json contains all projects and platforms
+# The modified_projects_matrix.json contains only the modified projects and platforms from the base branch
+# The format of the JSON files are as follows: [ { "project": "project_name", "platform": "platform_name" }, ... ]
+
+# Usage: python3 get_projects_matrix.py <base_git_branch>
+# Example: python3 get_projects_matrix.py main
+
+import json
+import os
+import subprocess
+import sys
+
+# Global variables
+projects_dir = "firmware/projects"
+filter_projects_list = ["debug"]
+filter_platforms_list = ["linux", "raspi", "sil"]
+
+# Get all projects in projects directory with a "platforms" subdirectory
+def get_all_projects(projects_dir):
+    all_projects = set()
+
+    for root, dirs, _ in os.walk(projects_dir):
+        if "platforms" in dirs:
+            project_path = os.path.relpath(root, projects_dir)
+            all_projects.add(project_path)
+
+    return list(all_projects)
+
+# Get projects that have been modified compared to the base branch
+def get_modified_projects(projects_dir, projects, base_branch):
+    # subprocess.run(["git", "fetch", "origin", base_branch])
+
+    modified_files = subprocess.check_output(
+        ["git", "diff", "--name-only", f"origin/{base_branch}", '--', projects_dir],
+        universal_newlines=True
+    ).splitlines()
+
+    modified_projects = set()
+    for modified_file in modified_files:
+        for project in projects:
+            if project in modified_file:
+                modified_projects.add(project)
+                break
+
+    modified_projects = list(modified_projects)
+    return modified_projects
+
+# Search for directories in the "platforms" subdirectory of a project
+def get_project_platforms(projects_dir, project_dir):
+    platforms_dir = os.path.join(projects_dir, project_dir, "platforms")
+    return os.listdir(platforms_dir)
+
+# Format projects and platforms into a list for GitHub Actions matrix
+def create_matrix(projects):
+    matrix = []
+    for project in projects:
+        platforms = get_project_platforms(projects_dir, project)
+        platforms = filter_platforms(platforms, filter_platforms_list)
+
+        for platform in platforms:
+            matrix.append({"project": project, "platform": platform})
+
+    return matrix
+
+# Remove projects that contain any of the items in the remove_list
+def filter_projects(projects, remove_list):
+    return [project for project in projects if not any(item in project for item in remove_list)]
+
+# Remove platforms that match any of the items in the remove_list
+def filter_platforms(platforms, remove_list):
+    return [platform for platform in platforms if platform not in remove_list]
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python modified_projects.py <base_branch>")
+        sys.exit(1)
+
+    base_branch = sys.argv[1]
+    print(f"Base branch: {base_branch}")
+
+    if not os.path.isdir(projects_dir):
+        print(f"Error: {projects_dir} is not a directory")
+        sys.exit(1)
+
+    projects = get_all_projects(projects_dir)
+    projects = filter_projects(projects, filter_projects_list)
+    print(f"All projects: {projects}")
+
+    modified_projects = get_modified_projects(projects_dir, projects, base_branch)
+    print(f"Modified projects: {modified_projects}")
+
+    all_projects_matrix = create_matrix(projects)
+    modified_projects_matrix = create_matrix(modified_projects)
+
+    print(f"All projects matrix: {json.dumps(all_projects_matrix, indent=4)}")
+    print(f"Modified projects matrix: {json.dumps(modified_projects_matrix, indent=4)}")
+
+    with open("all_projects_matrix.json", "w") as f: json.dump(all_projects_matrix, f)
+    with open("modified_projects_matrix.json", "w") as f: json.dump(modified_projects_matrix, f)

.github/workflows/deploy-project.yml

@@ -0,0 +1,48 @@
+name: Deploy Project
+
+on:
+  workflow_dispatch:
+    inputs:
+      project:
+        description: 'Project (i.e. Demo/Blink)'
+        required: true
+        type: string
+      platform:
+        description: 'Platform (i.e. cli, stm32f767)'
+        required: true
+        type: string
+      commit:
+        description: 'Full commit hash'
+        required: true
+        type: string
+
+jobs:
+  deploy-to-raspberry-pi:
+    runs-on: unbuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Tailscale
+        run: |
+            curl -fsSL https://tailscale.com/install.sh | sh
+
+      - name: Add Machine to Tailscale Network
+        uses: tailscale/github-action@v2
+        with:
+            oauth-client-id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}
+            oauth-secret: ${{ secrets.TAILSCALE_OAUTH_CLIENT_SECRET }}
+            tags: tag:ci
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: build-${{ inputs.project }}-${{ inputs.platform }}-${{ inputs.commit }}
+
+      - name: Deploy to Raspberry Pi
+        run: |
+          echo "Deploying to Raspberry Pi"
+          echo "project=${{ inputs.project }}"
+          echo "platform=${{ inputs.platform }}"
+          echo "commit=${{ inputs.commit }}"
+          echo "sha=${{ github.sha }}"

.github/workflows/manual-test-self-hosted.yml

@@ -0,0 +1,38 @@
+name: Manual Test Self-Hosted Runner
+
+on:
+  workflow_call:
+    inputs:
+      project:
+        description: 'Project (i.e. Demo/Blink)'
+        required: true
+        type: string
+      platform:
+        description: 'Platform (i.e. cli, stm32f767)'
+        required: true
+        type: string
+      commit:
+        description: 'Commit hash (first 7 characters)'
+        required: true
+        type: string
+
+jobs:
+  download-artifact:
+    runs-on: self-hosted
+
+    steps:
+      - name: Prepare artifact name
+        id: prepare-artifact-name
+        run: |
+          project_name="${{ inputs.project }}" 
+          project_name="${project_name//\//-}"
+          github_sha="${{ inputs.commit }}"
+          github_sha="${github_sha:-${{ github.sha }}}"
+          github_sha="${github_sha:0:7}"
+          artifact_name="build-${project_name}-${{ inputs.platform }}-${github_sha}"
+          echo "artifact_name=${artifact_name}" >> $GITHUB_OUTPUT
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+            name: ${{ steps.prepare-artifact-name.outputs.artifact_name }}

.github/workflows/reusable-build-project.yml

@@ -0,0 +1,54 @@
+name: Build Project
+
+on:
+  workflow_call:
+    inputs:
+      project:
+        description: 'Project to build (i.e. Demo/Blink)'
+        required: true
+        type: string
+      platform:
+        description: 'Platform to build for (i.e. cli, stm32f767)'
+        required: true
+        type: string
+
+jobs:
+  build-upload-project:
+    runs-on: 'self-hosted'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+            submodules: true
+
+      - name: Setup Python
+        run: |
+          python3 -m venv .venv
+          source .venv/bin/activate
+          pip install -e scripts/cangen
+
+      - name: Build ${{ inputs.project }} for ${{ inputs.platform }}
+        run: |
+          source .venv/bin/activate
+          cd firmware
+          make PROJECT=${{ inputs.project }} PLATFORM=${{ inputs.platform }} build
+
+      - name: Prepare artifact name
+        id: prepare-artifact-name
+        run: |
+          project_name="${{ inputs.project }}" 
+          project_name="${project_name//\//-}"
+          github_sha="${{ github.sha }}"
+          github_sha="${github_sha:0:7}"
+          artifact_name="build-${project_name}-${{ inputs.platform }}-${github_sha}"
+          echo "artifact_name=${artifact_name}" >> $GITHUB_OUTPUT
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          path: firmware/build/${{ inputs.project }}/${{ inputs.platform }}
+          name: ${{ steps.prepare-artifact-name.outputs.artifact_name }}
+
+      - name: Upload File to Raspberry Pi
+        run: |
+          scp -r firmware/build/${{ inputs.project }}/${{ inputs.platform }} macformula@100.73.87.83:${{ steps.prepare-artifact-name.outputs.artifact_name }}/

.github/workflows/setup-raspberry-pi.yml

@@ -0,0 +1,33 @@
+name: Setup Raspberry Pi
+
+on:
+  # push:
+  workflow_dispatch:
+
+jobs:
+  setup-raspberry-pi:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Install Tailscale
+      run: |
+        curl -fsSL https://tailscale.com/install.sh | sh
+
+    - name: Add Machine to Tailscale Network
+      uses: tailscale/github-action@v2
+      with:
+        oauth-client-id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }}
+        oauth-secret: ${{ secrets.TAILSCALE_OAUTH_CLIENT_SECRET }}
+        tags: tag:ci
+
+    - name: Install Ansible
+      run: |
+        sudo apt update
+        sudo apt install -y ansible
+
+    - name: Run Ansible Setup Playbook on Raspberry Pi
+      run: |
+        ansible-playbook -i scripts/hil/inventory.yml scripts/hil/setup_raspberry_pi.yml

.github/workflows/test-self-hosted.yml

@@ -0,0 +1,73 @@
+name: Test Self-Hosted Runner
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+  workflow_dispatch:
+
+jobs:
+  setup:
+    runs-on: self-hosted
+    outputs:
+      matrix_all_projects: ${{ steps.projects.outputs.matrix_all_projects }}
+      matrix_modified_projects: ${{ steps.projects.outputs.matrix_modified_projects }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        run: |
+          python3 -m venv .venv
+
+      - name: Get projects
+        id: projects
+        run: |
+          source .venv/bin/activate
+          BASE_REF="${{ github.base_ref }}" # Target branch of the PR
+          BASE_REF="${BASE_REF:-HEAD^}" # Previous commit on push to main
+          python3 .github/scripts/get_projects_matrix.py $BASE_REF
+          echo "matrix_all_projects=$(cat all_projects_matrix.json)" >> $GITHUB_OUTPUT
+          echo "matrix_modified_projects=$(cat modified_projects_matrix.json)" >> $GITHUB_OUTPUT
+
+  build-all-projects:
+    if: false
+    needs: setup
+    strategy:
+      fail-fast: false
+      matrix: 
+        include: ${{ fromJson(needs.setup.outputs.matrix_all_projects) }}
+    uses: ./.github/workflows/reusable-build-project.yml
+    with:
+      project: ${{ matrix.project }}
+      platform: ${{ matrix.platform }}
+
+  build-modified-projects:
+    needs: setup
+    strategy:
+      fail-fast: false
+      matrix: 
+        include: ${{ fromJson(needs.setup.outputs.matrix_modified_projects) }}
+    uses: ./.github/workflows/reusable-build-project.yml
+    with:
+      project: ${{ matrix.project }}
+      platform: ${{ matrix.platform }}
+
+  manual-test-self-hosted:
+    if: false
+    needs: [setup, build-modified-projects]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.setup.outputs.matrix_modified_projects) }}
+    uses: ./.github/workflows/manual-test-self-hosted.yml
+    with:
+      project: ${{ matrix.project }}
+      platform: ${{ matrix.platform }}
+      commit: ''

scripts/hil/inventory.yml

@@ -0,0 +1,6 @@
+all:
+  hosts:
+    raspberry_pi:
+      ansible_host: 100.73.87.83
+      ansible_user: macformula
+      ansible_ssh_extra_args: "-o StrictHostKeyChecking=no"

scripts/hil/setup_raspberry_pi.yml

@@ -0,0 +1,25 @@
+- name: Setup Raspberry Pi
+  hosts: raspberry_pi
+  become: true
+
+  tasks:
+    - name: Update and upgrade apt packages
+      apt:
+        update_cache: yes
+        upgrade: dist
+
+    - name: Install clangd
+      apt:
+        name: clangd
+        state: present
+
+    - name: Install clang-format
+      apt:
+        name: clang-format
+        state: present
+
+    - name: Verify clangd installation
+      command: clangd --version
+
+    - name: Verify clang-format installation
+      command: clang-format --version