diff --git a/.github/scripts/get_projects_matrix.py b/.github/scripts/get_projects_matrix.py new file mode 100644 index 000000000..4aa7af9d1 --- /dev/null +++ b/.github/scripts/get_projects_matrix.py @@ -0,0 +1,102 @@ +# This file is used to generate a matrix of projects and platforms for GitHub Actions +# The matrix is used to build all projects its respective platforms +# The output is two JSON files: all_projects_matrix.json and modified_projects_matrix.json +# The all_projects_matrix.json contains all projects and platforms +# The modified_projects_matrix.json contains only the modified projects and platforms from the base branch +# The format of the JSON files are as follows: [ { "project": "project_name", "platform": "platform_name" }, ... ] + +# Usage: python3 get_projects_matrix.py +# Example: python3 get_projects_matrix.py main + +import json +import os +import subprocess +import sys + +# Global variables +projects_dir = "firmware/projects" +filter_projects_list = ["debug"] +filter_platforms_list = ["linux", "raspi", "sil"] + +# Get all projects in projects directory with a "platforms" subdirectory +def get_all_projects(projects_dir): + all_projects = set() + + for root, dirs, _ in os.walk(projects_dir): + if "platforms" in dirs: + project_path = os.path.relpath(root, projects_dir) + all_projects.add(project_path) + + return list(all_projects) + +# Get projects that have been modified compared to the base branch +def get_modified_projects(projects_dir, projects, base_branch): + # subprocess.run(["git", "fetch", "origin", base_branch]) + + modified_files = subprocess.check_output( + ["git", "diff", "--name-only", f"origin/{base_branch}", '--', projects_dir], + universal_newlines=True + ).splitlines() + + modified_projects = set() + for modified_file in modified_files: + for project in projects: + if project in modified_file: + modified_projects.add(project) + break + + modified_projects = list(modified_projects) + return modified_projects + +# Search for directories in the "platforms" subdirectory of a project +def get_project_platforms(projects_dir, project_dir): + platforms_dir = os.path.join(projects_dir, project_dir, "platforms") + return os.listdir(platforms_dir) + +# Format projects and platforms into a list for GitHub Actions matrix +def create_matrix(projects): + matrix = [] + for project in projects: + platforms = get_project_platforms(projects_dir, project) + platforms = filter_platforms(platforms, filter_platforms_list) + + for platform in platforms: + matrix.append({"project": project, "platform": platform}) + + return matrix + +# Remove projects that contain any of the items in the remove_list +def filter_projects(projects, remove_list): + return [project for project in projects if not any(item in project for item in remove_list)] + +# Remove platforms that match any of the items in the remove_list +def filter_platforms(platforms, remove_list): + return [platform for platform in platforms if platform not in remove_list] + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python modified_projects.py ") + sys.exit(1) + + base_branch = sys.argv[1] + print(f"Base branch: {base_branch}") + + if not os.path.isdir(projects_dir): + print(f"Error: {projects_dir} is not a directory") + sys.exit(1) + + projects = get_all_projects(projects_dir) + projects = filter_projects(projects, filter_projects_list) + print(f"All projects: {projects}") + + modified_projects = get_modified_projects(projects_dir, projects, base_branch) + print(f"Modified projects: {modified_projects}") + + all_projects_matrix = create_matrix(projects) + modified_projects_matrix = create_matrix(modified_projects) + + print(f"All projects matrix: {json.dumps(all_projects_matrix, indent=4)}") + print(f"Modified projects matrix: {json.dumps(modified_projects_matrix, indent=4)}") + + with open("all_projects_matrix.json", "w") as f: json.dump(all_projects_matrix, f) + with open("modified_projects_matrix.json", "w") as f: json.dump(modified_projects_matrix, f) diff --git a/.github/workflows/deploy-project.yml b/.github/workflows/deploy-project.yml new file mode 100644 index 000000000..67174f5e8 --- /dev/null +++ b/.github/workflows/deploy-project.yml @@ -0,0 +1,48 @@ +name: Deploy Project + +on: + workflow_dispatch: + inputs: + project: + description: 'Project (i.e. Demo/Blink)' + required: true + type: string + platform: + description: 'Platform (i.e. cli, stm32f767)' + required: true + type: string + commit: + description: 'Full commit hash' + required: true + type: string + +jobs: + deploy-to-raspberry-pi: + runs-on: unbuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Tailscale + run: | + curl -fsSL https://tailscale.com/install.sh | sh + + - name: Add Machine to Tailscale Network + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TAILSCALE_OAUTH_CLIENT_SECRET }} + tags: tag:ci + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: build-${{ inputs.project }}-${{ inputs.platform }}-${{ inputs.commit }} + + - name: Deploy to Raspberry Pi + run: | + echo "Deploying to Raspberry Pi" + echo "project=${{ inputs.project }}" + echo "platform=${{ inputs.platform }}" + echo "commit=${{ inputs.commit }}" + echo "sha=${{ github.sha }}" diff --git a/.github/workflows/manual-test-self-hosted.yml b/.github/workflows/manual-test-self-hosted.yml new file mode 100644 index 000000000..3dea3b61e --- /dev/null +++ b/.github/workflows/manual-test-self-hosted.yml @@ -0,0 +1,38 @@ +name: Manual Test Self-Hosted Runner + +on: + workflow_call: + inputs: + project: + description: 'Project (i.e. Demo/Blink)' + required: true + type: string + platform: + description: 'Platform (i.e. cli, stm32f767)' + required: true + type: string + commit: + description: 'Commit hash (first 7 characters)' + required: true + type: string + +jobs: + download-artifact: + runs-on: self-hosted + + steps: + - name: Prepare artifact name + id: prepare-artifact-name + run: | + project_name="${{ inputs.project }}" + project_name="${project_name//\//-}" + github_sha="${{ inputs.commit }}" + github_sha="${github_sha:-${{ github.sha }}}" + github_sha="${github_sha:0:7}" + artifact_name="build-${project_name}-${{ inputs.platform }}-${github_sha}" + echo "artifact_name=${artifact_name}" >> $GITHUB_OUTPUT + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: ${{ steps.prepare-artifact-name.outputs.artifact_name }} diff --git a/.github/workflows/reusable-build-project.yml b/.github/workflows/reusable-build-project.yml new file mode 100644 index 000000000..67cfb035e --- /dev/null +++ b/.github/workflows/reusable-build-project.yml @@ -0,0 +1,54 @@ +name: Build Project + +on: + workflow_call: + inputs: + project: + description: 'Project to build (i.e. Demo/Blink)' + required: true + type: string + platform: + description: 'Platform to build for (i.e. cli, stm32f767)' + required: true + type: string + +jobs: + build-upload-project: + runs-on: 'self-hosted' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Setup Python + run: | + python3 -m venv .venv + source .venv/bin/activate + pip install -e scripts/cangen + + - name: Build ${{ inputs.project }} for ${{ inputs.platform }} + run: | + source .venv/bin/activate + cd firmware + make PROJECT=${{ inputs.project }} PLATFORM=${{ inputs.platform }} build + + - name: Prepare artifact name + id: prepare-artifact-name + run: | + project_name="${{ inputs.project }}" + project_name="${project_name//\//-}" + github_sha="${{ github.sha }}" + github_sha="${github_sha:0:7}" + artifact_name="build-${project_name}-${{ inputs.platform }}-${github_sha}" + echo "artifact_name=${artifact_name}" >> $GITHUB_OUTPUT + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + path: firmware/build/${{ inputs.project }}/${{ inputs.platform }} + name: ${{ steps.prepare-artifact-name.outputs.artifact_name }} + + - name: Upload File to Raspberry Pi + run: | + scp -r firmware/build/${{ inputs.project }}/${{ inputs.platform }} macformula@100.73.87.83:${{ steps.prepare-artifact-name.outputs.artifact_name }}/ diff --git a/.github/workflows/setup-raspberry-pi.yml b/.github/workflows/setup-raspberry-pi.yml new file mode 100644 index 000000000..f1d28d503 --- /dev/null +++ b/.github/workflows/setup-raspberry-pi.yml @@ -0,0 +1,33 @@ +name: Setup Raspberry Pi + +on: + # push: + workflow_dispatch: + +jobs: + setup-raspberry-pi: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Tailscale + run: | + curl -fsSL https://tailscale.com/install.sh | sh + + - name: Add Machine to Tailscale Network + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TAILSCALE_OAUTH_CLIENT_SECRET }} + tags: tag:ci + + - name: Install Ansible + run: | + sudo apt update + sudo apt install -y ansible + + - name: Run Ansible Setup Playbook on Raspberry Pi + run: | + ansible-playbook -i scripts/hil/inventory.yml scripts/hil/setup_raspberry_pi.yml diff --git a/.github/workflows/test-self-hosted.yml b/.github/workflows/test-self-hosted.yml new file mode 100644 index 000000000..5c05adb74 --- /dev/null +++ b/.github/workflows/test-self-hosted.yml @@ -0,0 +1,73 @@ +name: Test Self-Hosted Runner + +on: + push: + branches: + - main + pull_request: + types: + - opened + - reopened + - synchronize + workflow_dispatch: + +jobs: + setup: + runs-on: self-hosted + outputs: + matrix_all_projects: ${{ steps.projects.outputs.matrix_all_projects }} + matrix_modified_projects: ${{ steps.projects.outputs.matrix_modified_projects }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python + run: | + python3 -m venv .venv + + - name: Get projects + id: projects + run: | + source .venv/bin/activate + BASE_REF="${{ github.base_ref }}" # Target branch of the PR + BASE_REF="${BASE_REF:-HEAD^}" # Previous commit on push to main + python3 .github/scripts/get_projects_matrix.py $BASE_REF + echo "matrix_all_projects=$(cat all_projects_matrix.json)" >> $GITHUB_OUTPUT + echo "matrix_modified_projects=$(cat modified_projects_matrix.json)" >> $GITHUB_OUTPUT + + build-all-projects: + if: false + needs: setup + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.setup.outputs.matrix_all_projects) }} + uses: ./.github/workflows/reusable-build-project.yml + with: + project: ${{ matrix.project }} + platform: ${{ matrix.platform }} + + build-modified-projects: + needs: setup + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.setup.outputs.matrix_modified_projects) }} + uses: ./.github/workflows/reusable-build-project.yml + with: + project: ${{ matrix.project }} + platform: ${{ matrix.platform }} + + manual-test-self-hosted: + if: false + needs: [setup, build-modified-projects] + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.setup.outputs.matrix_modified_projects) }} + uses: ./.github/workflows/manual-test-self-hosted.yml + with: + project: ${{ matrix.project }} + platform: ${{ matrix.platform }} + commit: '' \ No newline at end of file diff --git a/scripts/hil/inventory.yml b/scripts/hil/inventory.yml new file mode 100644 index 000000000..abfde237b --- /dev/null +++ b/scripts/hil/inventory.yml @@ -0,0 +1,6 @@ +all: + hosts: + raspberry_pi: + ansible_host: 100.73.87.83 + ansible_user: macformula + ansible_ssh_extra_args: "-o StrictHostKeyChecking=no" diff --git a/scripts/hil/setup_raspberry_pi.yml b/scripts/hil/setup_raspberry_pi.yml new file mode 100644 index 000000000..2913a9836 --- /dev/null +++ b/scripts/hil/setup_raspberry_pi.yml @@ -0,0 +1,25 @@ +- name: Setup Raspberry Pi + hosts: raspberry_pi + become: true + + tasks: + - name: Update and upgrade apt packages + apt: + update_cache: yes + upgrade: dist + + - name: Install clangd + apt: + name: clangd + state: present + + - name: Install clang-format + apt: + name: clang-format + state: present + + - name: Verify clangd installation + command: clangd --version + + - name: Verify clang-format installation + command: clang-format --version diff --git a/scripts/hil/test_script.yml b/scripts/hil/test_script.yml new file mode 100644 index 000000000..f3bcae863 --- /dev/null +++ b/scripts/hil/test_script.yml @@ -0,0 +1,25 @@ +- name: Test Script + hosts: raspberry_pi + become: true + + tasks: + - name: Create a test directory + file: + path: /home/macformula/test-directory + state: directory + owner: macformula + group: macformula + mode: '0755' + + - name: Create a test file with content + copy: + dest: /home/macformula/test-directory/hello.txt + content: "Hello, Raspberry Pi!" + owner: macformula + group: macformula + mode: '0644' + + - name: Delete the test directory + file: + path: /home/macformula/test-directory + state: absent