From 3a1c1679539d0de2e02e05d315ba7b3bbd5bc0d6 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Mon, 9 Sep 2024 17:21:36 +0200 Subject: [PATCH 01/61] Test action --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..442b2260e11 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,13 @@ +name: Compile SOFA and run Tests +on: + workflow_dispatch: + commit: + +jobs: + check_title: + runs-on: [self-hosted, sh-ubuntu] + steps: + - name: Test env + shell: bash + run: | + env | grep GITHUB \ No newline at end of file From 978ef1a3e35516394f37a4ad2d31fc013ecdf99b Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Mon, 9 Sep 2024 17:24:06 +0200 Subject: [PATCH 02/61] Fix action --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 442b2260e11..d2993c7cea0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: Compile SOFA and run Tests on: workflow_dispatch: - commit: + push: jobs: check_title: From 0c2f7f093790358c7a713eca4de436deeb779a50 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Mon, 9 Sep 2024 17:52:27 +0200 Subject: [PATCH 03/61] Test pre job hook --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2993c7cea0..cec27a34ff5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,17 @@ name: Compile SOFA and run Tests on: workflow_dispatch: + pull_request: push: jobs: - check_title: + test_env: runs-on: [self-hosted, sh-ubuntu] steps: - name: Test env shell: bash run: | - env | grep GITHUB \ No newline at end of file + WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) + cd $WORKSPACE + echo $(pwd) + echo $WORKSPACE From 1a874b19913b83839872d744d34961fd15672998 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Mon, 9 Sep 2024 18:18:59 +0200 Subject: [PATCH 04/61] Add setup_build step --- .github/workflows/ci.yml | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cec27a34ff5..5654aecec33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,13 +5,41 @@ on: push: jobs: - test_env: + setup_build: runs-on: [self-hosted, sh-ubuntu] steps: - - name: Test env + - name: Clone SOFA and CI shell: bash run: | WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) cd $WORKSPACE - echo $(pwd) - echo $WORKSPACE + + + echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" + mkdir src & cd src + SRC_DIR=$(pwd) + + git init + git remote add origin https://www.github.com/$GITHUB_REPOSITORY_OWNER/sofa.git + git fetch origin $GITHUB_WORKFLOW_SHA + git checkout FETCH_HEAD + + cd $WORKSPACE + + + echo "Cloning CI" + mkdir ci & cd ci + CI_DIR=$(pwd) + + git init + git remote add origin https://www.github.com/sofa-framework/ci.git + git fetch origin master + git checkout FETCH_HEAD + + cd $WORKSPACE + + + mkdir build + + + From 033bd3c5820c6a4b370d35c15051830e766a1e7d Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Mon, 9 Sep 2024 18:20:44 +0200 Subject: [PATCH 05/61] Fix action --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5654aecec33..99b5c99c0f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,9 @@ jobs: WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) cd $WORKSPACE - + echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" - mkdir src & cd src + mkdir $WORKSPACE/src && cd $WORKSPACE/src SRC_DIR=$(pwd) git init @@ -28,7 +28,7 @@ jobs: echo "Cloning CI" - mkdir ci & cd ci + mkdir $WORKSPACE/ci && cd $WORKSPACE/ci CI_DIR=$(pwd) git init From c5fda9760bab546ecfc509ce057badc7249b3067 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 10 Sep 2024 14:46:42 +0200 Subject: [PATCH 06/61] FIX when folders already exist + TODOS --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99b5c99c0f0..82e7aede90e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,9 +13,13 @@ jobs: run: | WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) cd $WORKSPACE - echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" + #TODO (for PR) : clone branch against which the PR is compared then merge the PR + + if [ -d $WORKSPACE/src ]; then + rm -rf $WORKSPACE/src + fi mkdir $WORKSPACE/src && cd $WORKSPACE/src SRC_DIR=$(pwd) @@ -26,8 +30,12 @@ jobs: cd $WORKSPACE - echo "Cloning CI" + #TODO : clone ci branch with same name as the sofa branch against which the PR is compared OR the branch from which the commit comes + + if [ -d $WORKSPACE/ci ]; then + rm -rf $WORKSPACE/ci + fi mkdir $WORKSPACE/ci && cd $WORKSPACE/ci CI_DIR=$(pwd) @@ -38,8 +46,23 @@ jobs: cd $WORKSPACE + #TODO: transform infos to force-full-build + if [ -d $WORKSPACE/build ]; then + rm -rf $WORKSPACE/build + fi + mkdir $WORKSPACE/build + + #TODO: transform job-type (PR, commit, nightly) in a set of parameter (core, standard, full) + #TODO: transform job-type + infos from messages in a set of tests to run (unit, scenes, regression) + #TODO: transform infos from messages in a set of ci-depends-on - mkdir build - + #TODO: launch the container if needed (e.g. for ubuntu) + #TODO: launch sofa-ci-build-action with parameters : building scope (core, standard, full), + # OS type, + # python version, + # tests sets (list taken from {unit, scenes, regression}), + # ci-depends-on, + # OOT cmake option list (empty here) + From 14d7d2a85b31f8fa10caf99226a03b7425588241 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Mon, 10 Mar 2025 09:20:17 +0100 Subject: [PATCH 07/61] Add filter action and scripts --- .github/workflows/filter-build.yml | 128 +++++++++++++++ scripts/github_CI/checkPRInfoBeforeBuild.py | 172 ++++++++++++++++++++ scripts/github_CI/trigger-build.sh | 18 ++ 3 files changed, 318 insertions(+) create mode 100644 .github/workflows/filter-build.yml create mode 100644 scripts/github_CI/checkPRInfoBeforeBuild.py create mode 100644 scripts/github_CI/trigger-build.sh diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml new file mode 100644 index 00000000000..9ad2a6765b6 --- /dev/null +++ b/.github/workflows/filter-build.yml @@ -0,0 +1,128 @@ +name: Filter build action + + +# =============================================================== +# =============================================================== + + +on: + # On-demand binary generation + workflow_dispatch: + inputs: + branch: + description: 'Specify the stable branch to use to generate the new binaries' + required: true + commit_sha: + description: 'Commit SHA to check out' + required: true + default: 'head' + + # Nightly build + schedule: + - cron: '0 2 * * *' # Evevery night + + # PR-related build (open, labels, push) + pull_request: + types: [opened, synchronize] + + # Comment on PR + pull_request_review_comment: + types: [created, edited] + + # CI for dashboard master + push: + branches: + - 'master' + +# =============================================================== +# =============================================================== + +jobs: + filter_build: + runs-on: ubuntu-latest + ###if: ${{ github.repository_owner == 'sofa-framework' }} + + steps: + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pip packages + run: | + pip install python-graphql-client + pip install requests + + - name: Run on dispatch + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + echo "This step runs only for binary generation." + + BRANCH=${{ github.event.inputs.branch }} + COMMIT=${{ github.event.inputs.commit }} + + # Validate branch format (e.g., v25.06) + if [[ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ]]; then + echo "Invalid branch name format: $BRANCH" + exit 1 + fi + echo "Branch name $BRANCH is valid." + + # Check commit SHA format + if [[ ! "$COMMIT" =~ ^[a-fA-F0-9]{40}$ ]]; then + echo "Invalid commit SHA format: $COMMIT" + exit 1 + fi + # Check the commit exists in the repository + if git cat-file -e "$COMMIT" 2>/dev/null; then + echo "Commit $COMMIT is valid." + else + echo "Commit $COMMIT does not exist in the repository." + exit 1 + fi + + ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} $BRANCH $COMMIT full None False True False True + + - name: Run when nightly + if: github.event.schedule == '0 2 * * *' + run: | + echo "This step runs only for nightly build." + ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} standard None False True False True + + - name: Check push on master case + if: ${{ github.event_name == 'push'}} + run: | + echo "This step runs only when a commit is pushed on the master branch." + ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} full None True True False False + + - name: Run when PR is opened + if: ${{ github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize') }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} + COMMIT_SHA: ${{ github.sha }} + run: | + echo "This step runs only when a PR is opened or synchronized." + + # Trigger the Build action + python scripts/github_CI/checkPRInfoBeforeBuild.py + + - name: Run when PR comment is edited + if: ${{ github.event_name == 'pull_request_review_comment' && ( github.event.action == 'created'|| github.event.action == 'edited' ) }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} + COMMIT_SHA: ${{ github.sha }} + run: | + echo "This step runs only when a PR comment is created or edited." + + COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) + echo "Comment: $COMMENT_BODY" + + # Trigger the Build action if [ci-build] is in the comment + if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then + python scripts/github_CI/checkPRInfoBeforeBuild.py + fi + diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py new file mode 100644 index 00000000000..36d0b97c3f9 --- /dev/null +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -0,0 +1,172 @@ +#!python + +import os +import requests +from python_graphql_client import GraphqlClient + + +client = GraphqlClient(endpoint="https://api.github.com/graphql") +GITHUB_TOKEN = os.getenv('GITHUB_TOKEN') +PR_NUMBER = os.getenv('PR_NUMBER') +OWNER_NAME = os.getenv('OWNER_NAME') +COMMIT_SHA = os.getenv('COMMIT_SHA') + + +if not GITHUB_TOKEN or not PR_NUMBER or not REPO_NAME: + print("Error: Missing required environment variables.") + exit(1) + + +# GitHub API base URL +API_URL = f"https://api.github.com/repos/sofa-framework/sofa" + +# Headers for authentication +HEADERS = { + "Authorization": f"Bearer {GITHUB_TOKEN}", + "Accept": "application/vnd.github.v3+json" +} + + +# Flags to determine actions +to_review_label_found = False +is_draft_pr = False +with_all_tests_found = False +force_full_build_found = False + + +# ======================================================================== + +# Check PR labels +def check_labels(): + global to_review_label_found + labels_url = f"{API_URL}/issues/{PR_NUMBER}/labels" + response = requests.get(labels_url, headers=HEADERS) + + if response.status_code != 200: + print(f"Failed to fetch labels: {response.status_code}") + exit(1) + + labels = [label['name'].lower() for label in response.json()] + print(f"Labels found: {labels}") + + if "pr: status to review" in labels: + to_review_label_found = True + print("PR is marked as 'to review'.") + +# ======================================================================== + +# Check the PR draft status +def check_if_draft(): + global is_draft_pr + pr_url = f"{API_URL}/pulls/{PR_NUMBER}" + response = requests.get(pr_url, headers=HEADERS) + + if response.status_code != 200: + print(f"Failed to fetch pull request details: {response.status_code}") + exit(1) + + pr_data = response.json() + is_draft_pr = pr_data.get('draft', False) + + if is_draft_pr: + print("The pull request is a draft. The Bash script will not run.") + +# ======================================================================== + +# Check PR comments for "[with-all-tests]" and "[force-full-build]" +def check_comments(): + global with_all_tests_found + global force_full_build_found + comments_url = f"{API_URL}/issues/{PR_NUMBER}/comments" + response = requests.get(comments_url, headers=HEADERS) + + if response.status_code != 200: + print(f"Failed to fetch comments: {response.status_code}") + exit(1) + + comments = [comment['body'].lower() for comment in response.json()] + print(f"Comments found: {comments}") + + if any("[with-all-tests]" in comment for comment in comments): + with_all_tests_found = True + print("Found a comment containing 'with-all-tests'.") + if any("[force-full-build]" in comment for comment in comments): + force_full_build_found = True + print("Found a comment containing 'with-all-tests'.") + + +# ======================================================================== + + +# Execute the checks +check_labels() +check_if_draft() + + +# Trigger the build if conditions are met +if to_review_label_found and not is_draft_pr: + + # Check compilation options in PR comments + check_comments() + + pr_url = f"{API_URL}/pulls/{PR_NUMBER}" + response = requests.get(pr_url, headers=HEADERS) + pr_data = response.json() + branch = pr_data.get("head", {}).get("ref", None) + + + # Extract the PR description and look for [ci-depends-on ...] patterns + pr_body = pr_data.get("body", "") + ci_depends_on = [] + + for line in pr_body.splitlines(): + match = re.search(r'\[ci-depends-on (.+?)\]', line) + if match: + dependency = match.group(1).strip() + ci_depends_on.append(dependency) + print(f"Found ci-depends-on dependency: {dependency}") + + # GitHub repository details + API_URL_DISPATCH = f"https://api.github.com/repos/sofa-framework/sofa/dispatches" + + # JSON payload for the dispatch event + PAYLOAD = { + "event_type": "CI Build", + "client_payload": { + "owner": {OWNER_NAME}, + "branch": {branch}, + "commit_hash": {COMMIT_SHA}, + "preset": "full", + "ci-depends-on": {ci_depends_on}, + "with-all-tests": {with_all_tests_found}, + "force-full-build": {force_full_build_found}, + "out-of-tree-build": "False", + "generate-binaries": "False", + }, + } + + # Headers for the GitHub API request + REQUEST_HEADERS = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {GITHUB_TOKEN}", + "X-GitHub-Api-Version": "2022-11-28", + } + + # Perform the API request + try: + response = requests.post(API_URL_DISPATCH, json=PAYLOAD, headers=REQUEST_HEADERS) + + # Check for successful request + if response.status_code == 204: + print("CI Build event triggered successfully.") + else: + print(f"Failed to trigger CI Build event. Status code: {response.status_code}") + print("Response:", response.text) + sys.exit(1) + except requests.RequestException as e: + print("Error during the API request:", e) + sys.exit(1) +else: + print("Conditions not met. Bash script will not run.") + +# ======================================================================== diff --git a/scripts/github_CI/trigger-build.sh b/scripts/github_CI/trigger-build.sh new file mode 100644 index 00000000000..16b1c0d67a2 --- /dev/null +++ b/scripts/github_CI/trigger-build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Check if exactly 8 arguments are provided +if [ "$#" -ne 9 ]; then + echo "Error: Exactly 9 arguments are required." + echo "Usage: $0 " + exit 1 +fi + +# Trigger the build action with all needed input: +# branch, hash, preset, ci-depends-on, with-all-tests, force-full-build, out-of-tree-build, label-wip/postponed, generate-binaries +sudo apt install curl +curl -L -X POST \ +-H "Accept: application/vnd.github+json" \ +-H "Authorization: Bearer $1" \ +-H "X-GitHub-Api-Version: 2022-11-28" \ +https://api.github.com/repos/sofa-framework/sofa/dispatches \ +-d '{"event_type":"CI Build","client_payload":{"owner":"sofa-framework","branch":"$2","commit_hash":"$3","preset":"$4","ci-depends-on":"$5", "with-all-tests":"$6", "force-full-build":"7", "out-of-tree-build":"$8", "generate-binaries":"$9"}}' From 9725eb2efe284a0c0ba940e4b7d3fd983ce4a649 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Thu, 13 Mar 2025 10:53:57 +0100 Subject: [PATCH 08/61] replace sofa-framework by bakpaul --- .github/workflows/filter-build.yml | 2 +- scripts/github_CI/checkPRInfoBeforeBuild.py | 4 ++-- scripts/github_CI/trigger-build.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 9ad2a6765b6..fce5540ce3d 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -40,7 +40,7 @@ on: jobs: filter_build: runs-on: ubuntu-latest - ###if: ${{ github.repository_owner == 'sofa-framework' }} + ### if: ${{ github.repository_owner == 'sofa-framework' }} steps: - name: Set up python diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py index 36d0b97c3f9..44ff9527752 100644 --- a/scripts/github_CI/checkPRInfoBeforeBuild.py +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -18,7 +18,7 @@ # GitHub API base URL -API_URL = f"https://api.github.com/repos/sofa-framework/sofa" +API_URL = f"https://api.github.com/repos/bakpaul/sofa" # Headers for authentication HEADERS = { @@ -127,7 +127,7 @@ def check_comments(): print(f"Found ci-depends-on dependency: {dependency}") # GitHub repository details - API_URL_DISPATCH = f"https://api.github.com/repos/sofa-framework/sofa/dispatches" + API_URL_DISPATCH = f"https://api.github.com/repos/bakpaul/sofa/dispatches" # JSON payload for the dispatch event PAYLOAD = { diff --git a/scripts/github_CI/trigger-build.sh b/scripts/github_CI/trigger-build.sh index 16b1c0d67a2..d45755c92c8 100644 --- a/scripts/github_CI/trigger-build.sh +++ b/scripts/github_CI/trigger-build.sh @@ -14,5 +14,5 @@ curl -L -X POST \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $1" \ -H "X-GitHub-Api-Version: 2022-11-28" \ -https://api.github.com/repos/sofa-framework/sofa/dispatches \ --d '{"event_type":"CI Build","client_payload":{"owner":"sofa-framework","branch":"$2","commit_hash":"$3","preset":"$4","ci-depends-on":"$5", "with-all-tests":"$6", "force-full-build":"7", "out-of-tree-build":"$8", "generate-binaries":"$9"}}' +https://api.github.com/repos/bakpaul/sofa/dispatches \ +-d '{"event_type":"CI Build","client_payload":{"owner":"bakpaul","branch":"$2","commit_hash":"$3","preset":"$4","ci-depends-on":"$5", "with-all-tests":"$6", "force-full-build":"7", "out-of-tree-build":"$8", "generate-binaries":"$9"}}' From c0c59758ba6c1ee0bee24a027c65d964b8b0a6e7 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Thu, 13 Mar 2025 11:11:45 +0100 Subject: [PATCH 09/61] fix indentation --- .github/workflows/filter-build.yml | 140 ++++++++++++++--------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index fce5540ce3d..ce9812024cf 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -48,81 +48,81 @@ jobs: with: python-version: '3.12' - - name: Install pip packages - run: | - pip install python-graphql-client - pip install requests + - name: Install pip packages + run: | + pip install python-graphql-client + pip install requests - - name: Run on dispatch - if: ${{ github.event_name == 'workflow_dispatch' }} - run: | - echo "This step runs only for binary generation." + - name: Run on dispatch + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + echo "This step runs only for binary generation." - BRANCH=${{ github.event.inputs.branch }} - COMMIT=${{ github.event.inputs.commit }} + BRANCH=${{ github.event.inputs.branch }} + COMMIT=${{ github.event.inputs.commit }} - # Validate branch format (e.g., v25.06) - if [[ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ]]; then - echo "Invalid branch name format: $BRANCH" - exit 1 - fi - echo "Branch name $BRANCH is valid." + # Validate branch format (e.g., v25.06) + if [[ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ]]; then + echo "Invalid branch name format: $BRANCH" + exit 1 + fi + echo "Branch name $BRANCH is valid." - # Check commit SHA format - if [[ ! "$COMMIT" =~ ^[a-fA-F0-9]{40}$ ]]; then - echo "Invalid commit SHA format: $COMMIT" - exit 1 - fi - # Check the commit exists in the repository - if git cat-file -e "$COMMIT" 2>/dev/null; then - echo "Commit $COMMIT is valid." - else - echo "Commit $COMMIT does not exist in the repository." - exit 1 - fi + # Check commit SHA format + if [[ ! "$COMMIT" =~ ^[a-fA-F0-9]{40}$ ]]; then + echo "Invalid commit SHA format: $COMMIT" + exit 1 + fi + # Check the commit exists in the repository + if git cat-file -e "$COMMIT" 2>/dev/null; then + echo "Commit $COMMIT is valid." + else + echo "Commit $COMMIT does not exist in the repository." + exit 1 + fi - ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} $BRANCH $COMMIT full None False True False True + ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} $BRANCH $COMMIT full None False True False True - - name: Run when nightly - if: github.event.schedule == '0 2 * * *' - run: | - echo "This step runs only for nightly build." - ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} standard None False True False True - - - name: Check push on master case - if: ${{ github.event_name == 'push'}} - run: | - echo "This step runs only when a commit is pushed on the master branch." - ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} full None True True False False - - - name: Run when PR is opened - if: ${{ github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize') }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} - COMMIT_SHA: ${{ github.sha }} - run: | - echo "This step runs only when a PR is opened or synchronized." - - # Trigger the Build action - python scripts/github_CI/checkPRInfoBeforeBuild.py - - - name: Run when PR comment is edited - if: ${{ github.event_name == 'pull_request_review_comment' && ( github.event.action == 'created'|| github.event.action == 'edited' ) }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} - COMMIT_SHA: ${{ github.sha }} - run: | - echo "This step runs only when a PR comment is created or edited." + - name: Run when nightly + if: github.event.schedule == '0 2 * * *' + run: | + echo "This step runs only for nightly build." + ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} standard None False True False True + + - name: Check push on master case + if: ${{ github.event_name == 'push'}} + run: | + echo "This step runs only when a commit is pushed on the master branch." + ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} full None True True False False + + - name: Run when PR is opened + if: ${{ github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize') }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} + COMMIT_SHA: ${{ github.sha }} + run: | + echo "This step runs only when a PR is opened or synchronized." + + # Trigger the Build action + python scripts/github_CI/checkPRInfoBeforeBuild.py + + + - name: Run when PR comment is edited + if: ${{ github.event_name == 'pull_request_review_comment' && ( github.event.action == 'created'|| github.event.action == 'edited' ) }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} + COMMIT_SHA: ${{ github.sha }} + run: | + echo "This step runs only when a PR comment is created or edited." - COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) - echo "Comment: $COMMENT_BODY" + COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) + echo "Comment: $COMMENT_BODY" - # Trigger the Build action if [ci-build] is in the comment - if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then - python scripts/github_CI/checkPRInfoBeforeBuild.py - fi - + # Trigger the Build action if [ci-build] is in the comment + if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then + python scripts/github_CI/checkPRInfoBeforeBuild.py + fi From 1fade703e727a673a998ee983806922502f30275 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Thu, 13 Mar 2025 11:17:53 +0100 Subject: [PATCH 10/61] add checkout code action --- .github/workflows/filter-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index ce9812024cf..92adb307b72 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -52,6 +52,9 @@ jobs: run: | pip install python-graphql-client pip install requests + + - name: Check out code + uses: actions/checkout@v2 - name: Run on dispatch if: ${{ github.event_name == 'workflow_dispatch' }} From a521a1ae227a02e7fa2dbb18273dd0f0fb0766e2 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 18 Mar 2025 11:03:18 +0100 Subject: [PATCH 11/61] Add backbone --- .github/workflows/build-and-tests.yml | 124 ++++++++++++++++++++++++++ .github/workflows/ci.yml | 68 -------------- 2 files changed, 124 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/build-and-tests.yml delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/build-and-tests.yml b/.github/workflows/build-and-tests.yml new file mode 100644 index 00000000000..5e17541170d --- /dev/null +++ b/.github/workflows/build-and-tests.yml @@ -0,0 +1,124 @@ +name: Compile SOFA and run Tests +on: + workflow_call: + inputs: + owner: + type: string + required: true + branch: + type: string + required: true + commit_hash: + type: string + required: true + preset: + type: string + required: true + python_version: + type: string + required: true + ci-depends-on: + type: string + required: false + with-all-tests: + type: boolean + required: false + default: false + force-full-build: + type: boolean + required: false + default: false + generate-binaries: + type: boolean + required: false + default: false + out-of-tree-build: + type: string + required: false + +jobs: + build: + runs-on: [sh-ubuntu] + steps: + - name: Clone SOFA and CI + shell: bash + run: | + WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) + cd $WORKSPACE + + echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" + #TODO (for PR) : clone branch against which the PR is compared then merge the PR + + if [ -d $WORKSPACE/src ]; then + rm -rf $WORKSPACE/src + fi + mkdir $WORKSPACE/src && cd $WORKSPACE/src + SRC_DIR=$(pwd) + + git init + git remote add origin https://www.github.com/$GITHUB_REPOSITORY_OWNER/sofa.git + git fetch origin $GITHUB_WORKFLOW_SHA + git checkout FETCH_HEAD + + cd $WORKSPACE + + echo "Cloning CI" + #TODO : clone ci branch with same name as the sofa branch against which the PR is compared OR the branch from which the commit comes + + if [ -d $WORKSPACE/ci ]; then + rm -rf $WORKSPACE/ci + fi + mkdir $WORKSPACE/ci && cd $WORKSPACE/ci + CI_DIR=$(pwd) + + git init + git remote add origin https://www.github.com/sofa-framework/ci.git + git fetch origin master + git checkout FETCH_HEAD + + cd $WORKSPACE + + #TODO: transform infos to force-full-build + if [ -d $WORKSPACE/build ]; then + rm -rf $WORKSPACE/build + fi + mkdir $WORKSPACE/build + + - name: Notify dashboard + shell: bash + run: | + #TODO + + - name: Build + continue-on-error: true + shell: bash + run: | + #TODO + #HEre we use docker if possible + + - name: Notify dashboard + shell: bash + run: | + #TODO + + - name: Launch tests + continue-on-error: true + if: steps.Build.outcome == 'success' + shell: bash + run: | + #TODO + #Use new action with workflow call + + - name: Publish artifacts + if: steps.Build.outcome == 'success' && inputs.generate-binaries == true + shell: bash + run: | + #TODO + #Use new action with workflow call + + - name: Publish logs + if: always() + shell: bash + run: | + #TODO + #Use new action with workflow call \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 82e7aede90e..00000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Compile SOFA and run Tests -on: - workflow_dispatch: - pull_request: - push: - -jobs: - setup_build: - runs-on: [self-hosted, sh-ubuntu] - steps: - - name: Clone SOFA and CI - shell: bash - run: | - WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) - cd $WORKSPACE - - echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" - #TODO (for PR) : clone branch against which the PR is compared then merge the PR - - if [ -d $WORKSPACE/src ]; then - rm -rf $WORKSPACE/src - fi - mkdir $WORKSPACE/src && cd $WORKSPACE/src - SRC_DIR=$(pwd) - - git init - git remote add origin https://www.github.com/$GITHUB_REPOSITORY_OWNER/sofa.git - git fetch origin $GITHUB_WORKFLOW_SHA - git checkout FETCH_HEAD - - cd $WORKSPACE - - echo "Cloning CI" - #TODO : clone ci branch with same name as the sofa branch against which the PR is compared OR the branch from which the commit comes - - if [ -d $WORKSPACE/ci ]; then - rm -rf $WORKSPACE/ci - fi - mkdir $WORKSPACE/ci && cd $WORKSPACE/ci - CI_DIR=$(pwd) - - git init - git remote add origin https://www.github.com/sofa-framework/ci.git - git fetch origin master - git checkout FETCH_HEAD - - cd $WORKSPACE - - #TODO: transform infos to force-full-build - if [ -d $WORKSPACE/build ]; then - rm -rf $WORKSPACE/build - fi - mkdir $WORKSPACE/build - - #TODO: transform job-type (PR, commit, nightly) in a set of parameter (core, standard, full) - #TODO: transform job-type + infos from messages in a set of tests to run (unit, scenes, regression) - #TODO: transform infos from messages in a set of ci-depends-on - - #TODO: launch the container if needed (e.g. for ubuntu) - #TODO: launch sofa-ci-build-action with parameters : building scope (core, standard, full), - # OS type, - # python version, - # tests sets (list taken from {unit, scenes, regression}), - # ci-depends-on, - # OOT cmake option list (empty here) - - - From 6331407be05c3ca88cf7a121a53eb1e8342892b1 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 18 Mar 2025 12:03:40 +0100 Subject: [PATCH 12/61] Cloning SOFA working --- .github/workflows/build-and-tests.yml | 50 ++++++++++++++++++--------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-and-tests.yml b/.github/workflows/build-and-tests.yml index 5e17541170d..6f9dee40c1e 100644 --- a/.github/workflows/build-and-tests.yml +++ b/.github/workflows/build-and-tests.yml @@ -2,19 +2,25 @@ name: Compile SOFA and run Tests on: workflow_call: inputs: - owner: + sofa-branch-name: type: string required: true - branch: + sofa-commit-sha: type: string required: true - commit_hash: + pr-owner-url: type: string - required: true + required: false + pr-branch-name: + type: string + required: false + pr-commit-sha: + type: string + required: false preset: type: string required: true - python_version: + python-version: type: string required: true ci-depends-on: @@ -43,25 +49,37 @@ jobs: - name: Clone SOFA and CI shell: bash run: | + ## Go to workspace given by the runner (this file is generated by the pre-build script) WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) cd $WORKSPACE + ## Clone sofa and merge origin master echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" - #TODO (for PR) : clone branch against which the PR is compared then merge the PR + if [ -d $WORKSPACE/sofa ]; then + rm -rf $WORKSPACE/sofa + if - if [ -d $WORKSPACE/src ]; then - rm -rf $WORKSPACE/src - fi - mkdir $WORKSPACE/src && cd $WORKSPACE/src - SRC_DIR=$(pwd) + SRC_DIR=$(pwd)/sofa - git init - git remote add origin https://www.github.com/$GITHUB_REPOSITORY_OWNER/sofa.git - git fetch origin $GITHUB_WORKFLOW_SHA - git checkout FETCH_HEAD + if [ -z "${{ inputs.pr-owner-url }}" ]; then + echo "This is a PR, merging branch ${{ inputs.pr-branch-name }} from remote ${{ inputs.pr-owner-url }} into origin branch ${{ inputs.sofa-branch-name }}" + git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/sofa-framework/sofa + git remote add pr ${{ inputs.pr-owner-url }} + git fetch pr + + if [ "${{ inputs.pr-commit-sha }}" == "HEAD" ]; then + git merge ${{ inputs.pr-branch-name }} + else + git merge ${{ inputs.pr-commit-sha }} + fi + else + echo "This is not a PR: checking out sha ${{ inputs.sofa-commit-sha }} from branch ${{ inputs.sofa-branch-name }}" + git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/sofa-framework/sofa + git checkout ${{ inputs.sofa-commit-sha }} + fi + cd $WORKSPACE - echo "Cloning CI" #TODO : clone ci branch with same name as the sofa branch against which the PR is compared OR the branch from which the commit comes From 896e09cd3d80911c905a955dac6e8289bff36e36 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 18 Mar 2025 12:06:08 +0100 Subject: [PATCH 13/61] Forgot to save --- .github/workflows/build-and-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-tests.yml b/.github/workflows/build-and-tests.yml index 6f9dee40c1e..d4d8c62d1b8 100644 --- a/.github/workflows/build-and-tests.yml +++ b/.github/workflows/build-and-tests.yml @@ -78,7 +78,10 @@ jobs: git checkout ${{ inputs.sofa-commit-sha }} fi - + + ## Clone CI and use ci-depends-on structure + # Use jq to explore json graph to see if ci is depends-on + cd $WORKSPACE echo "Cloning CI" #TODO : clone ci branch with same name as the sofa branch against which the PR is compared OR the branch from which the commit comes From 7c2d16a363016a10b4db16fd07451015fdad6851 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Tue, 18 Mar 2025 09:45:28 +0100 Subject: [PATCH 14/61] add chmod to run script --- .github/workflows/filter-build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 92adb307b72..c7844f10c72 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -83,19 +83,22 @@ jobs: echo "Commit $COMMIT does not exist in the repository." exit 1 fi - + + chmod +x /scripts/github_CI/trigger-build.sh ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} $BRANCH $COMMIT full None False True False True - name: Run when nightly if: github.event.schedule == '0 2 * * *' run: | echo "This step runs only for nightly build." + chmod +x /scripts/github_CI/trigger-build.sh ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} standard None False True False True - name: Check push on master case if: ${{ github.event_name == 'push'}} run: | echo "This step runs only when a commit is pushed on the master branch." + chmod +x /scripts/github_CI/trigger-build.sh ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} full None True True False False - name: Run when PR is opened From c2db438814a4c9525863d712c02cc00c061a61c3 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Tue, 18 Mar 2025 14:14:52 +0100 Subject: [PATCH 15/61] Add call-workflow, refactor env variables, update python script --- .github/workflows/filter-build.yml | 257 +++++++++++++------- scripts/github_CI/checkPRInfoBeforeBuild.py | 162 +++++++----- scripts/github_CI/trigger-build.sh | 18 -- 3 files changed, 271 insertions(+), 166 deletions(-) delete mode 100644 scripts/github_CI/trigger-build.sh diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index c7844f10c72..80fe433c218 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -12,11 +12,23 @@ on: branch: description: 'Specify the stable branch to use to generate the new binaries' required: true + type: string commit_sha: description: 'Commit SHA to check out' required: true default: 'head' - + type: string + python_version: + description: 'Version of Python used' + required: true + default: '3.12' + type: string + generate_binaries: + description: 'Boolean to generate the binaries' + required: true + default: 'true' + type: boolean + # Nightly build schedule: - cron: '0 2 * * *' # Evevery night @@ -26,7 +38,7 @@ on: types: [opened, synchronize] # Comment on PR - pull_request_review_comment: + issue_comment: types: [created, edited] # CI for dashboard master @@ -37,98 +49,169 @@ on: # =============================================================== # =============================================================== +env: + SOFA_BRANCH_NAME: "master" + SOFA_COMMIT_SHA: "HEAD" + PRESET: "full" + PYTHON_VERSION: "3.12" + CI_DEPENDS_ON: "" + WITH_ALL_TESTS: false + FORCE_FULL_BUILD: false + OUT_OF_TREE_BUILD: "" + GENERATE_BINARIES: false + PR_OWNER_URL: "" + PR_BRANCH_NAME: "" + PR_COMMIT_SHA: "HEAD" + jobs: filter_build: runs-on: ubuntu-latest + strategy: + matrix: + sofa_version: [master, v24.12] ### if: ${{ github.repository_owner == 'sofa-framework' }} steps: - - name: Set up python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install pip packages - run: | - pip install python-graphql-client - pip install requests - - - name: Check out code - uses: actions/checkout@v2 - - - name: Run on dispatch - if: ${{ github.event_name == 'workflow_dispatch' }} - run: | - echo "This step runs only for binary generation." + - name: Run on dispatch + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + echo "This step runs only for binary generation." + + BRANCH=${{ github.event.inputs.branch }} + COMMIT=${{ github.event.inputs.commit_sha }} + PYTHON=${{ github.event.inputs.python_version }} + PRESET=${{ github.event.inputs.preset }} + GENERATE_BINARIES=${{ github.event.inputs.generate_binaries }} + + # Validate branch format (e.g., v25.06) + if [[ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ]]; then + echo "Error: Invalid branch name format: $BRANCH" + exit 1 + fi + echo "Branch name $BRANCH is valid." + + # Check commit SHA format + if [[ ! "$COMMIT" =~ ^[a-fA-F0-9]{40}$ ]]; then + echo "Error: Invalid commit SHA format: $COMMIT" + exit 1 + fi + # Check the commit exists in the repository + if git cat-file -e "$COMMIT" 2>/dev/null; then + echo "Commit $COMMIT is valid." + else + echo "Commit $COMMIT does not exist in the repository." + exit 1 + fi - BRANCH=${{ github.event.inputs.branch }} - COMMIT=${{ github.event.inputs.commit }} + # Validate Python version format (e.g., 3.12) + if [[ ! "PYTHON" =~ ^[0-9]{1}\.[0-9]{2}$ ]]; then + if [[ ! "PYTHON" =~ ^3\.(9|1[0-8])$ ]]; then + echo "Error: Invalid Python version format: PYTHON" + exit 1 + fi + fi + echo "Python version $PYTHON is valid." - # Validate branch format (e.g., v25.06) - if [[ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ]]; then - echo "Invalid branch name format: $BRANCH" - exit 1 - fi - echo "Branch name $BRANCH is valid." - - # Check commit SHA format - if [[ ! "$COMMIT" =~ ^[a-fA-F0-9]{40}$ ]]; then - echo "Invalid commit SHA format: $COMMIT" - exit 1 - fi - # Check the commit exists in the repository - if git cat-file -e "$COMMIT" 2>/dev/null; then - echo "Commit $COMMIT is valid." - else - echo "Commit $COMMIT does not exist in the repository." - exit 1 - fi - - chmod +x /scripts/github_CI/trigger-build.sh - ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} $BRANCH $COMMIT full None False True False True - - - name: Run when nightly - if: github.event.schedule == '0 2 * * *' - run: | - echo "This step runs only for nightly build." - chmod +x /scripts/github_CI/trigger-build.sh - ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} standard None False True False True - - - name: Check push on master case - if: ${{ github.event_name == 'push'}} - run: | - echo "This step runs only when a commit is pushed on the master branch." - chmod +x /scripts/github_CI/trigger-build.sh - ./scripts/github_CI/trigger-build.sh ${{ secrets.GITHUB_TOKEN }} master ${{ github.sha }} full None True True False False - - - name: Run when PR is opened - if: ${{ github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize') }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} - COMMIT_SHA: ${{ github.sha }} - run: | - echo "This step runs only when a PR is opened or synchronized." + # Check if the preset is valid + ALLOWED_PRESETS=("minimal" "standard" "full" "minimal-dev" "standard-dev" "full-dev") + if [[ ! " ${ALLOWED_PRESETS[@]} " =~ " ${PRESET} " ]]; then + echo "Error: Invalid preset $PRESET. Must be one of: ${ALLOWED_PRESETS[*]}" + exit 1 + fi + echo "Preset $PRESET is valid." + + # Validate generate_binaries (must be true or false) + if [[ "$GENERATE_BINARIES" != "true" && "$GENERATE_BINARIES" != "false" ]]; then + echo "Error: 'generate_binaries' must be either 'true' or 'false'." + exit 1 + fi + + # Save all information in environment variables + echo "SOFA_BRANCH_NAME=$BRANCH" >> $GITHUB_ENV + echo "SOFA_COMMIT_SHA=$COMMIT" >> $GITHUB_ENV + echo "PRESET=$PRESET" >> $GITHUB_ENV + echo "PYTHON_VERSION=$PYTHON" >> $GITHUB_ENV + echo "GENERATE_BINARIES=$GENERATE_BINARIES" >> $GITHUB_ENV + + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: ${{ PYTHON_VERSION }} + + - name: Install pip packages + run: | + pip install python-graphql-client + pip install requests + + - name: Check out code + uses: actions/checkout@v2 + + - name: Run when nightly + if: github.event.schedule == '0 2 * * *' + run: | + echo "This step runs only for nightly builds." + echo "SOFA_BRANCH_NAME=${{ matrix.sofa_version }}" >> $GITHUB_ENV + echo "PRESET=standard" >> $GITHUB_ENV + echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV + echo "GENERATE_BINARIES=true" >> $GITHUB_ENV + + - name: Check push on master case + if: ${{ github.event_name == 'push'}} + run: | + echo "This step runs only for push on the master branch." + echo "COMMIT_SHA=${{ github.after }}>> $GITHUB_ENV + echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV + echo "FORCE_FULL_BUILD=true" >> $GITHUB_ENV + + - name: Run when PR is opened + if: ${{ github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize') }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} + PR_COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + run: | + echo "This step runs only when a PR is opened or synchronized." + + # Trigger the Build action + python scripts/github_CI/checkPRInfoBeforeBuild.py - # Trigger the Build action - python scripts/github_CI/checkPRInfoBeforeBuild.py - - - - name: Run when PR comment is edited - if: ${{ github.event_name == 'pull_request_review_comment' && ( github.event.action == 'created'|| github.event.action == 'edited' ) }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - OWNER_NAME: ${{ github.event.pull_request.head.repo.owner.login }} - COMMIT_SHA: ${{ github.sha }} - run: | - echo "This step runs only when a PR comment is created or edited." - - COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) - echo "Comment: $COMMENT_BODY" + + - name: Run when PR comment is edited + if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request && ( github.event.action == 'created'|| github.event.action == 'edited' ) }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.pull_request.number }} + OWNER_NAME: ${{ github.event.issue.pull_request.head.repo.owner.login }} + PR_COMMIT_SHA: ${{ github.event.issue.pull_request.head.sha }} + run: | + echo "This step runs only when a PR comment is created or edited." + COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) + echo "Comment: $COMMENT_BODY" - # Trigger the Build action if [ci-build] is in the comment - if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then - python scripts/github_CI/checkPRInfoBeforeBuild.py - fi + # Trigger the Build action if [ci-build] is in the comment + if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then + python scripts/github_CI/checkPRInfoBeforeBuild.py + fi + + # =============================================================== + # =============================================================== + + # Trigger the build with all parameters + call-workflow-passing-data: + uses: sofa-framework/sofa/.github/workflows/build-and-test.yml@master + with: + branch-name: ${{ SOFA_BRANCH_NAME }} + commit-sha: ${{ SOFA_COMMIT_SHA }} + preset: ${{ PRESET }} + python-version: ${{ PYTHON_VERSION }} + ci-depends-on: ${{ CI_DEPENDS_ON }} + with-all-tests: ${{ WITH_ALL_TESTS }} + force-full-build: ${{ FORCE_FULL_BUILD }} + out-of-tree-build: ${{ OUT_OF_TREE_BUILD }} + generate-binaries: ${{ GENERATE_BINARIES }} + pr-owner-url: ${{ PR_OWNER_URL }} + pr-branch-name: ${{ PR_BRANCH_NAME }} + pr-commit-sha: ${{ PR_COMMIT_SHA }} + secrets: + personal_access_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py index 44ff9527752..ef34186c835 100644 --- a/scripts/github_CI/checkPRInfoBeforeBuild.py +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -1,15 +1,11 @@ #!python -import os -import requests -from python_graphql_client import GraphqlClient +import os, re, requests - -client = GraphqlClient(endpoint="https://api.github.com/graphql") GITHUB_TOKEN = os.getenv('GITHUB_TOKEN') PR_NUMBER = os.getenv('PR_NUMBER') OWNER_NAME = os.getenv('OWNER_NAME') -COMMIT_SHA = os.getenv('COMMIT_SHA') +PR_COMMIT_SHA = os.getenv('PR_COMMIT_SHA') if not GITHUB_TOKEN or not PR_NUMBER or not REPO_NAME: @@ -32,7 +28,7 @@ is_draft_pr = False with_all_tests_found = False force_full_build_found = False - +dependency_dict = {} # ======================================================================== @@ -53,6 +49,7 @@ def check_labels(): to_review_label_found = True print("PR is marked as 'to review'.") + # ======================================================================== # Check the PR draft status @@ -71,6 +68,9 @@ def check_if_draft(): if is_draft_pr: print("The pull request is a draft. The Bash script will not run.") + ## TODO : save PR_OWNER_URL, PR_BRANCH_NAME, PR_COMMIT_SHA + + # ======================================================================== # Check PR comments for "[with-all-tests]" and "[force-full-build]" @@ -97,28 +97,53 @@ def check_comments(): # ======================================================================== +# Export all needed PR information +def export_pr_info(): + pr_url = f"{API_URL}/pulls/{PR_NUMBER}" + response = requests.get(pr_url, headers=HEADERS) -# Execute the checks -check_labels() -check_if_draft() + if response.status_code != 200: + print(f"Failed to fetch pull request details: {response.status_code}") + exit(1) + pr_data = response.json() + + pr_url = str(pr_data['user']['html_url']) + str(pr_data['repo']['name']) + pr_branch_name = pr_data['head']['ref'] + pr_commit_sha = pr_data['head']['sha'] + + print("PR comes from the repository: "+str(pr_url)) + print("PR branch name is: "+str(pr_branch_name)) + print("PR commit sha is: "+str(pr_commit_sha)) + + with open(os.environ["GITHUB_ENV"], "a") as env_file: + env_file.write(f"PR_OWNER_URL={pr_url}\n") + env_file.write(f"PR_BRANCH_NAME={pr_branch_name}\n") + env_file.write(f"PR_COMMIT_SHA={pr_commit_sha}\n") + + ## TODO : pr_data.get('mergeable', False) could also let us know if it is mergeable + + +# ======================================================================== + +# Extract repositories from ci-depends-on +def extract_ci_depends_on(): + global dependency_dict -# Trigger the build if conditions are met -if to_review_label_found and not is_draft_pr: - - # Check compilation options in PR comments - check_comments() - pr_url = f"{API_URL}/pulls/{PR_NUMBER}" response = requests.get(pr_url, headers=HEADERS) + + if response.status_code != 200: + print(f"Failed to fetch pull request details: {response.status_code}") + exit(1) + pr_data = response.json() - branch = pr_data.get("head", {}).get("ref", None) - - + # Extract the PR description and look for [ci-depends-on ...] patterns pr_body = pr_data.get("body", "") ci_depends_on = [] + # Search in each line for the pattern "[ci-depends-on ...]" for line in pr_body.splitlines(): match = re.search(r'\[ci-depends-on (.+?)\]', line) if match: @@ -126,47 +151,62 @@ def check_comments(): ci_depends_on.append(dependency) print(f"Found ci-depends-on dependency: {dependency}") - # GitHub repository details - API_URL_DISPATCH = f"https://api.github.com/repos/bakpaul/sofa/dispatches" - - # JSON payload for the dispatch event - PAYLOAD = { - "event_type": "CI Build", - "client_payload": { - "owner": {OWNER_NAME}, - "branch": {branch}, - "commit_hash": {COMMIT_SHA}, - "preset": "full", - "ci-depends-on": {ci_depends_on}, - "with-all-tests": {with_all_tests_found}, - "force-full-build": {force_full_build_found}, - "out-of-tree-build": "False", - "generate-binaries": "False", - }, - } - - # Headers for the GitHub API request - REQUEST_HEADERS = { - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {GITHUB_TOKEN}", - "X-GitHub-Api-Version": "2022-11-28", - } - - # Perform the API request - try: - response = requests.post(API_URL_DISPATCH, json=PAYLOAD, headers=REQUEST_HEADERS) - - # Check for successful request - if response.status_code == 204: - print("CI Build event triggered successfully.") - else: - print(f"Failed to trigger CI Build event. Status code: {response.status_code}") - print("Response:", response.text) - sys.exit(1) - except requests.RequestException as e: - print("Error during the API request:", e) - sys.exit(1) -else: - print("Conditions not met. Bash script will not run.") + # Ensure the URL is in the expected dependency format, e.g. https://github.com/sofa-framework/Sofa.Qt/pull/6 + parts = dependency.split('/') + if len(parts) != 6 or parts[0] != 'https:' or parts[1] != '' or parts[2] != 'github.com': + raise ValueError("") + print(f"Invalid URL ci-depends-on format: {dependency}") + exit(1) + + owner = parts[3] + repo = parts[4] + pull_number = parts[5] + dependency_request_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pull_number}" + + response = requests.get(dependency_request_url, headers=HEADERS) + + if response.status_code != 200: + print(f"Failed to fetch pull request details: {response.status_code}") + exit(1) + + dependency_pr_data = response.json() + + key = dependency_pr_data['repo']['name'] #Sofa.Qt + repo_url = dependency_pr_data['html_url'] #https://github.com/sofa-framework/Sofa.Qt + branch_name = dependency_pr_data['head']['ref'] #my_feature_branch + + dependency_dict[key] = { + "repo_url": repo_url, + "branch_name": branch_name + } + # ======================================================================== +# Script core +# ======================================================================== + + +# Execute the checks +check_labels() +check_if_draft() + +# Trigger the build if conditions are met +if to_review_label_found and not is_draft_pr: + # Export PR information (url, name, sha) + export_pr_info() + + # Check compilation options in PR comments + check_comments() + + with open(os.environ["GITHUB_ENV"], "a") as env_file: + env_file.write(f"WITH_ALL_TESTS={with_all_tests_found}\n") + env_file.write(f"FORCE_FULL_BUILD={force_full_build_found}\n") + + # Extract dependency repositories + extract_ci_depends_on() + + with open(os.environ["GITHUB_ENV"], "a") as env_file: + env_file.write(f"CI_DEPENDS_ON={dependency_dict}\n") + + +# ======================================================================== \ No newline at end of file diff --git a/scripts/github_CI/trigger-build.sh b/scripts/github_CI/trigger-build.sh deleted file mode 100644 index d45755c92c8..00000000000 --- a/scripts/github_CI/trigger-build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Check if exactly 8 arguments are provided -if [ "$#" -ne 9 ]; then - echo "Error: Exactly 9 arguments are required." - echo "Usage: $0 " - exit 1 -fi - -# Trigger the build action with all needed input: -# branch, hash, preset, ci-depends-on, with-all-tests, force-full-build, out-of-tree-build, label-wip/postponed, generate-binaries -sudo apt install curl -curl -L -X POST \ --H "Accept: application/vnd.github+json" \ --H "Authorization: Bearer $1" \ --H "X-GitHub-Api-Version: 2022-11-28" \ -https://api.github.com/repos/bakpaul/sofa/dispatches \ --d '{"event_type":"CI Build","client_payload":{"owner":"bakpaul","branch":"$2","commit_hash":"$3","preset":"$4","ci-depends-on":"$5", "with-all-tests":"$6", "force-full-build":"7", "out-of-tree-build":"$8", "generate-binaries":"$9"}}' From a40fb3545e276e0ddaf3af88fb142497cefa6d0d Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Tue, 18 Mar 2025 14:20:05 +0100 Subject: [PATCH 16/61] remove useless comment --- scripts/github_CI/checkPRInfoBeforeBuild.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py index ef34186c835..eda5f104579 100644 --- a/scripts/github_CI/checkPRInfoBeforeBuild.py +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -68,8 +68,6 @@ def check_if_draft(): if is_draft_pr: print("The pull request is a draft. The Bash script will not run.") - ## TODO : save PR_OWNER_URL, PR_BRANCH_NAME, PR_COMMIT_SHA - # ======================================================================== From 9a68617f8805e3c75398e454b9de52061b2ba660 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 19 Mar 2025 14:30:03 +0100 Subject: [PATCH 17/61] Apply suggestions from code review --- .github/workflows/filter-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 80fe433c218..af82cd221e7 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -201,8 +201,8 @@ jobs: call-workflow-passing-data: uses: sofa-framework/sofa/.github/workflows/build-and-test.yml@master with: - branch-name: ${{ SOFA_BRANCH_NAME }} - commit-sha: ${{ SOFA_COMMIT_SHA }} + sofa-branch-name: ${{ SOFA_BRANCH_NAME }} + sofa-commit-sha: ${{ SOFA_COMMIT_SHA }} preset: ${{ PRESET }} python-version: ${{ PYTHON_VERSION }} ci-depends-on: ${{ CI_DEPENDS_ON }} From 08cf72e1bd1f87d213c0f0a07ec727132ab401d3 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 19 Mar 2025 14:33:32 +0100 Subject: [PATCH 18/61] Apply suggestions from code review --- scripts/github_CI/checkPRInfoBeforeBuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py index eda5f104579..f1bc7d89ad0 100644 --- a/scripts/github_CI/checkPRInfoBeforeBuild.py +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -207,4 +207,4 @@ def extract_ci_depends_on(): env_file.write(f"CI_DEPENDS_ON={dependency_dict}\n") -# ======================================================================== \ No newline at end of file +# ======================================================================== From a9013652f5234ca0e22e2c1ca0ca3c40031194bb Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 19 Mar 2025 16:27:12 +0100 Subject: [PATCH 19/61] Update filter-build.yml --- .github/workflows/filter-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index af82cd221e7..2587f72ecc4 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -199,7 +199,7 @@ jobs: # Trigger the build with all parameters call-workflow-passing-data: - uses: sofa-framework/sofa/.github/workflows/build-and-test.yml@master + uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master with: sofa-branch-name: ${{ SOFA_BRANCH_NAME }} sofa-commit-sha: ${{ SOFA_COMMIT_SHA }} From 3ff4ae389a2559921b6cb198cfb1b99209072db3 Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Wed, 19 Mar 2025 16:40:10 +0100 Subject: [PATCH 20/61] Finalize clone sofa and ci (#7) * Push space to creatye PR * Finished clone sofa and ci stage * Rename action * fix some typos in filter script --- ...build-and-tests.yml => build-and-test.yml} | 45 ++++++++++++------- .github/workflows/filter-build.yml | 33 ++++++++------ 2 files changed, 47 insertions(+), 31 deletions(-) rename .github/workflows/{build-and-tests.yml => build-and-test.yml} (73%) diff --git a/.github/workflows/build-and-tests.yml b/.github/workflows/build-and-test.yml similarity index 73% rename from .github/workflows/build-and-tests.yml rename to .github/workflows/build-and-test.yml index d4d8c62d1b8..8276a78a5f2 100644 --- a/.github/workflows/build-and-tests.yml +++ b/.github/workflows/build-and-test.yml @@ -51,15 +51,17 @@ jobs: run: | ## Go to workspace given by the runner (this file is generated by the pre-build script) WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) + echo "WORKSPACE=$WORKSPACE" >> $GITHUB_ENV + cd $WORKSPACE ## Clone sofa and merge origin master echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" if [ -d $WORKSPACE/sofa ]; then - rm -rf $WORKSPACE/sofa + rm -rf $WORKSPACE/sofa if - SRC_DIR=$(pwd)/sofa + echo "SRC_DIR=$SRC_DIR" >> $GITHUB_ENV if [ -z "${{ inputs.pr-owner-url }}" ]; then echo "This is a PR, merging branch ${{ inputs.pr-branch-name }} from remote ${{ inputs.pr-owner-url }} into origin branch ${{ inputs.sofa-branch-name }}" @@ -78,32 +80,41 @@ jobs: git checkout ${{ inputs.sofa-commit-sha }} fi - + ## Clone CI and use ci-depends-on structure - # Use jq to explore json graph to see if ci is depends-on + ci_branch="inputs.sofa-branch-name" + ci_repo_url="https://www.github.com/sofa-framework/ci" - cd $WORKSPACE - echo "Cloning CI" - #TODO : clone ci branch with same name as the sofa branch against which the PR is compared OR the branch from which the commit comes + # check if ci has a ci-depends-on + ci_ci_depends_on=$(echo "${{ inputs.ci-depends-on }}" | jq .ci) + if [ "$ci_ci_depends_on" != "null" ]; then + ci_repo_url=$(echo "${{ inputs.ci-depends-on }}" | jq .ci.repo_url) + ci_branch=$(echo "${{ inputs.ci-depends-on }}" | jq .ci.branch_name) + fi + + echo "Cloning CI from remote ${ci_repo_url}, selecting branch ${ci_branch}" if [ -d $WORKSPACE/ci ]; then - rm -rf $WORKSPACE/ci + rm -rf $WORKSPACE/ci fi - mkdir $WORKSPACE/ci && cd $WORKSPACE/ci - CI_DIR=$(pwd) + CI_DIR=$WORKSPACE/ci + echo "CI_DIR=$CI_DIR" >> $GITHUB_ENV git init - git remote add origin https://www.github.com/sofa-framework/ci.git - git fetch origin master - git checkout FETCH_HEAD + git clone -b ${ci_branch} --single-branch ${ci_repo_url} cd $WORKSPACE - #TODO: transform infos to force-full-build - if [ -d $WORKSPACE/build ]; then - rm -rf $WORKSPACE/build + ## Setup build folder + if [ "${{ inputs.force-full-build }}" == "true" ] && [ -d $WORKSPACE/build ]; then + rm -rf $WORKSPACE/build fi - mkdir $WORKSPACE/build + if [ ! -d $WORKSPACE/build ]; then + mkdir $WORKSPACE/build + fi + BUILD_DIR=$WORKSPACE/build + echo "BUILD_DIR=$BUILD_DIR" >> $GITHUB_ENV + - name: Notify dashboard shell: bash diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 2587f72ecc4..368ee0e8ad6 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -23,10 +23,15 @@ on: required: true default: '3.12' type: string + preset: + description: 'Preset to use from CMakePresets.json file in main repository' + required: true + default: 'full' + type: string generate_binaries: description: 'Boolean to generate the binaries' required: true - default: 'true' + default: true type: boolean # Nightly build @@ -136,7 +141,7 @@ jobs: - name: Set up python uses: actions/setup-python@v5 with: - python-version: ${{ PYTHON_VERSION }} + python-version: ${{ env.PYTHON_VERSION }} - name: Install pip packages run: | @@ -201,17 +206,17 @@ jobs: call-workflow-passing-data: uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master with: - sofa-branch-name: ${{ SOFA_BRANCH_NAME }} - sofa-commit-sha: ${{ SOFA_COMMIT_SHA }} - preset: ${{ PRESET }} - python-version: ${{ PYTHON_VERSION }} - ci-depends-on: ${{ CI_DEPENDS_ON }} - with-all-tests: ${{ WITH_ALL_TESTS }} - force-full-build: ${{ FORCE_FULL_BUILD }} - out-of-tree-build: ${{ OUT_OF_TREE_BUILD }} - generate-binaries: ${{ GENERATE_BINARIES }} - pr-owner-url: ${{ PR_OWNER_URL }} - pr-branch-name: ${{ PR_BRANCH_NAME }} - pr-commit-sha: ${{ PR_COMMIT_SHA }} + sofa-branch-name: ${{ env.SOFA_BRANCH_NAME }} + sofa-commit-sha: ${{ env.SOFA_COMMIT_SHA }} + preset: ${{ env.PRESET }} + python-version: ${{ env.PYTHON_VERSION }} + ci-depends-on: ${{ env.CI_DEPENDS_ON }} + with-all-tests: ${{ env.WITH_ALL_TESTS }} + force-full-build: ${{ env.FORCE_FULL_BUILD }} + out-of-tree-build: ${{ env.OUT_OF_TREE_BUILD }} + generate-binaries: ${{ env.GENERATE_BINARIES }} + pr-owner-url: ${{ env.PR_OWNER_URL }} + pr-branch-name: ${{ env.PR_BRANCH_NAME }} + pr-commit-sha: ${{ env.PR_COMMIT_SHA }} secrets: personal_access_token: ${{ secrets.GITHUB_TOKEN }} From 22aa451d4fe3df6de347b10832a38a13d5d4319a Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Fri, 21 Mar 2025 11:14:58 +0100 Subject: [PATCH 21/61] Add build step (#8) * Commit to open a PR * revert env modification * Move steps around + add os configuration * Configure step should be working * Move comment around * Step build --- .github/workflows/build-and-test.yml | 93 +++++++++++++++++++++++++--- .github/workflows/filter-build.yml | 24 +++---- 2 files changed, 98 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 8276a78a5f2..2111757cefc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -41,20 +41,52 @@ on: out-of-tree-build: type: string required: false + builder-os: + type: string + required: true + default: '["sh-ubuntu_gcc_release"]' + dash-info: + type: string + required: false + default: 'NONE' + description: 'Json scructure with three parameters {"COMMIT_HASH":"fullhsashofthecommit", }' jobs: build: - runs-on: [sh-ubuntu] + strategy: + matrix: + os: ${{ fromJson(inputs.os) }} + runs-on: ${{ matrix.os }} steps: - - name: Clone SOFA and CI + - name: Configure builder shell: bash run: | - ## Go to workspace given by the runner (this file is generated by the pre-build script) - WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) - echo "WORKSPACE=$WORKSPACE" >> $GITHUB_ENV - - cd $WORKSPACE + ## Go to workspace given by the runner (this file is generated by the pre-build script) + WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) + echo "WORKSPACE=$WORKSPACE" >> $GITHUB_ENV + + cd $WORKSPACE + + ## Setup github + # Git config (needed by CMake ExternalProject) + if ! git config --get user.name; then + git config --system user.name 'SOFA Bot' > /dev/null 2>&1 || + git config --global user.name 'SOFA Bot' > /dev/null 2>&1 || + git config user.name 'SOFA Bot' > /dev/null 2>&1 || + echo "WARNING: cannot setup git config" + fi + if ! git config --get user.email; then + git config --system user.email '<>' > /dev/null 2>&1 || + git config --global user.email '<>' > /dev/null 2>&1 || + git config user.email '<>' > /dev/null 2>&1 || + echo "WARNING: cannot setup git config" + fi + + ##TODO (if required) fix path too long for windows (see main.sh:201) + - name: Clone SOFA and CI + shell: bash + run: | ## Clone sofa and merge origin master echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" if [ -d $WORKSPACE/sofa ]; then @@ -119,7 +151,12 @@ jobs: - name: Notify dashboard shell: bash run: | + . ${CI_DIR}/scripts/utils.sh + . ${CI_DIR}/scripts/dashboard.sh + + #Really necessary ? #TODO + #Need more information. Might be good to - name: Build continue-on-error: true @@ -128,6 +165,48 @@ jobs: #TODO #HEre we use docker if possible + . ${CI_DIR}/scripts/utils.sh + + ## Setup env variables + CI_PYTHON3_VERSION=${{ inputs.python-version }} # Needed by load-vm-env + # TODO: be able to get hostname to load correct env file --> set + host_name="sofa-ci-dev-ubuntu2204-0" + NODE_NAME="$host_name" # Needed by load-vm-env + # then run + load-vm-env + + ## Deal with ci-depends-on + # Loop over each key-value pair in the JSON avoiding ci repository if inside + CMAKE_OPTIONS="" + for key in $(echo "${{ inputs.ci-depends-on }}" | jq -r 'keys[]' | grep -v '^ci$'); do + + repo_url=$(echo "${{ inputs.ci-depends-on }}" | jq -r ".\"$key\".repo_url") + branch_name=$(echo "${{ inputs.ci-depends-on }}" | jq -r ".\"$key\".branch_name") + + # Format the CMake flags for this key and append to the result + flag_repository="-D$(echo "$key" | gsub("\\.";"_" | ascii_upcase)_GIT_REPOSITORY=\"$repo_url\")" + flag_tag="-D$(echo "$key" | gsub("\\.";"_" | ascii_upcase)_GIT_TAG=\"$branch_name\")" + + # Append both flags to the result string with a space + CMAKE_OPTIONS="$CMAKE_OPTIONS $flag_repository $flag_tag " + done + + ## Configure the build: setup cmake variables + # retrive build type and compiler + BUILD_TYPE=$(echo ${{ matrix.os }} | awk -F '_' '{print $3}') + CONFIG=$(echo ${{ matrix.os }} | awk -F '_' '{print $1"_"$2}') + + BUILD_OPTIONS="build-scope-${{ inputs.preset }}" + if [ "${{ inputs.generate-binaries }}" == "true" ]; then + BUILD_OPTIONS="$BUILD_OPTIONS build-release-package" + fi + + . ${CI_DIR}/scripts/configure.sh "$BUILD_DIR" "$SRC_DIR" "$CONFIG" "$BUILD_TYPE" "$BUILD_OPTIONS" "$CMAKE_OPTIONS" + + ## Call to build + $SCRIPT_DIR/compile.sh "$BUILD_DIR" "$CONFIG" "$BUILD_OPTIONS" + + - name: Notify dashboard shell: bash run: | diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 368ee0e8ad6..0ae618171df 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -206,17 +206,17 @@ jobs: call-workflow-passing-data: uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master with: - sofa-branch-name: ${{ env.SOFA_BRANCH_NAME }} - sofa-commit-sha: ${{ env.SOFA_COMMIT_SHA }} - preset: ${{ env.PRESET }} - python-version: ${{ env.PYTHON_VERSION }} - ci-depends-on: ${{ env.CI_DEPENDS_ON }} - with-all-tests: ${{ env.WITH_ALL_TESTS }} - force-full-build: ${{ env.FORCE_FULL_BUILD }} - out-of-tree-build: ${{ env.OUT_OF_TREE_BUILD }} - generate-binaries: ${{ env.GENERATE_BINARIES }} - pr-owner-url: ${{ env.PR_OWNER_URL }} - pr-branch-name: ${{ env.PR_BRANCH_NAME }} - pr-commit-sha: ${{ env.PR_COMMIT_SHA }} + sofa-branch-name: ${{ SOFA_BRANCH_NAME }} + sofa-commit-sha: ${{ SOFA_COMMIT_SHA }} + preset: ${{ PRESET }} + python-version: ${{ PYTHON_VERSION }} + ci-depends-on: ${{ CI_DEPENDS_ON }} + with-all-tests: ${{ WITH_ALL_TESTS }} + force-full-build: ${{ FORCE_FULL_BUILD }} + out-of-tree-build: ${{ OUT_OF_TREE_BUILD }} + generate-binaries: ${{ GENERATE_BINARIES }} + pr-owner-url: ${{ PR_OWNER_URL }} + pr-branch-name: ${{ PR_BRANCH_NAME }} + pr-commit-sha: ${{ PR_COMMIT_SHA }} secrets: personal_access_token: ${{ secrets.GITHUB_TOKEN }} From 8831138fb9250508d9cf5bfbccd717c76fa2813b Mon Sep 17 00:00:00 2001 From: Hugo Date: Mon, 7 Apr 2025 16:29:35 +0200 Subject: [PATCH 22/61] Use step outputs to send variables in call workflow (#9) * Use step outputs to send variables in call workflow * add BUILDER_OS environment variable * add check for all tests in PR body * remove useless secret * udpate PR python script * add missing runs-on: ubuntu-latest * Fix build action/ ci-depends-on * remove runs-on * update python script * clean python script * fix export of booleans * Fix action --------- Co-authored-by: Paul Baksic --- .github/workflows/build-and-test.yml | 10 +- .github/workflows/filter-build.yml | 107 ++++++++++++++------ scripts/github_CI/checkPRInfoBeforeBuild.py | 30 ++++-- 3 files changed, 107 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2111757cefc..6ffce0b6021 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -55,7 +55,7 @@ jobs: build: strategy: matrix: - os: ${{ fromJson(inputs.os) }} + os: ${{ fromJson(inputs.builder-os) }} runs-on: ${{ matrix.os }} steps: - name: Configure builder @@ -123,6 +123,8 @@ jobs: ci_repo_url=$(echo "${{ inputs.ci-depends-on }}" | jq .ci.repo_url) ci_branch=$(echo "${{ inputs.ci-depends-on }}" | jq .ci.branch_name) fi + echo "CI_BRANCH=$ci_branch" >> $GITHUB_ENV + echo "Cloning CI from remote ${ci_repo_url}, selecting branch ${ci_branch}" @@ -165,6 +167,7 @@ jobs: #TODO #HEre we use docker if possible + . ${CI_DIR}/scripts/utils.sh ## Setup env variables @@ -184,8 +187,9 @@ jobs: branch_name=$(echo "${{ inputs.ci-depends-on }}" | jq -r ".\"$key\".branch_name") # Format the CMake flags for this key and append to the result - flag_repository="-D$(echo "$key" | gsub("\\.";"_" | ascii_upcase)_GIT_REPOSITORY=\"$repo_url\")" - flag_tag="-D$(echo "$key" | gsub("\\.";"_" | ascii_upcase)_GIT_TAG=\"$branch_name\")" + fixed_name=$(echo "$key" | awk '{gsub(/\./, "_"); print toupper($0)}') + flag_repository="-D${fixed_name}_GIT_REPOSITORY=\"$repo_url\")" + flag_tag="-D${fixed_name}_GIT_TAG=\"$branch_name\")" # Append both flags to the result string with a space CMAKE_OPTIONS="$CMAKE_OPTIONS $flag_repository $flag_tag " diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 0ae618171df..e1e8235ccbe 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -33,6 +33,10 @@ on: required: true default: true type: boolean + builder_os: + type: string + required: true + default: '["sh-ubuntu_gcc_release"]' # Nightly build schedule: @@ -54,19 +58,6 @@ on: # =============================================================== # =============================================================== -env: - SOFA_BRANCH_NAME: "master" - SOFA_COMMIT_SHA: "HEAD" - PRESET: "full" - PYTHON_VERSION: "3.12" - CI_DEPENDS_ON: "" - WITH_ALL_TESTS: false - FORCE_FULL_BUILD: false - OUT_OF_TREE_BUILD: "" - GENERATE_BINARIES: false - PR_OWNER_URL: "" - PR_BRANCH_NAME: "" - PR_COMMIT_SHA: "HEAD" jobs: filter_build: @@ -75,8 +66,38 @@ jobs: matrix: sofa_version: [master, v24.12] ### if: ${{ github.repository_owner == 'sofa-framework' }} - + outputs: + SOFA_BRANCH_NAME: ${{ steps.export-vars.outputs.SOFA_BRANCH_NAME }} + SOFA_COMMIT_SHA: ${{ steps.export-vars.outputs.SOFA_COMMIT_SHA }} + PRESET: ${{ steps.export-vars.outputs.PRESET }} + PYTHON_VERSION: ${{ steps.export-vars.outputs.PYTHON_VERSION }} + CI_DEPENDS_ON: ${{ steps.export-vars.outputs.CI_DEPENDS_ON }} + WITH_ALL_TESTS: ${{ steps.export-vars.outputs.WITH_ALL_TESTS }} + FORCE_FULL_BUILD: ${{ steps.export-vars.outputs.FORCE_FULL_BUILD }} + OUT_OF_TREE_BUILD: ${{ steps.export-vars.outputs.OUT_OF_TREE_BUILD }} + GENERATE_BINARIES: ${{ steps.export-vars.outputs.GENERATE_BINARIES }} + PR_OWNER_URL: ${{ steps.export-vars.outputs.PR_OWNER_URL }} + PR_BRANCH_NAME: ${{ steps.export-vars.outputs.PR_BRANCH_NAME }} + PR_COMMIT_SHA: ${{ steps.export-vars.outputs.PR_COMMIT_SHA }} + BUILDER_OS: ${{ steps.export-vars.outputs.BUILDER_OS }} + steps: + - name: Default values of environment variables + run: | + echo "SOFA_BRANCH_NAME=master" >> $GITHUB_ENV # SOFA_BRANCH_NAME: "master" + echo "SOFA_COMMIT_SHA=HEAD" >> $GITHUB_ENV # SOFA_COMMIT_SHA: "HEAD" + echo "PRESET=full" >> $GITHUB_ENV # PRESET: "full" + echo "PYTHON_VERSION=3.12" >> $GITHUB_ENV # PYTHON_VERSION: "3.12" + echo "CI_DEPENDS_ON=" >> $GITHUB_ENV # CI_DEPENDS_ON: "" + echo "WITH_ALL_TESTS=false" >> $GITHUB_ENV # WITH_ALL_TESTS: false + echo "FORCE_FULL_BUILD=false" >> $GITHUB_ENV # FORCE_FULL_BUILD: false + echo "OUT_OF_TREE_BUILD=" >> $GITHUB_ENV # OUT_OF_TREE_BUILD: "" + echo "GENERATE_BINARIES=false" >> $GITHUB_ENV # GENERATE_BINARIES: false + echo "PR_OWNER_URL=" >> $GITHUB_ENV # PR_OWNER_URL: "" + echo "PR_BRANCH_NAME=" >> $GITHUB_ENV # PR_BRANCH_NAME: "" + echo "PR_COMMIT_SHA=HEAD" >> $GITHUB_ENV # PR_COMMIT_SHA: "HEAD" + echo 'BUILDER_OS=["sh-ubuntu_gcc_release"]' >> $GITHUB_ENV # BUILDER_OS: ["sh-ubuntu_gcc_release"] + - name: Run on dispatch if: ${{ github.event_name == 'workflow_dispatch' }} run: | @@ -87,6 +108,7 @@ jobs: PYTHON=${{ github.event.inputs.python_version }} PRESET=${{ github.event.inputs.preset }} GENERATE_BINARIES=${{ github.event.inputs.generate_binaries }} + BUILDER_OS=${{ github.event.inputs.builder_os }} # Validate branch format (e.g., v25.06) if [[ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ]]; then @@ -130,6 +152,13 @@ jobs: echo "Error: 'generate_binaries' must be either 'true' or 'false'." exit 1 fi + + # Validate builder_os (must be a list) + python3 -c "import ast; x=ast.literal_eval('$BUILDER_OS'); assert isinstance(x, list)" 2>/dev/null + if [[ $? -ne 0 ]]; then + echo "Error: 'builder_os' must be a list." + return 1 + fi # Save all information in environment variables echo "SOFA_BRANCH_NAME=$BRANCH" >> $GITHUB_ENV @@ -137,6 +166,7 @@ jobs: echo "PRESET=$PRESET" >> $GITHUB_ENV echo "PYTHON_VERSION=$PYTHON" >> $GITHUB_ENV echo "GENERATE_BINARIES=$GENERATE_BINARIES" >> $GITHUB_ENV + echo "BUILDER_OS=$BUILDER_OS" >> $GITHUB_ENV - name: Set up python uses: actions/setup-python@v5 @@ -159,6 +189,7 @@ jobs: echo "PRESET=standard" >> $GITHUB_ENV echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV echo "GENERATE_BINARIES=true" >> $GITHUB_ENV + echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV - name: Check push on master case if: ${{ github.event_name == 'push'}} @@ -167,6 +198,7 @@ jobs: echo "COMMIT_SHA=${{ github.after }}>> $GITHUB_ENV echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV echo "FORCE_FULL_BUILD=true" >> $GITHUB_ENV + echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-ubuntu_clang_release","sh-ubuntu_clang_debug","sh-fedora_clang_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV - name: Run when PR is opened if: ${{ github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize') }} @@ -181,7 +213,6 @@ jobs: # Trigger the Build action python scripts/github_CI/checkPRInfoBeforeBuild.py - - name: Run when PR comment is edited if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request && ( github.event.action == 'created'|| github.event.action == 'edited' ) }} env: @@ -199,24 +230,42 @@ jobs: python scripts/github_CI/checkPRInfoBeforeBuild.py fi + - name: Export environment variables as outputs + id: export-vars + run: | + echo "SOFA_BRANCH_NAME=${SOFA_BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "SOFA_COMMIT_SHA=${SOFA_COMMIT_SHA}" >> $GITHUB_OUTPUT + echo "PRESET=${PRESET}" >> $GITHUB_OUTPUT + echo "PYTHON_VERSION=${PYTHON_VERSION}" >> $GITHUB_OUTPUT + echo "CI_DEPENDS_ON=${CI_DEPENDS_ON}" >> $GITHUB_OUTPUT + echo "WITH_ALL_TESTS=${WITH_ALL_TESTS}" >> $GITHUB_OUTPUT + echo "FORCE_FULL_BUILD=${FORCE_FULL_BUILD}" >> $GITHUB_OUTPUT + echo "OUT_OF_TREE_BUILD=${OUT_OF_TREE_BUILD}" >> $GITHUB_OUTPUT + echo "GENERATE_BINARIES=${GENERATE_BINARIES}" >> $GITHUB_OUTPUT + echo "PR_OWNER_URL=${PR_OWNER_URL}" >> $GITHUB_OUTPUT + echo "PR_BRANCH_NAME=${PR_BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "PR_COMMIT_SHA=${PR_COMMIT_SHA}" >> $GITHUB_OUTPUT + echo "BUILDER_OS=${BUILDER_OS}" >> $GITHUB_OUTPUT + + # =============================================================== # =============================================================== # Trigger the build with all parameters call-workflow-passing-data: + needs: filter_build uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master with: - sofa-branch-name: ${{ SOFA_BRANCH_NAME }} - sofa-commit-sha: ${{ SOFA_COMMIT_SHA }} - preset: ${{ PRESET }} - python-version: ${{ PYTHON_VERSION }} - ci-depends-on: ${{ CI_DEPENDS_ON }} - with-all-tests: ${{ WITH_ALL_TESTS }} - force-full-build: ${{ FORCE_FULL_BUILD }} - out-of-tree-build: ${{ OUT_OF_TREE_BUILD }} - generate-binaries: ${{ GENERATE_BINARIES }} - pr-owner-url: ${{ PR_OWNER_URL }} - pr-branch-name: ${{ PR_BRANCH_NAME }} - pr-commit-sha: ${{ PR_COMMIT_SHA }} - secrets: - personal_access_token: ${{ secrets.GITHUB_TOKEN }} + sofa-branch-name: ${{ needs.set-vars.outputs.SOFA_BRANCH_NAME }} + sofa-commit-sha: ${{ needs.set-vars.outputs.SOFA_COMMIT_SHA }} + preset: ${{ needs.set-vars.outputs.PRESET }} + python-version: ${{ needs.set-vars.outputs.PYTHON_VERSION }} + ci-depends-on: ${{ needs.set-vars.outputs.CI_DEPENDS_ON }} + with-all-tests: ${{ needs.set-vars.outputs.WITH_ALL_TESTS == 'true'}} + force-full-build: ${{ needs.set-vars.outputs.FORCE_FULL_BUILD == 'true'}} + out-of-tree-build: ${{ needs.set-vars.outputs.OUT_OF_TREE_BUILD }} + generate-binaries: ${{ needs.set-vars.outputs.GENERATE_BINARIES == 'true'}} + pr-owner-url: ${{ needs.set-vars.outputs.PR_OWNER_URL }} + pr-branch-name: ${{ needs.set-vars.outputs.PR_BRANCH_NAME }} + pr-commit-sha: ${{ needs.set-vars.outputs.PR_COMMIT_SHA }} + builder-os: ${{ needs.set-vars.outputs.BUILDER_OS }} \ No newline at end of file diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py index f1bc7d89ad0..70ecea4ff1d 100644 --- a/scripts/github_CI/checkPRInfoBeforeBuild.py +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -8,8 +8,16 @@ PR_COMMIT_SHA = os.getenv('PR_COMMIT_SHA') -if not GITHUB_TOKEN or not PR_NUMBER or not REPO_NAME: +if (not GITHUB_TOKEN) or (not PR_NUMBER) or (not OWNER_NAME) or (not PR_COMMIT_SHA): print("Error: Missing required environment variables.") + if (not GITHUB_TOKEN): + print(" - Missing GITHUB_TOKEN") + if (not PR_NUMBER): + print(" - Missing PR_NUMBER") + if (not OWNER_NAME): + print(" - Missing OWNER_NAME") + if (not PR_COMMIT_SHA): + print(" - Missing PR_COMMIT_SHA") exit(1) @@ -43,11 +51,14 @@ def check_labels(): exit(1) labels = [label['name'].lower() for label in response.json()] - print(f"Labels found: {labels}") + print(f"Labels found: {labels}.") if "pr: status to review" in labels: to_review_label_found = True print("PR is marked as 'to review'.") + else: + print(f"Flag to review has not been found. CI will stop.") + exit(1) # ======================================================================== @@ -83,7 +94,6 @@ def check_comments(): exit(1) comments = [comment['body'].lower() for comment in response.json()] - print(f"Comments found: {comments}") if any("[with-all-tests]" in comment for comment in comments): with_all_tests_found = True @@ -106,7 +116,7 @@ def export_pr_info(): pr_data = response.json() - pr_url = str(pr_data['user']['html_url']) + str(pr_data['repo']['name']) + pr_url = str(pr_data['user']['html_url']) + "/" + str(pr_data['base']['repo']['name']) pr_branch_name = pr_data['head']['ref'] pr_commit_sha = pr_data['head']['sha'] @@ -177,6 +187,10 @@ def extract_ci_depends_on(): "repo_url": repo_url, "branch_name": branch_name } + + match = re.search(r'\[with-all-tests\]', line) + if match: + with_all_tests_found = True # ======================================================================== @@ -196,15 +210,15 @@ def extract_ci_depends_on(): # Check compilation options in PR comments check_comments() - with open(os.environ["GITHUB_ENV"], "a") as env_file: - env_file.write(f"WITH_ALL_TESTS={with_all_tests_found}\n") - env_file.write(f"FORCE_FULL_BUILD={force_full_build_found}\n") - # Extract dependency repositories extract_ci_depends_on() + # Export all environment variables specific to pull-requests with open(os.environ["GITHUB_ENV"], "a") as env_file: + env_file.write(f"WITH_ALL_TESTS={with_all_tests_found}\n") + env_file.write(f"FORCE_FULL_BUILD={force_full_build_found}\n") env_file.write(f"CI_DEPENDS_ON={dependency_dict}\n") + env_file.write(f'BUILDER_OS=["sh-ubuntu_gcc_release","sh-fedora_clang_release","sh-windows_vs2022_release","sh-macos_clang_release"]') # ======================================================================== From 949946a9f2bf9a14eefa93ab0b4724196d910a82 Mon Sep 17 00:00:00 2001 From: Hugo Date: Mon, 7 Apr 2025 17:32:10 +0200 Subject: [PATCH 23/61] Update checkPRInfoBeforeBuild.py (#11) * Update checkPRInfoBeforeBuild.py * Apply suggestions from code review Co-authored-by: Paul Baksic <30337881+bakpaul@users.noreply.github.com> * Apply suggestions from code review * Apply suggestions from code review * Test * Use this script * fix * test fromJson * add debug * Final fix * revert change * Refacto display action * Fix filter --------- Co-authored-by: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Co-authored-by: Paul Baksic --- .github/workflows/build-and-test.yml | 21 +++++++++++++++++++++ .github/workflows/filter-build.yml | 27 ++++++++++++++------------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 6ffce0b6021..6a3a276a345 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -52,6 +52,27 @@ on: description: 'Json scructure with three parameters {"COMMIT_HASH":"fullhsashofthecommit", }' jobs: + display-inputs: + runs-on: ubuntu-latest + steps: + - name: Display + shell: bash + run: | + echo "Build and test launched with following parameters:" + echo "sofa-branch-name : ${{ inputs.sofa-branch-name }}" + echo "sofa-commit-sha : ${{ inputs.sofa-commit-sha }}" + echo "pr-owner-url : ${{ inputs.pr-owner-url }}" + echo "pr-branch-name : ${{ inputs.pr-branch-name }}" + echo "pr-commit-sha : ${{ inputs.pr-commit-sha }}" + echo "preset : ${{ inputs.preset }}" + echo "python-version : ${{ inputs.python-version }}" + echo "ci-depends-on : ${{ inputs.ci-depends-on }}" + echo "with-all-tests : ${{ inputs.with-all-tests }}" + echo "force-full-build : ${{ inputs.force-full-build }}" + echo "generate-binaries : ${{ inputs.generate-binaries }}" + echo "out-of-tree-build : ${{ inputs.out-of-tree-build }}" + echo "builder-os : ${{ inputs.builder-os }}" + build: strategy: matrix: diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index e1e8235ccbe..2e5c1f9a286 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -248,6 +248,7 @@ jobs: echo "BUILDER_OS=${BUILDER_OS}" >> $GITHUB_OUTPUT + # =============================================================== # =============================================================== @@ -256,16 +257,16 @@ jobs: needs: filter_build uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master with: - sofa-branch-name: ${{ needs.set-vars.outputs.SOFA_BRANCH_NAME }} - sofa-commit-sha: ${{ needs.set-vars.outputs.SOFA_COMMIT_SHA }} - preset: ${{ needs.set-vars.outputs.PRESET }} - python-version: ${{ needs.set-vars.outputs.PYTHON_VERSION }} - ci-depends-on: ${{ needs.set-vars.outputs.CI_DEPENDS_ON }} - with-all-tests: ${{ needs.set-vars.outputs.WITH_ALL_TESTS == 'true'}} - force-full-build: ${{ needs.set-vars.outputs.FORCE_FULL_BUILD == 'true'}} - out-of-tree-build: ${{ needs.set-vars.outputs.OUT_OF_TREE_BUILD }} - generate-binaries: ${{ needs.set-vars.outputs.GENERATE_BINARIES == 'true'}} - pr-owner-url: ${{ needs.set-vars.outputs.PR_OWNER_URL }} - pr-branch-name: ${{ needs.set-vars.outputs.PR_BRANCH_NAME }} - pr-commit-sha: ${{ needs.set-vars.outputs.PR_COMMIT_SHA }} - builder-os: ${{ needs.set-vars.outputs.BUILDER_OS }} \ No newline at end of file + sofa-branch-name: ${{ needs.filter_build.outputs.SOFA_BRANCH_NAME }} + sofa-commit-sha: ${{ needs.filter_build.outputs.SOFA_COMMIT_SHA }} + preset: ${{ needs.filter_build.outputs.PRESET }} + python-version: ${{ needs.filter_build.outputs.PYTHON_VERSION }} + ci-depends-on: ${{ needs.filter_build.outputs.CI_DEPENDS_ON }} + with-all-tests: ${{ needs.filter_build.outputs.WITH_ALL_TESTS == 'true'}} + force-full-build: ${{ needs.filter_build.outputs.FORCE_FULL_BUILD == 'true'}} + out-of-tree-build: ${{ needs.filter_build.outputs.OUT_OF_TREE_BUILD }} + generate-binaries: ${{ needs.filter_build.outputs.GENERATE_BINARIES == 'true'}} + pr-owner-url: ${{ needs.filter_build.outputs.PR_OWNER_URL }} + pr-branch-name: ${{ needs.filter_build.outputs.PR_BRANCH_NAME }} + pr-commit-sha: ${{ needs.filter_build.outputs.PR_COMMIT_SHA }} + builder-os: ${{ needs.filter_build.outputs.BUILDER_OS }} \ No newline at end of file From 0ce7781c379ec6d620f31542ff0279bd188aed7e Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:43:23 +0200 Subject: [PATCH 24/61] Fix build script (#13) * TO BE REVERTED BEFORE MERGE * Fix syntax * Fix git handle * fix action location * fix typo * Fix call to load vm env --- .github/workflows/build-and-test.yml | 20 +++++++++++--------- .github/workflows/filter-build.yml | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 6a3a276a345..16aea681051 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -108,17 +108,20 @@ jobs: - name: Clone SOFA and CI shell: bash run: | + cd $WORKSPACE + ## Clone sofa and merge origin master echo "Cloning SOFA at commit $GITHUB_WORKFLOW_SHA" if [ -d $WORKSPACE/sofa ]; then rm -rf $WORKSPACE/sofa - if + fi SRC_DIR=$(pwd)/sofa echo "SRC_DIR=$SRC_DIR" >> $GITHUB_ENV if [ -z "${{ inputs.pr-owner-url }}" ]; then echo "This is a PR, merging branch ${{ inputs.pr-branch-name }} from remote ${{ inputs.pr-owner-url }} into origin branch ${{ inputs.sofa-branch-name }}" git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/sofa-framework/sofa + cd sofa git remote add pr ${{ inputs.pr-owner-url }} git fetch pr @@ -130,12 +133,14 @@ jobs: else echo "This is not a PR: checking out sha ${{ inputs.sofa-commit-sha }} from branch ${{ inputs.sofa-branch-name }}" git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/sofa-framework/sofa + cd sofa git checkout ${{ inputs.sofa-commit-sha }} fi + cd $WORKSPACE ## Clone CI and use ci-depends-on structure - ci_branch="inputs.sofa-branch-name" + ci_branch=${{ inputs.sofa-branch-name }} ci_repo_url="https://www.github.com/sofa-framework/ci" # check if ci has a ci-depends-on @@ -155,7 +160,6 @@ jobs: CI_DIR=$WORKSPACE/ci echo "CI_DIR=$CI_DIR" >> $GITHUB_ENV - git init git clone -b ${ci_branch} --single-branch ${ci_repo_url} cd $WORKSPACE @@ -188,15 +192,13 @@ jobs: #TODO #HEre we use docker if possible - + # Setup variables for following calls . ${CI_DIR}/scripts/utils.sh + NODE_NAME="$(hostname)" # Needed by load-vm-env --> TODO: be able to get hostname multiplatform + SCRIPT_DIR=${CI_DIR}/scripts/ # Needed by load-vm-env, might need to run this inside the docker env + CI_PYTHON3_VERSION=${{ inputs.python-version }} # Needed by load-vm-env, might need to run this inside the docker env ## Setup env variables - CI_PYTHON3_VERSION=${{ inputs.python-version }} # Needed by load-vm-env - # TODO: be able to get hostname to load correct env file --> set - host_name="sofa-ci-dev-ubuntu2204-0" - NODE_NAME="$host_name" # Needed by load-vm-env - # then run load-vm-env ## Deal with ci-depends-on diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 2e5c1f9a286..4a96fae36af 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -255,7 +255,7 @@ jobs: # Trigger the build with all parameters call-workflow-passing-data: needs: filter_build - uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master + uses: bakpaul/sofa/.github/workflows/build-and-test.yml@j2g_fix_build_action with: sofa-branch-name: ${{ needs.filter_build.outputs.SOFA_BRANCH_NAME }} sofa-commit-sha: ${{ needs.filter_build.outputs.SOFA_COMMIT_SHA }} From 93b612b3d0c4989df7095034ef74549688ac2170 Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Tue, 8 Apr 2025 16:22:18 +0200 Subject: [PATCH 25/61] fix script + revert wrong changes (#15) --- .github/workflows/build-and-test.yml | 7 ++++++- .github/workflows/filter-build.yml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 16aea681051..e461acacdf1 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -218,6 +218,11 @@ jobs: CMAKE_OPTIONS="$CMAKE_OPTIONS $flag_repository $flag_tag " done + if [[ ! -n "$CMAKE_OPTIONS" ]]; then + CMAKE_OPTIONS="no-ci-depends-on" + fi + + ## Configure the build: setup cmake variables # retrive build type and compiler BUILD_TYPE=$(echo ${{ matrix.os }} | awk -F '_' '{print $3}') @@ -228,7 +233,7 @@ jobs: BUILD_OPTIONS="$BUILD_OPTIONS build-release-package" fi - . ${CI_DIR}/scripts/configure.sh "$BUILD_DIR" "$SRC_DIR" "$CONFIG" "$BUILD_TYPE" "$BUILD_OPTIONS" "$CMAKE_OPTIONS" + . ${CI_DIR}/scripts/configure.sh "$BUILD_DIR" "$SRC_DIR" "$CONFIG" "$CMAKE_OPTIONS" "$BUILD_TYPE" "$BUILD_OPTIONS" ## Call to build $SCRIPT_DIR/compile.sh "$BUILD_DIR" "$CONFIG" "$BUILD_OPTIONS" diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 4a96fae36af..2e5c1f9a286 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -255,7 +255,7 @@ jobs: # Trigger the build with all parameters call-workflow-passing-data: needs: filter_build - uses: bakpaul/sofa/.github/workflows/build-and-test.yml@j2g_fix_build_action + uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master with: sofa-branch-name: ${{ needs.filter_build.outputs.SOFA_BRANCH_NAME }} sofa-commit-sha: ${{ needs.filter_build.outputs.SOFA_COMMIT_SHA }} From 3fab0e2ec47a916b1636f15e1098694dcc392923 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 8 Apr 2025 17:32:11 +0200 Subject: [PATCH 26/61] Update filter-build.yml (#16) --- .github/workflows/filter-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 2e5c1f9a286..e976f433060 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -195,7 +195,7 @@ jobs: if: ${{ github.event_name == 'push'}} run: | echo "This step runs only for push on the master branch." - echo "COMMIT_SHA=${{ github.after }}>> $GITHUB_ENV + echo "SOFA_COMMIT_SHA=${{ github.after }}">> $GITHUB_ENV echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV echo "FORCE_FULL_BUILD=true" >> $GITHUB_ENV echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-ubuntu_clang_release","sh-ubuntu_clang_debug","sh-fedora_clang_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV @@ -269,4 +269,4 @@ jobs: pr-owner-url: ${{ needs.filter_build.outputs.PR_OWNER_URL }} pr-branch-name: ${{ needs.filter_build.outputs.PR_BRANCH_NAME }} pr-commit-sha: ${{ needs.filter_build.outputs.PR_COMMIT_SHA }} - builder-os: ${{ needs.filter_build.outputs.BUILDER_OS }} \ No newline at end of file + builder-os: ${{ needs.filter_build.outputs.BUILDER_OS }} From 43dcd307aa80c681533c28d2bf5d904e405cbe2b Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 8 Apr 2025 17:56:47 +0200 Subject: [PATCH 27/61] Fix sha when commit pushed in master (#17) --- .github/workflows/build-and-test.yml | 2 +- .github/workflows/filter-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e461acacdf1..2d79d7a3a73 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -118,7 +118,7 @@ jobs: SRC_DIR=$(pwd)/sofa echo "SRC_DIR=$SRC_DIR" >> $GITHUB_ENV - if [ -z "${{ inputs.pr-owner-url }}" ]; then + if [ ! -z "${{ inputs.pr-owner-url }}" ]; then echo "This is a PR, merging branch ${{ inputs.pr-branch-name }} from remote ${{ inputs.pr-owner-url }} into origin branch ${{ inputs.sofa-branch-name }}" git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/sofa-framework/sofa cd sofa diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index e976f433060..5cdfaa5fb34 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -195,7 +195,7 @@ jobs: if: ${{ github.event_name == 'push'}} run: | echo "This step runs only for push on the master branch." - echo "SOFA_COMMIT_SHA=${{ github.after }}">> $GITHUB_ENV + echo "SOFA_COMMIT_SHA=${{ github.sha }}">> $GITHUB_ENV echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV echo "FORCE_FULL_BUILD=true" >> $GITHUB_ENV echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-ubuntu_clang_release","sh-ubuntu_clang_debug","sh-fedora_clang_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV From 9fcf6456eb03c46fab1ed490c0eb4c194eee0f50 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 8 Apr 2025 18:15:23 +0200 Subject: [PATCH 28/61] Temporarily change remote for non-PR (#18) --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2d79d7a3a73..f92564de8b2 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -132,7 +132,7 @@ jobs: fi else echo "This is not a PR: checking out sha ${{ inputs.sofa-commit-sha }} from branch ${{ inputs.sofa-branch-name }}" - git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/sofa-framework/sofa + git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/bakpaul/sofa ## TODO : REMOVE THIS WITH SOFA_FRAMEWORK @@@@@@@@@@@@@ cd sofa git checkout ${{ inputs.sofa-commit-sha }} fi @@ -264,4 +264,4 @@ jobs: shell: bash run: | #TODO - #Use new action with workflow call \ No newline at end of file + #Use new action with workflow call From 867f9107dc68619d5c4e4631a46f2d26e85c38a6 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 11 Apr 2025 20:28:42 +0200 Subject: [PATCH 29/61] 202505 evolve filter nightly (#19) * Isolate nightly build to allow for matrix only in nightly cases * Check that SOFA branch name is either master or vXX.XX * Attempt to detect existing running builds on PR * Fix check on ci-depends-on * fix indent and add generic condition for nightly build * Another fix for ci-depends-on * Another fix for ci-depends-on in json * Display info on PR comment * Display info on PR comment * Display info on PR comment --- .github/workflows/filter-build.yml | 166 +++++++++++++++++--- scripts/github_CI/checkPRInfoBeforeBuild.py | 6 +- 2 files changed, 147 insertions(+), 25 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 5cdfaa5fb34..2e45a4a963b 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -62,10 +62,6 @@ on: jobs: filter_build: runs-on: ubuntu-latest - strategy: - matrix: - sofa_version: [master, v24.12] - ### if: ${{ github.repository_owner == 'sofa-framework' }} outputs: SOFA_BRANCH_NAME: ${{ steps.export-vars.outputs.SOFA_BRANCH_NAME }} SOFA_COMMIT_SHA: ${{ steps.export-vars.outputs.SOFA_COMMIT_SHA }} @@ -111,8 +107,9 @@ jobs: BUILDER_OS=${{ github.event.inputs.builder_os }} # Validate branch format (e.g., v25.06) - if [[ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ]]; then - echo "Error: Invalid branch name format: $BRANCH" + if [ [ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ] && [ "$BRANCH" != "master" ] ]; then + echo "Error: Invalid branch name format: $BRANCH." + echo "Branch name should be either master or any release branch (e.g., v25.06)" exit 1 fi echo "Branch name $BRANCH is valid." @@ -181,17 +178,7 @@ jobs: - name: Check out code uses: actions/checkout@v2 - - name: Run when nightly - if: github.event.schedule == '0 2 * * *' - run: | - echo "This step runs only for nightly builds." - echo "SOFA_BRANCH_NAME=${{ matrix.sofa_version }}" >> $GITHUB_ENV - echo "PRESET=standard" >> $GITHUB_ENV - echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV - echo "GENERATE_BINARIES=true" >> $GITHUB_ENV - echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV - - - name: Check push on master case + - name: Check push on master case (e.g. merge) if: ${{ github.event_name == 'push'}} run: | echo "This step runs only for push on the master branch." @@ -200,8 +187,46 @@ jobs: echo "FORCE_FULL_BUILD=true" >> $GITHUB_ENV echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-ubuntu_clang_release","sh-ubuntu_clang_debug","sh-fedora_clang_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV - - name: Run when PR is opened - if: ${{ github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize') }} + - name: Check in a PR when a commit is pushed or when a comment is created or edited + if: ${{ (github.event_name == 'pull_request' && github.event.action == 'synchronize') || (github.event_name == 'issue_comment' && github.event.issue.pull_request ) }} + uses: actions/github-script@v7 + with: + script: | + const currentRunId = context.runId; + const workflowName = context.workflow; + const prNumber = ${{ github.event.pull_request.number }}; + + const runs = await github.rest.actions.listWorkflowRunsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100, + status: 'in_progress' + }); + + const matchingRuns = runs.data.workflow_runs.filter(run => { + return run.name === workflowName && + run.id !== currentRunId && + run.head_branch === "${{ github.event.pull_request.head.ref }}"; + }); + + for (const run of matchingRuns) { + console.log(`Cancelling previous run: ${run.id}`); + await github.rest.actions.cancelWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id + }); + } + + if (matchingRuns.length > 0) { + console.log("Sleeping for 5 seconds to allow previous runs to cancel..."); + await new Promise(resolve => setTimeout(resolve, 5000)); + } else { + console.log("No previous runs to cancel. Continuing..."); + } + + - name: Run when PR is opened or a commit is pushed in a PR + if: ${{ github.event_name == 'pull_request' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.pull_request.number }} @@ -212,9 +237,9 @@ jobs: # Trigger the Build action python scripts/github_CI/checkPRInfoBeforeBuild.py - - - name: Run when PR comment is edited - if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request && ( github.event.action == 'created'|| github.event.action == 'edited' ) }} + + - name: Run when PR comment is created or edited + if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.issue.pull_request.number }} @@ -230,6 +255,102 @@ jobs: python scripts/github_CI/checkPRInfoBeforeBuild.py fi + echo "PR_NUMBER=${PR_NUMBER}" + echo "OWNER_NAME=${OWNER_NAME}" + echo "PR_COMMIT_SHA=${PR_COMMIT_SHA}" + echo "github.event.issue.number=${github.event.issue.number}" + echo "github.event.issue.pull_request=${github.event.issue.pull_request}" + echo "github.event.issue=${github.event.issue}" + + + - name: Export environment variables as outputs + id: export-vars + run: | + # Validate branch format (e.g., v25.06) + if [ [ ! "$SOFA_BRANCH_NAME" =~ ^v[0-9]{2}\.[0-9]{2}$ ] && [ "$SOFA_BRANCH_NAME" != "master" ] ]; then + echo "Error: Invalid branch name format: $SOFA_BRANCH_NAME." + echo "Branch name should be either master or any release branch (e.g., v25.06)" + exit 1 + fi + + echo "SOFA_BRANCH_NAME=${SOFA_BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "SOFA_COMMIT_SHA=${SOFA_COMMIT_SHA}" >> $GITHUB_OUTPUT + echo "PRESET=${PRESET}" >> $GITHUB_OUTPUT + echo "PYTHON_VERSION=${PYTHON_VERSION}" >> $GITHUB_OUTPUT + echo "CI_DEPENDS_ON=${CI_DEPENDS_ON}" >> $GITHUB_OUTPUT + echo "WITH_ALL_TESTS=${WITH_ALL_TESTS}" >> $GITHUB_OUTPUT + echo "FORCE_FULL_BUILD=${FORCE_FULL_BUILD}" >> $GITHUB_OUTPUT + echo "OUT_OF_TREE_BUILD=${OUT_OF_TREE_BUILD}" >> $GITHUB_OUTPUT + echo "GENERATE_BINARIES=${GENERATE_BINARIES}" >> $GITHUB_OUTPUT + echo "PR_OWNER_URL=${PR_OWNER_URL}" >> $GITHUB_OUTPUT + echo "PR_BRANCH_NAME=${PR_BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "PR_COMMIT_SHA=${PR_COMMIT_SHA}" >> $GITHUB_OUTPUT + echo "BUILDER_OS=${BUILDER_OS}" >> $GITHUB_OUTPUT + + + + + nightly_build: + runs-on: ubuntu-latest + if: github.event.schedule == '0 2 * * *' + strategy: + matrix: + sofa_version: [master, v24.12] + ### if: ${{ github.repository_owner == 'sofa-framework' }} + outputs: + SOFA_BRANCH_NAME: ${{ steps.export-vars.outputs.SOFA_BRANCH_NAME }} + SOFA_COMMIT_SHA: ${{ steps.export-vars.outputs.SOFA_COMMIT_SHA }} + PRESET: ${{ steps.export-vars.outputs.PRESET }} + PYTHON_VERSION: ${{ steps.export-vars.outputs.PYTHON_VERSION }} + CI_DEPENDS_ON: ${{ steps.export-vars.outputs.CI_DEPENDS_ON }} + WITH_ALL_TESTS: ${{ steps.export-vars.outputs.WITH_ALL_TESTS }} + FORCE_FULL_BUILD: ${{ steps.export-vars.outputs.FORCE_FULL_BUILD }} + OUT_OF_TREE_BUILD: ${{ steps.export-vars.outputs.OUT_OF_TREE_BUILD }} + GENERATE_BINARIES: ${{ steps.export-vars.outputs.GENERATE_BINARIES }} + PR_OWNER_URL: ${{ steps.export-vars.outputs.PR_OWNER_URL }} + PR_BRANCH_NAME: ${{ steps.export-vars.outputs.PR_BRANCH_NAME }} + PR_COMMIT_SHA: ${{ steps.export-vars.outputs.PR_COMMIT_SHA }} + BUILDER_OS: ${{ steps.export-vars.outputs.BUILDER_OS }} + + steps: + - name: Default values of environment variables + run: | + echo "SOFA_BRANCH_NAME=master" >> $GITHUB_ENV # SOFA_BRANCH_NAME: "master" + echo "SOFA_COMMIT_SHA=HEAD" >> $GITHUB_ENV # SOFA_COMMIT_SHA: "HEAD" + echo "PRESET=full" >> $GITHUB_ENV # PRESET: "full" + echo "PYTHON_VERSION=3.12" >> $GITHUB_ENV # PYTHON_VERSION: "3.12" + echo "CI_DEPENDS_ON=" >> $GITHUB_ENV # CI_DEPENDS_ON: "" + echo "WITH_ALL_TESTS=false" >> $GITHUB_ENV # WITH_ALL_TESTS: false + echo "FORCE_FULL_BUILD=false" >> $GITHUB_ENV # FORCE_FULL_BUILD: false + echo "OUT_OF_TREE_BUILD=" >> $GITHUB_ENV # OUT_OF_TREE_BUILD: "" + echo "GENERATE_BINARIES=false" >> $GITHUB_ENV # GENERATE_BINARIES: false + echo "PR_OWNER_URL=" >> $GITHUB_ENV # PR_OWNER_URL: "" + echo "PR_BRANCH_NAME=" >> $GITHUB_ENV # PR_BRANCH_NAME: "" + echo "PR_COMMIT_SHA=HEAD" >> $GITHUB_ENV # PR_COMMIT_SHA: "HEAD" + echo 'BUILDER_OS=["sh-ubuntu_gcc_release"]' >> $GITHUB_ENV # BUILDER_OS: ["sh-ubuntu_gcc_release"] + + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install pip packages + run: | + pip install python-graphql-client + pip install requests + + - name: Check out code + uses: actions/checkout@v2 + + - name: Run when nightly + run: | + echo "This step runs only for nightly builds." + echo "SOFA_BRANCH_NAME=${{ matrix.sofa_version }}" >> $GITHUB_ENV + echo "PRESET=standard" >> $GITHUB_ENV + echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV + echo "GENERATE_BINARIES=true" >> $GITHUB_ENV + echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV + - name: Export environment variables as outputs id: export-vars run: | @@ -249,6 +370,7 @@ jobs: + # =============================================================== # =============================================================== diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py index 70ecea4ff1d..9bd85756b57 100644 --- a/scripts/github_CI/checkPRInfoBeforeBuild.py +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -161,14 +161,14 @@ def extract_ci_depends_on(): # Ensure the URL is in the expected dependency format, e.g. https://github.com/sofa-framework/Sofa.Qt/pull/6 parts = dependency.split('/') - if len(parts) != 6 or parts[0] != 'https:' or parts[1] != '' or parts[2] != 'github.com': + if len(parts) != 7 or parts[0] != 'https:' or parts[1] != '' or parts[2] != 'github.com': raise ValueError("") print(f"Invalid URL ci-depends-on format: {dependency}") exit(1) owner = parts[3] repo = parts[4] - pull_number = parts[5] + pull_number = parts[6] dependency_request_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pull_number}" response = requests.get(dependency_request_url, headers=HEADERS) @@ -179,7 +179,7 @@ def extract_ci_depends_on(): dependency_pr_data = response.json() - key = dependency_pr_data['repo']['name'] #Sofa.Qt + key = dependency_pr_data['head']['repo']['name'] #Sofa.Qt repo_url = dependency_pr_data['html_url'] #https://github.com/sofa-framework/Sofa.Qt branch_name = dependency_pr_data['head']['ref'] #my_feature_branch From 4c5ce49dc681d570eba24ca47caf76b5a42c5215 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 11 Apr 2025 20:45:45 +0200 Subject: [PATCH 30/61] Fix trigger through comments (#20) * Fix trigger through comments * remove check when action is already triggered * remove test comment --- .github/workflows/filter-build.yml | 39 ------------------------------ 1 file changed, 39 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 2e45a4a963b..6404cc4fcb1 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -187,44 +187,6 @@ jobs: echo "FORCE_FULL_BUILD=true" >> $GITHUB_ENV echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-ubuntu_clang_release","sh-ubuntu_clang_debug","sh-fedora_clang_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV - - name: Check in a PR when a commit is pushed or when a comment is created or edited - if: ${{ (github.event_name == 'pull_request' && github.event.action == 'synchronize') || (github.event_name == 'issue_comment' && github.event.issue.pull_request ) }} - uses: actions/github-script@v7 - with: - script: | - const currentRunId = context.runId; - const workflowName = context.workflow; - const prNumber = ${{ github.event.pull_request.number }}; - - const runs = await github.rest.actions.listWorkflowRunsForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - per_page: 100, - status: 'in_progress' - }); - - const matchingRuns = runs.data.workflow_runs.filter(run => { - return run.name === workflowName && - run.id !== currentRunId && - run.head_branch === "${{ github.event.pull_request.head.ref }}"; - }); - - for (const run of matchingRuns) { - console.log(`Cancelling previous run: ${run.id}`); - await github.rest.actions.cancelWorkflowRun({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: run.id - }); - } - - if (matchingRuns.length > 0) { - console.log("Sleeping for 5 seconds to allow previous runs to cancel..."); - await new Promise(resolve => setTimeout(resolve, 5000)); - } else { - console.log("No previous runs to cancel. Continuing..."); - } - - name: Run when PR is opened or a commit is pushed in a PR if: ${{ github.event_name == 'pull_request' }} env: @@ -262,7 +224,6 @@ jobs: echo "github.event.issue.pull_request=${github.event.issue.pull_request}" echo "github.event.issue=${github.event.issue}" - - name: Export environment variables as outputs id: export-vars run: | From bb10f396e4c9fad1eb92589b23d4f80165331e07 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Fri, 11 Apr 2025 22:01:13 +0200 Subject: [PATCH 31/61] move echo to read it --- .github/workflows/filter-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 6404cc4fcb1..29af76135da 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -212,17 +212,17 @@ jobs: COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) echo "Comment: $COMMENT_BODY" - # Trigger the Build action if [ci-build] is in the comment - if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then - python scripts/github_CI/checkPRInfoBeforeBuild.py - fi - echo "PR_NUMBER=${PR_NUMBER}" echo "OWNER_NAME=${OWNER_NAME}" echo "PR_COMMIT_SHA=${PR_COMMIT_SHA}" echo "github.event.issue.number=${github.event.issue.number}" echo "github.event.issue.pull_request=${github.event.issue.pull_request}" echo "github.event.issue=${github.event.issue}" + + # Trigger the Build action if [ci-build] is in the comment + if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then + python scripts/github_CI/checkPRInfoBeforeBuild.py + fi - name: Export environment variables as outputs id: export-vars From a32a3e599eed9624486b7734332e80bb7f183098 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Fri, 11 Apr 2025 22:10:27 +0200 Subject: [PATCH 32/61] attempt to access information PR comment --- .github/workflows/filter-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 29af76135da..896dac0668b 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -212,13 +212,13 @@ jobs: COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) echo "Comment: $COMMENT_BODY" - echo "PR_NUMBER=${PR_NUMBER}" - echo "OWNER_NAME=${OWNER_NAME}" - echo "PR_COMMIT_SHA=${PR_COMMIT_SHA}" + echo "PR_NUMBER=$PR_NUMBER" + echo "OWNER_NAME=$OWNER_NAME" + echo "PR_COMMIT_SHA=$PR_COMMIT_SHA" + echo "github.event.issue=${github.event.issue}" echo "github.event.issue.number=${github.event.issue.number}" echo "github.event.issue.pull_request=${github.event.issue.pull_request}" - echo "github.event.issue=${github.event.issue}" - + # Trigger the Build action if [ci-build] is in the comment if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then python scripts/github_CI/checkPRInfoBeforeBuild.py From 070f7ae02cd3997e245d82262e690dfcbea8443b Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Fri, 11 Apr 2025 22:14:49 +0200 Subject: [PATCH 33/61] attempt to access information PR comment --- .github/workflows/filter-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 896dac0668b..f105ab58210 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -215,9 +215,9 @@ jobs: echo "PR_NUMBER=$PR_NUMBER" echo "OWNER_NAME=$OWNER_NAME" echo "PR_COMMIT_SHA=$PR_COMMIT_SHA" - echo "github.event.issue=${github.event.issue}" - echo "github.event.issue.number=${github.event.issue.number}" - echo "github.event.issue.pull_request=${github.event.issue.pull_request}" + echo "github.event.issue=${{ github.event.issue }}" + echo "github.event.issue.number=${{ github.event.issue.number }}" + echo "github.event.issue.pull_request=${{ github.event.issue.pull_request }}" # Trigger the Build action if [ci-build] is in the comment if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then From 7dba7201799f56219dc924eb1ed8cd95d95f4720 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Fri, 11 Apr 2025 22:22:34 +0200 Subject: [PATCH 34/61] complete access information PR comment --- .github/workflows/filter-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index f105ab58210..400c75b1101 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -218,6 +218,9 @@ jobs: echo "github.event.issue=${{ github.event.issue }}" echo "github.event.issue.number=${{ github.event.issue.number }}" echo "github.event.issue.pull_request=${{ github.event.issue.pull_request }}" + echo "github.event.issue.pull_request.number=${{ github.event.issue.pull_request.number }}" + echo "github.event.issue.pull_request.head.repo.owner.login=${{ github.event.issue.pull_request.head.repo.owner.login }}" + echo "github.event.issue.pull_request.head.sha=${{ github.event.issue.pull_request.head.sha }}" # Trigger the Build action if [ci-build] is in the comment if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then From 8229d8386b240c2c9c4113dcef3276bd56cefc6c Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Fri, 11 Apr 2025 23:43:58 +0200 Subject: [PATCH 35/61] complete access information PR comment --- .github/workflows/filter-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 400c75b1101..b8c79c40360 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -221,6 +221,9 @@ jobs: echo "github.event.issue.pull_request.number=${{ github.event.issue.pull_request.number }}" echo "github.event.issue.pull_request.head.repo.owner.login=${{ github.event.issue.pull_request.head.repo.owner.login }}" echo "github.event.issue.pull_request.head.sha=${{ github.event.issue.pull_request.head.sha }}" + echo "github.event.pull_request.number=${{ github.event.issue.pull_request.number }}" + echo "github.event.pull_request.head.repo.owner.login=${{ github.event.issue.pull_request.head.repo.owner.login }}" + echo "github.event.pull_request.head.sha=${{ github.event.issue.pull_request.head.sha }}" # Trigger the Build action if [ci-build] is in the comment if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then From 37656b441370c2b8aec433a7bc74ed0c4a536e99 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Sat, 12 Apr 2025 00:04:37 +0200 Subject: [PATCH 36/61] Update search of PR information when comment_issue --- .github/workflows/filter-build.yml | 40 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index b8c79c40360..1ec1dd17556 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -204,29 +204,37 @@ jobs: if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.issue.pull_request.number }} - OWNER_NAME: ${{ github.event.issue.pull_request.head.repo.owner.login }} - PR_COMMIT_SHA: ${{ github.event.issue.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.issue.number }} + OWNER_NAME: "" # Must be retrieved + PR_COMMIT_SHA: "" # Must be retrieved run: | echo "This step runs only when a PR comment is created or edited." COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) echo "Comment: $COMMENT_BODY" - - echo "PR_NUMBER=$PR_NUMBER" - echo "OWNER_NAME=$OWNER_NAME" - echo "PR_COMMIT_SHA=$PR_COMMIT_SHA" - echo "github.event.issue=${{ github.event.issue }}" - echo "github.event.issue.number=${{ github.event.issue.number }}" - echo "github.event.issue.pull_request=${{ github.event.issue.pull_request }}" - echo "github.event.issue.pull_request.number=${{ github.event.issue.pull_request.number }}" - echo "github.event.issue.pull_request.head.repo.owner.login=${{ github.event.issue.pull_request.head.repo.owner.login }}" - echo "github.event.issue.pull_request.head.sha=${{ github.event.issue.pull_request.head.sha }}" - echo "github.event.pull_request.number=${{ github.event.issue.pull_request.number }}" - echo "github.event.pull_request.head.repo.owner.login=${{ github.event.issue.pull_request.head.repo.owner.login }}" - echo "github.event.pull_request.head.sha=${{ github.event.issue.pull_request.head.sha }}" # Trigger the Build action if [ci-build] is in the comment if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then + echo "Fetching PR #$PR_NUMBER information" + + pr_data=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/bakpaul/sofa/pulls/$PR_NUMBER") + + # Output full PR data if debugging + echo "$pr_data" | jq . + + # Extract data + pr_owner=$(echo "$pr_data" | jq -r '.user.login') + pr_sha=$(echo "$pr_data" | jq -r '.head.sha') + + echo "PR number : $PR_NUMBER" + echo "PR owner name : $pr_owner" + echo "PR commit SHA : $pr_sha" + + # Save to environment variables for future steps + echo "OWNER_NAME=$pr_owner" >> $GITHUB_ENV + echo "PR_COMMIT_SHA=$pr_sha" >> $GITHUB_ENV + python scripts/github_CI/checkPRInfoBeforeBuild.py fi From 06b1e67d80196709321ea901c4238868789dea7b Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Sat, 12 Apr 2025 00:08:53 +0200 Subject: [PATCH 37/61] Remove empty env var --- .github/workflows/filter-build.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 1ec1dd17556..7c1ffcf7b8e 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -205,8 +205,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.issue.number }} - OWNER_NAME: "" # Must be retrieved - PR_COMMIT_SHA: "" # Must be retrieved run: | echo "This step runs only when a PR comment is created or edited." COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) @@ -220,9 +218,6 @@ jobs: -H "Accept: application/vnd.github+json" \ "https://api.github.com/repos/bakpaul/sofa/pulls/$PR_NUMBER") - # Output full PR data if debugging - echo "$pr_data" | jq . - # Extract data pr_owner=$(echo "$pr_data" | jq -r '.user.login') pr_sha=$(echo "$pr_data" | jq -r '.head.sha') @@ -234,7 +229,7 @@ jobs: # Save to environment variables for future steps echo "OWNER_NAME=$pr_owner" >> $GITHUB_ENV echo "PR_COMMIT_SHA=$pr_sha" >> $GITHUB_ENV - + python scripts/github_CI/checkPRInfoBeforeBuild.py fi From 9bae225c2c0ea7d965f716b9cbdc4993ca26378e Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Sat, 12 Apr 2025 00:19:54 +0200 Subject: [PATCH 38/61] add check on OWNER_NAME var --- .github/workflows/filter-build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 7c1ffcf7b8e..0920167f1c6 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -222,14 +222,16 @@ jobs: pr_owner=$(echo "$pr_data" | jq -r '.user.login') pr_sha=$(echo "$pr_data" | jq -r '.head.sha') - echo "PR number : $PR_NUMBER" - echo "PR owner name : $pr_owner" - echo "PR commit SHA : $pr_sha" + echo "- PR number : $PR_NUMBER" + echo "- PR owner name : $pr_owner" + echo "- PR commit SHA : $pr_sha" # Save to environment variables for future steps echo "OWNER_NAME=$pr_owner" >> $GITHUB_ENV echo "PR_COMMIT_SHA=$pr_sha" >> $GITHUB_ENV + echo "Env: OWNER_NAME = $OWNER_NAME" + python scripts/github_CI/checkPRInfoBeforeBuild.py fi From 41dcadad971587becb1c9a9fd14f23217090fe62 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Sat, 12 Apr 2025 00:21:15 +0200 Subject: [PATCH 39/61] add check on OWNER_NAME var --- .github/workflows/filter-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 0920167f1c6..e0f2e57b29e 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -231,6 +231,7 @@ jobs: echo "PR_COMMIT_SHA=$pr_sha" >> $GITHUB_ENV echo "Env: OWNER_NAME = $OWNER_NAME" + echo "Env: PR_COMMIT_SHA = $PR_COMMIT_SHA" python scripts/github_CI/checkPRInfoBeforeBuild.py fi From 5675c333b11738ea44c9b2be781be27c51807026 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Sat, 12 Apr 2025 00:24:26 +0200 Subject: [PATCH 40/61] last attempt to fill env var for comment_issue --- .github/workflows/filter-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index e0f2e57b29e..2a4cbffb318 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -219,16 +219,16 @@ jobs: "https://api.github.com/repos/bakpaul/sofa/pulls/$PR_NUMBER") # Extract data - pr_owner=$(echo "$pr_data" | jq -r '.user.login') - pr_sha=$(echo "$pr_data" | jq -r '.head.sha') + OWNER_NAME=$(echo "$pr_data" | jq -r '.user.login') + PR_COMMIT_SHA=$(echo "$pr_data" | jq -r '.head.sha') echo "- PR number : $PR_NUMBER" - echo "- PR owner name : $pr_owner" - echo "- PR commit SHA : $pr_sha" + echo "- PR owner name : $OWNER_NAME" + echo "- PR commit SHA : $PR_COMMIT_SHA" # Save to environment variables for future steps - echo "OWNER_NAME=$pr_owner" >> $GITHUB_ENV - echo "PR_COMMIT_SHA=$pr_sha" >> $GITHUB_ENV + echo "OWNER_NAME=$OWNER_NAME" >> $GITHUB_ENV + echo "PR_COMMIT_SHA=$PR_COMMIT_SHA" >> $GITHUB_ENV echo "Env: OWNER_NAME = $OWNER_NAME" echo "Env: PR_COMMIT_SHA = $PR_COMMIT_SHA" From 67db07eb192b7f054531c05f4614f7870bdb81c1 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Sat, 12 Apr 2025 00:30:59 +0200 Subject: [PATCH 41/61] split in two steps for comment_issue --- .github/workflows/filter-build.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 2a4cbffb318..04830fd5990 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -200,7 +200,7 @@ jobs: # Trigger the Build action python scripts/github_CI/checkPRInfoBeforeBuild.py - - name: Run when PR comment is created or edited + - name: Run when PR comment is created or edited (set up environment var for python script) if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -232,10 +232,21 @@ jobs: echo "Env: OWNER_NAME = $OWNER_NAME" echo "Env: PR_COMMIT_SHA = $PR_COMMIT_SHA" - - python scripts/github_CI/checkPRInfoBeforeBuild.py fi + - name: Run when PR comment is created or edited (trigger python script) + if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) + echo "Comment: $COMMENT_BODY" + + # Trigger the Build action if [ci-build] is in the comment + if [[ "$COMMENT_BODY" == *"[ci-build]"* ]]; then + python scripts/github_CI/checkPRInfoBeforeBuild.py + fi + - name: Export environment variables as outputs id: export-vars run: | From 384f86c2fc25c307ed57261488028c6d813ccced Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Sat, 12 Apr 2025 00:32:38 +0200 Subject: [PATCH 42/61] add missing PR number for comment_issue (step 2) --- .github/workflows/filter-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 04830fd5990..0e93f70d152 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -238,6 +238,7 @@ jobs: if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number }} run: | COMMENT_BODY=$(jq -r '.comment.body' < $GITHUB_EVENT_PATH) echo "Comment: $COMMENT_BODY" From 77016424a3fba49b52b654df67b2a9f41a9bc3d7 Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:32:35 +0200 Subject: [PATCH 43/61] Start work on docker use (#10) * Start work * Add docker run for ubuntu and fedora * TOREVERT * Add insights and login * Fix action * Try fix docker autentification * .github/workflows/ * Add read only token to action * fix docker user name * Change pipe to or * Fix docker call * try fix docker * fix ? * Fix ci-depends-on mechanism * Fix quote outputed by jq * Restore branch name * fix python script * Fix ci-depends-on remote url * Fix script not launching * Display arguments * Use github env variable for node name * Unify workspace folder using matrix os * Add display * Add display * Update .github/workflows/filter-build.yml Co-authored-by: Hugo --------- Co-authored-by: Hugo --- .github/workflows/build-and-test.yml | 90 +++++++++++++++------ scripts/github_CI/checkPRInfoBeforeBuild.py | 8 +- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f92564de8b2..aa8ebd667a3 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -83,11 +83,16 @@ jobs: shell: bash run: | ## Go to workspace given by the runner (this file is generated by the pre-build script) - WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA) + WORKSPACE=$(cat $GITHUB_WORKFLOW_SHA)/${{ matrix.os }} echo "WORKSPACE=$WORKSPACE" >> $GITHUB_ENV + if [ ! -d $WORKSPACE ]; then + mkdir -p $WORKSPACE + fi cd $WORKSPACE + echo "Workspace folder for this build will be $WORKSPACE" + ## Setup github # Git config (needed by CMake ExternalProject) if ! git config --get user.name; then @@ -146,6 +151,7 @@ jobs: # check if ci has a ci-depends-on ci_ci_depends_on=$(echo "${{ inputs.ci-depends-on }}" | jq .ci) if [ "$ci_ci_depends_on" != "null" ]; then + echo "ci-depends-on for ci repository detected." ci_repo_url=$(echo "${{ inputs.ci-depends-on }}" | jq .ci.repo_url) ci_branch=$(echo "${{ inputs.ci-depends-on }}" | jq .ci.branch_name) fi @@ -160,7 +166,7 @@ jobs: CI_DIR=$WORKSPACE/ci echo "CI_DIR=$CI_DIR" >> $GITHUB_ENV - git clone -b ${ci_branch} --single-branch ${ci_repo_url} + git clone -b ${ci_branch//\"} --single-branch ${ci_repo_url//\"} cd $WORKSPACE @@ -171,6 +177,7 @@ jobs: if [ ! -d $WORKSPACE/build ]; then mkdir $WORKSPACE/build fi + BUILD_DIR=$WORKSPACE/build echo "BUILD_DIR=$BUILD_DIR" >> $GITHUB_ENV @@ -189,17 +196,11 @@ jobs: continue-on-error: true shell: bash run: | - #TODO - #HEre we use docker if possible - - # Setup variables for following calls - . ${CI_DIR}/scripts/utils.sh - NODE_NAME="$(hostname)" # Needed by load-vm-env --> TODO: be able to get hostname multiplatform - SCRIPT_DIR=${CI_DIR}/scripts/ # Needed by load-vm-env, might need to run this inside the docker env - CI_PYTHON3_VERSION=${{ inputs.python-version }} # Needed by load-vm-env, might need to run this inside the docker env - - ## Setup env variables - load-vm-env + ## Configure the build: setup cmake variables + # retrive build type and compiler + BUILD_TYPE=$(echo ${{ matrix.os }} | awk -F '_' '{print $3}') + CONFIG=$(echo ${{ matrix.os }} | awk -F '_' '{print $1"_"$2}') + NODE_NAME="${{ runner.name }}" # Needed by configure-and-build --> TODO: be able to get hostname multiplatform ## Deal with ci-depends-on # Loop over each key-value pair in the JSON avoiding ci repository if inside @@ -219,25 +220,64 @@ jobs: done if [[ ! -n "$CMAKE_OPTIONS" ]]; then - CMAKE_OPTIONS="no-ci-depends-on" + CMAKE_OPTIONS="no-ci-depends-on" + echo "No ci-depends-on detected." + else + echo "ci-depends-on detected, adding the following cmake variable : $CMAKE_OPTIONS" fi + echo "" + echo "------ Before build (and docker) ------" + echo "Main folders:" + echo " - WORKSPACE = $WORKSPACE" + echo " - SRC_DIR = $SRC_DIR" + echo " - CI_DIR = $CI_DIR" + echo " - BUILD_DIR = $BUILD_DIR" + echo "" + echo "Configuration:" + echo " - NODE_NAME = $NODE_NAME" + echo " - BUILD_TYPE = $BUILD_TYPE" + echo " - CONFIG = $CONFIG" + echo " - PYTHON_VERSION = ${{ inputs.python-version }}" + echo " - PRESET = ${{ inputs.preset }}" + echo " - GENERATE_BINARIES = ${{ inputs.generate-binaries }}" + echo " - ADDITIONNAL_CMAKE_OPTIONS = $CMAKE_OPTIONS" + echo "---------------------------------------" + echo "" + + #If os has got docker then use it + if [[ "${{ matrix.os }}" == *"ubuntu"* ]] || [[ "${{ matrix.os }}" == *"fedora"* ]]; then + builder_type=$(echo "${{ matrix.os }}" | awk -F '_' '{print $1}' | awk -F '-' '{print $2}') + if [[ "${{ inputs.sofa-branch-name }}" == "master" ]] || [[ "${{ inputs.sofa-branch-name }}" =~ ^v[0-9]{2}\.[0-9]{2}$ ]]; then + default_tag=${{ inputs.sofa-branch-name }} + else + default_tag=master + fi - ## Configure the build: setup cmake variables - # retrive build type and compiler - BUILD_TYPE=$(echo ${{ matrix.os }} | awk -F '_' '{print $3}') - CONFIG=$(echo ${{ matrix.os }} | awk -F '_' '{print $1"_"$2}') + echo "dckr_pat_ohBds9gXTs9Iy91QPKJZXGdVS5s" | docker login -u sofaframework --password-stdin - BUILD_OPTIONS="build-scope-${{ inputs.preset }}" - if [ "${{ inputs.generate-binaries }}" == "true" ]; then - BUILD_OPTIONS="$BUILD_OPTIONS build-release-package" - fi + echo "Pulling Docker image sofaframework/sofabuilder_${builder_type}:${CI_BRANCH} ..." + docker pull --quiet sofaframework/sofabuilder_${builder_type}:${CI_BRANCH} || true + DOCKER_IMAGE="sofaframework/sofabuilder_${builder_type}:${CI_BRANCH}" - . ${CI_DIR}/scripts/configure.sh "$BUILD_DIR" "$SRC_DIR" "$CONFIG" "$CMAKE_OPTIONS" "$BUILD_TYPE" "$BUILD_OPTIONS" + if [[ "$(docker image list --format "table {{.Repository}}:{{.Tag}}")" != *"sofaframework/sofabuilder_${builder_type}:${CI_BRANCH}"* ]]; then + echo "Docker image sofaframework/sofabuilder_${builder_type}:${CI_BRANCH} doesn't exist, pulling default tag ${default_tag} " + docker pull --quiet sofaframework/sofabuilder_${builder_type}:${default_tag} - ## Call to build - $SCRIPT_DIR/compile.sh "$BUILD_DIR" "$CONFIG" "$BUILD_OPTIONS" + DOCKER_IMAGE="sofaframework/sofabuilder_${builder_type}:${default_tag}" + fi + echo "Launching configuration and build through docker using image ${DOCKER_IMAGE}. Running ./ci/scripts/configure-and-build.sh /workspace/build/ /workspace/sofa/ /workspace/ci/scripts/ \"${NODE_NAME}\" \"${BUILD_TYPE}\" \"${CONFIG}\" \"${{ inputs.python-version }}\" \"${{ inputs.preset }}\" \"${{ inputs.generate-binaries }}\" \"${CMAKE_OPTIONS}\"" + docker run --rm \ + --user $(id -u):$(id -g) --network=host -v $WORKSPACE:/workspace \ + ${DOCKER_IMAGE} \ + /bin/bash -c "env ; cd /workspace; /bin/bash ./ci/scripts/configure-and-build.sh /workspace/build/ /workspace/sofa/ /workspace/ci/scripts/ \"${NODE_NAME}\" \"${BUILD_TYPE}\" \"${CONFIG}\" \"${{ inputs.python-version }}\" \"${{ inputs.preset }}\" \"${{ inputs.generate-binaries }}\" \"${CMAKE_OPTIONS}\"" + else + echo "Launching configuration and build" + + bash ${CI_DIR}/scripts/configure-and-build.sh "${BUILD_DIR}" "${SRC_DIR}" "${CI_DIR}/scripts/" "${NODE_NAME}" "${BUILD_TYPE}" "${CONFIG}" "${{ inputs.python-version }}" "${{ inputs.preset }}" "${{ inputs.generate-binaries }}" "${CMAKE_OPTIONS}" + fi + - name: Notify dashboard shell: bash diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py index 9bd85756b57..27082fd3cf8 100644 --- a/scripts/github_CI/checkPRInfoBeforeBuild.py +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -179,8 +179,8 @@ def extract_ci_depends_on(): dependency_pr_data = response.json() - key = dependency_pr_data['head']['repo']['name'] #Sofa.Qt - repo_url = dependency_pr_data['html_url'] #https://github.com/sofa-framework/Sofa.Qt + key = dependency_pr_data['base']['repo']['name'] #Sofa.Qt + repo_url = dependency_pr_data['head']['repo']['html_url'] #https://github.com/{remote from which pr comes}/Sofa.Qt branch_name = dependency_pr_data['head']['ref'] #my_feature_branch dependency_dict[key] = { @@ -217,7 +217,9 @@ def extract_ci_depends_on(): with open(os.environ["GITHUB_ENV"], "a") as env_file: env_file.write(f"WITH_ALL_TESTS={with_all_tests_found}\n") env_file.write(f"FORCE_FULL_BUILD={force_full_build_found}\n") - env_file.write(f"CI_DEPENDS_ON={dependency_dict}\n") + + ci_depends_on_str = f"{dependency_dict}".replace("'", "\\\"") + env_file.write(f"CI_DEPENDS_ON={ci_depends_on_str}\n") env_file.write(f'BUILDER_OS=["sh-ubuntu_gcc_release","sh-fedora_clang_release","sh-windows_vs2022_release","sh-macos_clang_release"]') From 242503a59af92e42f12f89f9708ee9c899373fbe Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 17 Apr 2025 16:33:31 +0200 Subject: [PATCH 44/61] Filter build only if not schedule, PR comment linked steps (#21) * Filter build only if not schedule, PR comment linked steps * Apply suggestions from code review --------- Co-authored-by: Paul Baksic <30337881+bakpaul@users.noreply.github.com> --- .github/workflows/filter-build.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 0e93f70d152..c525a4c8a64 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -60,7 +60,9 @@ on: jobs: + # Filter build handling : push in master, commits in PR, comments in PR and dispatch filter_build: + if: ${{ github.event_name != 'schedule'}} runs-on: ubuntu-latest outputs: SOFA_BRANCH_NAME: ${{ steps.export-vars.outputs.SOFA_BRANCH_NAME }} @@ -200,7 +202,8 @@ jobs: # Trigger the Build action python scripts/github_CI/checkPRInfoBeforeBuild.py - - name: Run when PR comment is created or edited (set up environment var for python script) + - name: Run when PR comment is created or edited > set up environment variables + id: pr-comment-setup if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -234,8 +237,8 @@ jobs: echo "Env: PR_COMMIT_SHA = $PR_COMMIT_SHA" fi - - name: Run when PR comment is created or edited (trigger python script) - if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }} + - name: Run when PR comment is created or edited > trigger python script + if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request && steps.pr-comment-setup.outcome == 'success' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.issue.number }} @@ -274,7 +277,7 @@ jobs: - + # Nightly build triggered once a day nightly_build: runs-on: ubuntu-latest if: github.event.schedule == '0 2 * * *' @@ -359,7 +362,7 @@ jobs: # =============================================================== # =============================================================== - # Trigger the build with all parameters + # Trigger the build and sharing all parameters from filter_build > outputs call-workflow-passing-data: needs: filter_build uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master From 805ccd6b0622a3abb95d0264041da16fc0bffa62 Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:48:06 +0200 Subject: [PATCH 45/61] Fix when ci-depends-on is empty (#22) --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index aa8ebd667a3..bce92736b04 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -150,7 +150,7 @@ jobs: # check if ci has a ci-depends-on ci_ci_depends_on=$(echo "${{ inputs.ci-depends-on }}" | jq .ci) - if [ "$ci_ci_depends_on" != "null" ]; then + if [ -n "${{ inputs.ci-depends-on }}" ] && [ "$ci_ci_depends_on" != "null" ]; then echo "ci-depends-on for ci repository detected." ci_repo_url=$(echo "${{ inputs.ci-depends-on }}" | jq .ci.repo_url) ci_branch=$(echo "${{ inputs.ci-depends-on }}" | jq .ci.branch_name) From 6232b5fbb0301d68ab81e197e096a459871b1d0d Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Thu, 17 Apr 2025 17:57:24 +0200 Subject: [PATCH 46/61] Manage out of tree plugins and simplify binaries generation (#23) * Manage out of tree plugins and simplify binaries generation * Add external plugins mechanism --- .github/workflows/build-and-test.yml | 26 ++++++++++--- .github/workflows/filter-build.yml | 58 ++++++++++++++-------------- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index bce92736b04..f09d54f84d7 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -38,7 +38,10 @@ on: type: boolean required: false default: false - out-of-tree-build: + external-plugins: + type: string + required: false + additionnal-cmake-flags: type: string required: false builder-os: @@ -70,7 +73,8 @@ jobs: echo "with-all-tests : ${{ inputs.with-all-tests }}" echo "force-full-build : ${{ inputs.force-full-build }}" echo "generate-binaries : ${{ inputs.generate-binaries }}" - echo "out-of-tree-build : ${{ inputs.out-of-tree-build }}" + echo "external-plugins : ${{ inputs.external-plugins }}" + echo "additionnal-cmake-flags : ${{ inputs.additionnal-cmake-flags }}" echo "builder-os : ${{ inputs.builder-os }}" build: @@ -181,6 +185,18 @@ jobs: BUILD_DIR=$WORKSPACE/build echo "BUILD_DIR=$BUILD_DIR" >> $GITHUB_ENV + if [ -n "${{ inputs.external-plugins }}" ]; then + echo "Setting up external plugins." + for plugin in ${{ inputs.external-plugins }}; do + plugin_base=${plugin%@*} + plugin_branch=${plugin##*@} + plugin_repo=$(basename "$plugin_base") + + echo "Adding line 'sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base)' to file ${SRC_DIR}/applications/CMakeLists.txt" + echo "sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base)' >> ${SRC_DIR}/applications/CMakeLists.txt" + done + fi + - name: Notify dashboard shell: bash @@ -204,7 +220,7 @@ jobs: ## Deal with ci-depends-on # Loop over each key-value pair in the JSON avoiding ci repository if inside - CMAKE_OPTIONS="" + CMAKE_OPTIONS="${{inputs.additionnal-cmake-flags}}" for key in $(echo "${{ inputs.ci-depends-on }}" | jq -r 'keys[]' | grep -v '^ci$'); do repo_url=$(echo "${{ inputs.ci-depends-on }}" | jq -r ".\"$key\".repo_url") @@ -220,10 +236,10 @@ jobs: done if [[ ! -n "$CMAKE_OPTIONS" ]]; then - CMAKE_OPTIONS="no-ci-depends-on" + CMAKE_OPTIONS="no-additionnal-cmake-flags" echo "No ci-depends-on detected." else - echo "ci-depends-on detected, adding the following cmake variable : $CMAKE_OPTIONS" + echo "Adding the following cmake variable : $CMAKE_OPTIONS" fi echo "" diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index c525a4c8a64..4ade99fdf5e 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -13,30 +13,20 @@ on: description: 'Specify the stable branch to use to generate the new binaries' required: true type: string - commit_sha: - description: 'Commit SHA to check out' - required: true - default: 'head' - type: string python_version: description: 'Version of Python used' required: true default: '3.12' type: string - preset: - description: 'Preset to use from CMakePresets.json file in main repository' - required: true - default: 'full' + external-plugins: + description: 'List of github repository and branch name to add to build tree, separated by a space. Syntax: https://www.github.com/me/myrepo@mybranch' + required: false type: string - generate_binaries: - description: 'Boolean to generate the binaries' - required: true - default: true - type: boolean - builder_os: + additionnal-cmake-flags: + description: 'CMake flags to add during the CMake call. This is to be used along external-plugins. The syntax is identical to a CMake call' + required: false type: string - required: true - default: '["sh-ubuntu_gcc_release"]' + # Nightly build schedule: @@ -72,7 +62,8 @@ jobs: CI_DEPENDS_ON: ${{ steps.export-vars.outputs.CI_DEPENDS_ON }} WITH_ALL_TESTS: ${{ steps.export-vars.outputs.WITH_ALL_TESTS }} FORCE_FULL_BUILD: ${{ steps.export-vars.outputs.FORCE_FULL_BUILD }} - OUT_OF_TREE_BUILD: ${{ steps.export-vars.outputs.OUT_OF_TREE_BUILD }} + EXTERNAL_PLUGINS: ${{ steps.export-vars.outputs.EXTERNAL_PLUGINS }} + ADDITIONNAL_CMAKE_FLAGS: ${{ steps.export-vars.outputs.ADDITIONNAL_CMAKE_FLAGS }} GENERATE_BINARIES: ${{ steps.export-vars.outputs.GENERATE_BINARIES }} PR_OWNER_URL: ${{ steps.export-vars.outputs.PR_OWNER_URL }} PR_BRANCH_NAME: ${{ steps.export-vars.outputs.PR_BRANCH_NAME }} @@ -89,7 +80,8 @@ jobs: echo "CI_DEPENDS_ON=" >> $GITHUB_ENV # CI_DEPENDS_ON: "" echo "WITH_ALL_TESTS=false" >> $GITHUB_ENV # WITH_ALL_TESTS: false echo "FORCE_FULL_BUILD=false" >> $GITHUB_ENV # FORCE_FULL_BUILD: false - echo "OUT_OF_TREE_BUILD=" >> $GITHUB_ENV # OUT_OF_TREE_BUILD: "" + echo "EXTERNAL_PLUGINS=" >> $GITHUB_ENV # EXTERNAL_PLUGINS: "" + echo "ADDITIONNAL_CMAKE_FLAGS=" >> $GITHUB_ENV # ADDITIONNAL_CMAKE_FLAGS: "" echo "GENERATE_BINARIES=false" >> $GITHUB_ENV # GENERATE_BINARIES: false echo "PR_OWNER_URL=" >> $GITHUB_ENV # PR_OWNER_URL: "" echo "PR_BRANCH_NAME=" >> $GITHUB_ENV # PR_BRANCH_NAME: "" @@ -102,11 +94,13 @@ jobs: echo "This step runs only for binary generation." BRANCH=${{ github.event.inputs.branch }} - COMMIT=${{ github.event.inputs.commit_sha }} + COMMIT="HEAD" PYTHON=${{ github.event.inputs.python_version }} - PRESET=${{ github.event.inputs.preset }} - GENERATE_BINARIES=${{ github.event.inputs.generate_binaries }} - BUILDER_OS=${{ github.event.inputs.builder_os }} + PRESET="full" + GENERATE_BINARIES=true + BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"] + EXTERNAL_PLUGINS="${{ inputs.external-plugins }}" + ADDITIONNAL_CMAKE_FLAGS="${{ inputs.additionnal-cmake-flags }}" # Validate branch format (e.g., v25.06) if [ [ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ] && [ "$BRANCH" != "master" ] ]; then @@ -166,6 +160,9 @@ jobs: echo "PYTHON_VERSION=$PYTHON" >> $GITHUB_ENV echo "GENERATE_BINARIES=$GENERATE_BINARIES" >> $GITHUB_ENV echo "BUILDER_OS=$BUILDER_OS" >> $GITHUB_ENV + echo "EXTERNAL_PLUGINS=$EXTERNAL_PLUGINS" >> $GITHUB_ENV + echo "ADDITIONNAL_CMAKE_FLAGS=$ADDITIONNAL_CMAKE_FLAGS" >> $GITHUB_ENV + - name: Set up python uses: actions/setup-python@v5 @@ -268,7 +265,8 @@ jobs: echo "CI_DEPENDS_ON=${CI_DEPENDS_ON}" >> $GITHUB_OUTPUT echo "WITH_ALL_TESTS=${WITH_ALL_TESTS}" >> $GITHUB_OUTPUT echo "FORCE_FULL_BUILD=${FORCE_FULL_BUILD}" >> $GITHUB_OUTPUT - echo "OUT_OF_TREE_BUILD=${OUT_OF_TREE_BUILD}" >> $GITHUB_OUTPUT + echo "EXTERNAL_PLUGINS=${EXTERNAL_PLUGINS}" >> $GITHUB_OUTPUT + echo "ADDITIONNAL_CMAKE_FLAGS=${ADDITIONNAL_CMAKE_FLAGS}" >> $GITHUB_OUTPUT echo "GENERATE_BINARIES=${GENERATE_BINARIES}" >> $GITHUB_OUTPUT echo "PR_OWNER_URL=${PR_OWNER_URL}" >> $GITHUB_OUTPUT echo "PR_BRANCH_NAME=${PR_BRANCH_NAME}" >> $GITHUB_OUTPUT @@ -293,7 +291,8 @@ jobs: CI_DEPENDS_ON: ${{ steps.export-vars.outputs.CI_DEPENDS_ON }} WITH_ALL_TESTS: ${{ steps.export-vars.outputs.WITH_ALL_TESTS }} FORCE_FULL_BUILD: ${{ steps.export-vars.outputs.FORCE_FULL_BUILD }} - OUT_OF_TREE_BUILD: ${{ steps.export-vars.outputs.OUT_OF_TREE_BUILD }} + EXTERNAL_PLUGINS: ${{ steps.export-vars.outputs.EXTERNAL_PLUGINS }} + ADDITIONNAL_CMAKE_FLAGS: ${{ steps.export-vars.outputs.ADDITIONNAL_CMAKE_FLAGS }} GENERATE_BINARIES: ${{ steps.export-vars.outputs.GENERATE_BINARIES }} PR_OWNER_URL: ${{ steps.export-vars.outputs.PR_OWNER_URL }} PR_BRANCH_NAME: ${{ steps.export-vars.outputs.PR_BRANCH_NAME }} @@ -310,7 +309,8 @@ jobs: echo "CI_DEPENDS_ON=" >> $GITHUB_ENV # CI_DEPENDS_ON: "" echo "WITH_ALL_TESTS=false" >> $GITHUB_ENV # WITH_ALL_TESTS: false echo "FORCE_FULL_BUILD=false" >> $GITHUB_ENV # FORCE_FULL_BUILD: false - echo "OUT_OF_TREE_BUILD=" >> $GITHUB_ENV # OUT_OF_TREE_BUILD: "" + echo "EXTERNAL_PLUGINS=" >> $GITHUB_ENV # EXTERNAL_PLUGINS: "" + echo "ADDITIONNAL_CMAKE_FLAGS=" >> $GITHUB_ENV # ADDITIONNAL_CMAKE_FLAGS: "" echo "GENERATE_BINARIES=false" >> $GITHUB_ENV # GENERATE_BINARIES: false echo "PR_OWNER_URL=" >> $GITHUB_ENV # PR_OWNER_URL: "" echo "PR_BRANCH_NAME=" >> $GITHUB_ENV # PR_BRANCH_NAME: "" @@ -349,7 +349,8 @@ jobs: echo "CI_DEPENDS_ON=${CI_DEPENDS_ON}" >> $GITHUB_OUTPUT echo "WITH_ALL_TESTS=${WITH_ALL_TESTS}" >> $GITHUB_OUTPUT echo "FORCE_FULL_BUILD=${FORCE_FULL_BUILD}" >> $GITHUB_OUTPUT - echo "OUT_OF_TREE_BUILD=${OUT_OF_TREE_BUILD}" >> $GITHUB_OUTPUT + echo "EXTERNAL_PLUGINS=${EXTERNAL_PLUGINS}" >> $GITHUB_OUTPUT + echo "ADDITIONNAL_CMAKE_FLAGS=${ADDITIONNAL_CMAKE_FLAGS}" >> $GITHUB_OUTPUT echo "GENERATE_BINARIES=${GENERATE_BINARIES}" >> $GITHUB_OUTPUT echo "PR_OWNER_URL=${PR_OWNER_URL}" >> $GITHUB_OUTPUT echo "PR_BRANCH_NAME=${PR_BRANCH_NAME}" >> $GITHUB_OUTPUT @@ -374,7 +375,8 @@ jobs: ci-depends-on: ${{ needs.filter_build.outputs.CI_DEPENDS_ON }} with-all-tests: ${{ needs.filter_build.outputs.WITH_ALL_TESTS == 'true'}} force-full-build: ${{ needs.filter_build.outputs.FORCE_FULL_BUILD == 'true'}} - out-of-tree-build: ${{ needs.filter_build.outputs.OUT_OF_TREE_BUILD }} + external-plugins: ${{ needs.filter_build.outputs.EXTERNAL_PLUGINS }} + additionnal-cmake-flags: ${{ needs.filter_build.outputs.ADDITIONNAL_CMAKE_FLAGS }} generate-binaries: ${{ needs.filter_build.outputs.GENERATE_BINARIES == 'true'}} pr-owner-url: ${{ needs.filter_build.outputs.PR_OWNER_URL }} pr-branch-name: ${{ needs.filter_build.outputs.PR_BRANCH_NAME }} From 9499d7d503eeecffc835532553b0fc5b8b0da472 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Thu, 17 Apr 2025 18:09:06 +0200 Subject: [PATCH 47/61] Fix filter build --- .github/workflows/filter-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 4ade99fdf5e..4c57b07e172 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -124,9 +124,9 @@ jobs: fi # Validate Python version format (e.g., 3.12) - if [[ ! "PYTHON" =~ ^[0-9]{1}\.[0-9]{2}$ ]]; then - if [[ ! "PYTHON" =~ ^3\.(9|1[0-8])$ ]]; then - echo "Error: Invalid Python version format: PYTHON" + if [[ ! "$PYTHON" =~ ^[0-9]{1}\.[0-9]{2}$ ]]; then + if [[ ! "$PYTHON" =~ ^3\.(9|1[0-8])$ ]]; then + echo "Error: Invalid Python version format: $PYTHON" exit 1 fi fi From 0a14604e61fb626766a669fb5343de4f88b8b841 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Thu, 17 Apr 2025 18:10:35 +0200 Subject: [PATCH 48/61] Restore changes --- .github/workflows/build-and-test.yml | 9 +++++++-- .github/workflows/filter-build.yml | 22 ++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f09d54f84d7..c2fd468d71e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -149,8 +149,13 @@ jobs: cd $WORKSPACE ## Clone CI and use ci-depends-on structure - ci_branch=${{ inputs.sofa-branch-name }} - ci_repo_url="https://www.github.com/sofa-framework/ci" + ### TODO: UNCOMMENT THIS WHEN PUSHIN TO REAL SOFA + # ci_branch=${{ inputs.sofa-branch-name }} + # ci_repo_url="https://www.github.com/sofa-framework/ci" + ### AND REMOVE THOSE + ci_branch=jenkins_gha_migration + ci_repo_url="https://www.github.com/bakpaul/ci" + # check if ci has a ci-depends-on ci_ci_depends_on=$(echo "${{ inputs.ci-depends-on }}" | jq .ci) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 4c57b07e172..d046748a473 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -111,16 +111,18 @@ jobs: echo "Branch name $BRANCH is valid." # Check commit SHA format - if [[ ! "$COMMIT" =~ ^[a-fA-F0-9]{40}$ ]]; then - echo "Error: Invalid commit SHA format: $COMMIT" - exit 1 - fi - # Check the commit exists in the repository - if git cat-file -e "$COMMIT" 2>/dev/null; then - echo "Commit $COMMIT is valid." - else - echo "Commit $COMMIT does not exist in the repository." - exit 1 + if [[ "$COMMIT" != "HEAD" ]]; then + if [[ ! "$COMMIT" =~ ^[a-fA-F0-9]{40}$ ]]; then + echo "Error: Invalid commit SHA format: $COMMIT" + exit 1 + fi + # Check the commit exists in the repository + if git cat-file -e "$COMMIT" 2>/dev/null; then + echo "Commit $COMMIT is valid." + else + echo "Commit $COMMIT does not exist in the repository." + exit 1 + fi fi # Validate Python version format (e.g., 3.12) From fd5e726302dffe597db53cc8bd4db796e96a0c15 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Thu, 17 Apr 2025 18:15:04 +0200 Subject: [PATCH 49/61] Remove unnecessary checks in workflow dispatch --- .github/workflows/filter-build.yml | 36 ------------------------------ 1 file changed, 36 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index d046748a473..36ad3ce1aaa 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -110,21 +110,6 @@ jobs: fi echo "Branch name $BRANCH is valid." - # Check commit SHA format - if [[ "$COMMIT" != "HEAD" ]]; then - if [[ ! "$COMMIT" =~ ^[a-fA-F0-9]{40}$ ]]; then - echo "Error: Invalid commit SHA format: $COMMIT" - exit 1 - fi - # Check the commit exists in the repository - if git cat-file -e "$COMMIT" 2>/dev/null; then - echo "Commit $COMMIT is valid." - else - echo "Commit $COMMIT does not exist in the repository." - exit 1 - fi - fi - # Validate Python version format (e.g., 3.12) if [[ ! "$PYTHON" =~ ^[0-9]{1}\.[0-9]{2}$ ]]; then if [[ ! "$PYTHON" =~ ^3\.(9|1[0-8])$ ]]; then @@ -134,27 +119,6 @@ jobs: fi echo "Python version $PYTHON is valid." - # Check if the preset is valid - ALLOWED_PRESETS=("minimal" "standard" "full" "minimal-dev" "standard-dev" "full-dev") - if [[ ! " ${ALLOWED_PRESETS[@]} " =~ " ${PRESET} " ]]; then - echo "Error: Invalid preset $PRESET. Must be one of: ${ALLOWED_PRESETS[*]}" - exit 1 - fi - echo "Preset $PRESET is valid." - - # Validate generate_binaries (must be true or false) - if [[ "$GENERATE_BINARIES" != "true" && "$GENERATE_BINARIES" != "false" ]]; then - echo "Error: 'generate_binaries' must be either 'true' or 'false'." - exit 1 - fi - - # Validate builder_os (must be a list) - python3 -c "import ast; x=ast.literal_eval('$BUILDER_OS'); assert isinstance(x, list)" 2>/dev/null - if [[ $? -ne 0 ]]; then - echo "Error: 'builder_os' must be a list." - return 1 - fi - # Save all information in environment variables echo "SOFA_BRANCH_NAME=$BRANCH" >> $GITHUB_ENV echo "SOFA_COMMIT_SHA=$COMMIT" >> $GITHUB_ENV From b5f457e76aa98b324500ebc213fce8c5d46f81c9 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Thu, 17 Apr 2025 18:18:20 +0200 Subject: [PATCH 50/61] Fix builder os matrix --- .github/workflows/filter-build.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 36ad3ce1aaa..5ba135a3b87 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -94,11 +94,7 @@ jobs: echo "This step runs only for binary generation." BRANCH=${{ github.event.inputs.branch }} - COMMIT="HEAD" PYTHON=${{ github.event.inputs.python_version }} - PRESET="full" - GENERATE_BINARIES=true - BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"] EXTERNAL_PLUGINS="${{ inputs.external-plugins }}" ADDITIONNAL_CMAKE_FLAGS="${{ inputs.additionnal-cmake-flags }}" @@ -121,11 +117,11 @@ jobs: # Save all information in environment variables echo "SOFA_BRANCH_NAME=$BRANCH" >> $GITHUB_ENV - echo "SOFA_COMMIT_SHA=$COMMIT" >> $GITHUB_ENV - echo "PRESET=$PRESET" >> $GITHUB_ENV + echo "SOFA_COMMIT_SHA=HEAD" >> $GITHUB_ENV + echo "PRESET=full" >> $GITHUB_ENV echo "PYTHON_VERSION=$PYTHON" >> $GITHUB_ENV - echo "GENERATE_BINARIES=$GENERATE_BINARIES" >> $GITHUB_ENV - echo "BUILDER_OS=$BUILDER_OS" >> $GITHUB_ENV + echo "GENERATE_BINARIES=true" >> $GITHUB_ENV + echo "BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]" >> $GITHUB_ENV echo "EXTERNAL_PLUGINS=$EXTERNAL_PLUGINS" >> $GITHUB_ENV echo "ADDITIONNAL_CMAKE_FLAGS=$ADDITIONNAL_CMAKE_FLAGS" >> $GITHUB_ENV From daee071406b3d768f5dbd5f168de3552f3a68452 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Thu, 17 Apr 2025 18:20:37 +0200 Subject: [PATCH 51/61] Fix builder os matrix --- .github/workflows/filter-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 5ba135a3b87..9ed6b59c10d 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -121,7 +121,7 @@ jobs: echo "PRESET=full" >> $GITHUB_ENV echo "PYTHON_VERSION=$PYTHON" >> $GITHUB_ENV echo "GENERATE_BINARIES=true" >> $GITHUB_ENV - echo "BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]" >> $GITHUB_ENV + echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV echo "EXTERNAL_PLUGINS=$EXTERNAL_PLUGINS" >> $GITHUB_ENV echo "ADDITIONNAL_CMAKE_FLAGS=$ADDITIONNAL_CMAKE_FLAGS" >> $GITHUB_ENV From 1deb2092e6bc839f41ec853791ec982477eb913e Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Thu, 17 Apr 2025 18:26:00 +0200 Subject: [PATCH 52/61] Activate plugins that are added (#25) --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index c2fd468d71e..f2c0b8a8734 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -197,8 +197,8 @@ jobs: plugin_branch=${plugin##*@} plugin_repo=$(basename "$plugin_base") - echo "Adding line 'sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base)' to file ${SRC_DIR}/applications/CMakeLists.txt" - echo "sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base)' >> ${SRC_DIR}/applications/CMakeLists.txt" + echo "Adding line 'sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base ON)' to file ${SRC_DIR}/applications/CMakeLists.txt" + echo "sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base ON)' >> ${SRC_DIR}/applications/CMakeLists.txt" done fi From 398970d601c6058acf80eb7dc5a3233d8c5d65b4 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 22 Apr 2025 15:30:10 +0200 Subject: [PATCH 53/61] fix adding external repo --- .github/workflows/build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f2c0b8a8734..740ad2fce31 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -198,7 +198,7 @@ jobs: plugin_repo=$(basename "$plugin_base") echo "Adding line 'sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base ON)' to file ${SRC_DIR}/applications/CMakeLists.txt" - echo "sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base ON)' >> ${SRC_DIR}/applications/CMakeLists.txt" + echo "sofa_add_external(plugin $plugin_repo GIT_REF $plugin_branch GIT_REPOSITORY $plugin_base ON)" >> "${SRC_DIR}/applications/CMakeLists.txt" done fi From 93b5b9303d03e7b9e1407549f8433f54ccc7af34 Mon Sep 17 00:00:00 2001 From: Paul Baksic Date: Tue, 22 Apr 2025 15:44:14 +0200 Subject: [PATCH 54/61] Force full build on workflow dispatch --- .github/workflows/filter-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/filter-build.yml index 9ed6b59c10d..c81dfcb892d 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/filter-build.yml @@ -124,7 +124,8 @@ jobs: echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV echo "EXTERNAL_PLUGINS=$EXTERNAL_PLUGINS" >> $GITHUB_ENV echo "ADDITIONNAL_CMAKE_FLAGS=$ADDITIONNAL_CMAKE_FLAGS" >> $GITHUB_ENV - + echo "FORCE_FULL_BUILD=true" >> $GITHUB_ENV + - name: Set up python uses: actions/setup-python@v5 From 27a49f2108d38975f65b84728a28ea0725e933ea Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Mon, 28 Apr 2025 18:50:27 +0200 Subject: [PATCH 55/61] Add missing inputs to filter build action (#26) * Test action * Fix action * Test pre job hook * Add setup_build step * Fix action * FIX when folders already exist + TODOS * Add filter action and scripts * replace sofa-framework by bakpaul * fix indentation * add checkout code action * Add backbone * Cloning SOFA working * Forgot to save * add chmod to run script * Add call-workflow, refactor env variables, update python script * remove useless comment * Apply suggestions from code review * Apply suggestions from code review * Update filter-build.yml * Finalize clone sofa and ci (#7) * Push space to creatye PR * Finished clone sofa and ci stage * Rename action * fix some typos in filter script * Add build step (#8) * Commit to open a PR * revert env modification * Move steps around + add os configuration * Configure step should be working * Move comment around * Step build * Use step outputs to send variables in call workflow (#9) * Use step outputs to send variables in call workflow * add BUILDER_OS environment variable * add check for all tests in PR body * remove useless secret * udpate PR python script * add missing runs-on: ubuntu-latest * Fix build action/ ci-depends-on * remove runs-on * update python script * clean python script * fix export of booleans * Fix action --------- Co-authored-by: Paul Baksic * Update checkPRInfoBeforeBuild.py (#11) * Update checkPRInfoBeforeBuild.py * Apply suggestions from code review Co-authored-by: Paul Baksic <30337881+bakpaul@users.noreply.github.com> * Apply suggestions from code review * Apply suggestions from code review * Test * Use this script * fix * test fromJson * add debug * Final fix * revert change * Refacto display action * Fix filter --------- Co-authored-by: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Co-authored-by: Paul Baksic * Fix build script (#13) * TO BE REVERTED BEFORE MERGE * Fix syntax * Fix git handle * fix action location * fix typo * Fix call to load vm env * fix script + revert wrong changes (#15) * Update filter-build.yml (#16) * Fix sha when commit pushed in master (#17) * Temporarily change remote for non-PR (#18) * 202505 evolve filter nightly (#19) * Isolate nightly build to allow for matrix only in nightly cases * Check that SOFA branch name is either master or vXX.XX * Attempt to detect existing running builds on PR * Fix check on ci-depends-on * fix indent and add generic condition for nightly build * Another fix for ci-depends-on * Another fix for ci-depends-on in json * Display info on PR comment * Display info on PR comment * Display info on PR comment * Fix trigger through comments (#20) * Fix trigger through comments * remove check when action is already triggered * remove test comment * move echo to read it * attempt to access information PR comment * attempt to access information PR comment * complete access information PR comment * complete access information PR comment * Update search of PR information when comment_issue * Remove empty env var * add check on OWNER_NAME var * add check on OWNER_NAME var * last attempt to fill env var for comment_issue * split in two steps for comment_issue * add missing PR number for comment_issue (step 2) * Start work on docker use (#10) * Start work * Add docker run for ubuntu and fedora * TOREVERT * Add insights and login * Fix action * Try fix docker autentification * .github/workflows/ * Add read only token to action * fix docker user name * Change pipe to or * Fix docker call * try fix docker * fix ? * Fix ci-depends-on mechanism * Fix quote outputed by jq * Restore branch name * fix python script * Fix ci-depends-on remote url * Fix script not launching * Display arguments * Use github env variable for node name * Unify workspace folder using matrix os * Add display * Add display * Update .github/workflows/filter-build.yml Co-authored-by: Hugo --------- Co-authored-by: Hugo * Filter build only if not schedule, PR comment linked steps (#21) * Filter build only if not schedule, PR comment linked steps * Apply suggestions from code review --------- Co-authored-by: Paul Baksic <30337881+bakpaul@users.noreply.github.com> * Fix when ci-depends-on is empty (#22) * Manage out of tree plugins and simplify binaries generation (#23) * Manage out of tree plugins and simplify binaries generation * Add external plugins mechanism * Fix filter build * Restore changes * Remove unnecessary checks in workflow dispatch * Fix builder os matrix * Fix builder os matrix * Activate plugins that are added (#25) * add missing inputs * fix typo * Set standard-dev preset for nightly and rename action * Various cleaning * Rename action --------- Co-authored-by: hugtalbot --- ...-build.yml => trigger-build-and-tests.yml} | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) rename .github/workflows/{filter-build.yml => trigger-build-and-tests.yml} (91%) diff --git a/.github/workflows/filter-build.yml b/.github/workflows/trigger-build-and-tests.yml similarity index 91% rename from .github/workflows/filter-build.yml rename to .github/workflows/trigger-build-and-tests.yml index c81dfcb892d..e2d3dc0567e 100644 --- a/.github/workflows/filter-build.yml +++ b/.github/workflows/trigger-build-and-tests.yml @@ -1,5 +1,4 @@ -name: Filter build action - +name: Trigger build and tests # =============================================================== # =============================================================== @@ -26,6 +25,27 @@ on: description: 'CMake flags to add during the CMake call. This is to be used along external-plugins. The syntax is identical to a CMake call' required: false type: string + preset: + type: choice + description: Which preset to use for compilation + options: + - standard + - supported-plugins + - full + - standard-dev + - supported-plugins-dev + - full-dev + default: 'full' + builder-os: + type: choice + description: On which OS run the binaries generation + options: + - '["sh-ubuntu_gcc_release"]' + - '["sh-windows_vs2022_release"]' + - '["sh-macos_clang_release"]' + - '["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' + default: '["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' + # Nightly build @@ -95,8 +115,6 @@ jobs: BRANCH=${{ github.event.inputs.branch }} PYTHON=${{ github.event.inputs.python_version }} - EXTERNAL_PLUGINS="${{ inputs.external-plugins }}" - ADDITIONNAL_CMAKE_FLAGS="${{ inputs.additionnal-cmake-flags }}" # Validate branch format (e.g., v25.06) if [ [ ! "$BRANCH" =~ ^v[0-9]{2}\.[0-9]{2}$ ] && [ "$BRANCH" != "master" ] ]; then @@ -116,15 +134,15 @@ jobs: echo "Python version $PYTHON is valid." # Save all information in environment variables - echo "SOFA_BRANCH_NAME=$BRANCH" >> $GITHUB_ENV + echo "SOFA_BRANCH_NAME=${{ github.event.inputs.branch }}" >> $GITHUB_ENV echo "SOFA_COMMIT_SHA=HEAD" >> $GITHUB_ENV - echo "PRESET=full" >> $GITHUB_ENV - echo "PYTHON_VERSION=$PYTHON" >> $GITHUB_ENV + echo "PRESET=${{ github.event.inputs.preset }}" >> $GITHUB_ENV + echo "PYTHON_VERSION=${{ github.event.inputs.python_version }}" >> $GITHUB_ENV echo "GENERATE_BINARIES=true" >> $GITHUB_ENV - echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV - echo "EXTERNAL_PLUGINS=$EXTERNAL_PLUGINS" >> $GITHUB_ENV - echo "ADDITIONNAL_CMAKE_FLAGS=$ADDITIONNAL_CMAKE_FLAGS" >> $GITHUB_ENV + echo 'BUILDER_OS=${{ github.event.inputs.builder-os }}' >> $GITHUB_ENV echo "FORCE_FULL_BUILD=true" >> $GITHUB_ENV + echo "EXTERNAL_PLUGINS=${{ inputs.external-plugins }}" >> $GITHUB_ENV + echo "ADDITIONNAL_CMAKE_FLAGS=${{ inputs.additionnal-cmake-flags }}" >> $GITHUB_ENV - name: Set up python @@ -272,8 +290,8 @@ jobs: echo "CI_DEPENDS_ON=" >> $GITHUB_ENV # CI_DEPENDS_ON: "" echo "WITH_ALL_TESTS=false" >> $GITHUB_ENV # WITH_ALL_TESTS: false echo "FORCE_FULL_BUILD=false" >> $GITHUB_ENV # FORCE_FULL_BUILD: false - echo "EXTERNAL_PLUGINS=" >> $GITHUB_ENV # EXTERNAL_PLUGINS: "" - echo "ADDITIONNAL_CMAKE_FLAGS=" >> $GITHUB_ENV # ADDITIONNAL_CMAKE_FLAGS: "" + echo "EXTERNAL_PLUGINS=" >> $GITHUB_ENV # EXTERNAL_PLUGINS: "" + echo "ADDITIONNAL_CMAKE_FLAGS=" >> $GITHUB_ENV # ADDITIONNAL_CMAKE_FLAGS: "" echo "GENERATE_BINARIES=false" >> $GITHUB_ENV # GENERATE_BINARIES: false echo "PR_OWNER_URL=" >> $GITHUB_ENV # PR_OWNER_URL: "" echo "PR_BRANCH_NAME=" >> $GITHUB_ENV # PR_BRANCH_NAME: "" @@ -297,7 +315,7 @@ jobs: run: | echo "This step runs only for nightly builds." echo "SOFA_BRANCH_NAME=${{ matrix.sofa_version }}" >> $GITHUB_ENV - echo "PRESET=standard" >> $GITHUB_ENV + echo "PRESET=standard-dev" >> $GITHUB_ENV echo "WITH_ALL_TESTS=true" >> $GITHUB_ENV echo "GENERATE_BINARIES=true" >> $GITHUB_ENV echo 'BUILDER_OS=["sh-ubuntu_gcc_release","sh-windows_vs2022_release","sh-macos_clang_release"]' >> $GITHUB_ENV From 5be44765a6a25c331918c7e2936e5c20efb5ed12 Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:52:39 +0200 Subject: [PATCH 56/61] Add test step (#28) * Start work on testin * Temporary change ref in filter build to test this action * Run unit tests even when with-all-tests is not on * Fix test launching * Fix script dir * Update .github/workflows/filter-build.yml Co-authored-by: Hugo * Apply suggestions from code review --------- Co-authored-by: Hugo --- .github/workflows/build-and-test.yml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 740ad2fce31..e7f2d1ad09c 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -214,6 +214,7 @@ jobs: #Need more information. Might be good to - name: Build + id: build-step continue-on-error: true shell: bash run: | @@ -222,6 +223,7 @@ jobs: BUILD_TYPE=$(echo ${{ matrix.os }} | awk -F '_' '{print $3}') CONFIG=$(echo ${{ matrix.os }} | awk -F '_' '{print $1"_"$2}') NODE_NAME="${{ runner.name }}" # Needed by configure-and-build --> TODO: be able to get hostname multiplatform + echo "NODE_NAME=$NODE_NAME" >> $GITHUB_ENV ## Deal with ci-depends-on # Loop over each key-value pair in the JSON avoiding ci repository if inside @@ -288,6 +290,9 @@ jobs: DOCKER_IMAGE="sofaframework/sofabuilder_${builder_type}:${default_tag}" fi + echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> $GITHUB_ENV + + echo "Launching configuration and build through docker using image ${DOCKER_IMAGE}. Running ./ci/scripts/configure-and-build.sh /workspace/build/ /workspace/sofa/ /workspace/ci/scripts/ \"${NODE_NAME}\" \"${BUILD_TYPE}\" \"${CONFIG}\" \"${{ inputs.python-version }}\" \"${{ inputs.preset }}\" \"${{ inputs.generate-binaries }}\" \"${CMAKE_OPTIONS}\"" docker run --rm \ --user $(id -u):$(id -g) --network=host -v $WORKSPACE:/workspace \ @@ -301,24 +306,37 @@ jobs: - name: Notify dashboard + if: always() shell: bash run: | #TODO - name: Launch tests continue-on-error: true - if: steps.Build.outcome == 'success' + if: steps.build-step.outcome == 'success' shell: bash run: | - #TODO - #Use new action with workflow call + # If os has got docker then use it + # Here no need to fetch it because it has already been taken care of in the build step + if [[ "${{ matrix.os }}" == *"ubuntu"* ]] || [[ "${{ matrix.os }}" == *"fedora"* ]]; then + + echo "Launching test suite through docker using image ${DOCKER_IMAGE}. Running ./ci/scripts/test.sh /workspace/build/ /workspace/sofa/ /workspace/ci/ ${NODE_NAME} ${{ inputs.python-version }}" + docker run --rm \ + --user $(id -u):$(id -g) --network=host -v $WORKSPACE:/workspace \ + ${DOCKER_IMAGE} \ + /bin/bash -c "env ; cd /workspace; /bin/bash ./ci/scripts/test.sh /workspace/build/ /workspace/sofa/ /workspace/ci/scripts/ ${NODE_NAME} ${{ inputs.python-version }} ${{ inputs.with-all-tests }}" + else + echo "Launching configuration and build" + + bash ${CI_DIR}/scripts/test.sh "${BUILD_DIR}" "${SRC_DIR}" "${CI_DIR}/scripts/" "${NODE_NAME}" "${{ inputs.python-version }}" "${{ inputs.with-all-tests }}" + fi + - name: Publish artifacts - if: steps.Build.outcome == 'success' && inputs.generate-binaries == true + if: steps.build-step.outcome == 'success' && inputs.generate-binaries == true shell: bash run: | #TODO - #Use new action with workflow call - name: Publish logs if: always() From e95b597f020a71f36eb23aaa6a6b2efc63239838 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Tue, 29 Apr 2025 11:13:38 +0200 Subject: [PATCH 57/61] Identify TODOs to ease final cleaning before merge in SOFA --- .github/workflows/trigger-build-and-tests.yml | 12 +++++++----- scripts/github_CI/checkPRInfoBeforeBuild.py | 7 +++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/trigger-build-and-tests.yml b/.github/workflows/trigger-build-and-tests.yml index e2d3dc0567e..595a0261c1f 100644 --- a/.github/workflows/trigger-build-and-tests.yml +++ b/.github/workflows/trigger-build-and-tests.yml @@ -72,7 +72,8 @@ on: jobs: # Filter build handling : push in master, commits in PR, comments in PR and dispatch filter_build: - if: ${{ github.event_name != 'schedule'}} + if: ${{ github.event_name != 'schedule'}} #TODO_before_PR_in_SOFA - remove + ### if: ${{ github.repository_owner == 'sofa-framework' && github.event_name != 'schedule' }} #TODO_before_PR_in_SOFA - uncomment runs-on: ubuntu-latest outputs: SOFA_BRANCH_NAME: ${{ steps.export-vars.outputs.SOFA_BRANCH_NAME }} @@ -197,7 +198,7 @@ jobs: pr_data=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \ -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/bakpaul/sofa/pulls/$PR_NUMBER") + "https://api.github.com/repos/bakpaul/sofa/pulls/$PR_NUMBER") #TODO_before_PR_in_SOFA - replace with sofa-framework # Extract data OWNER_NAME=$(echo "$pr_data" | jq -r '.user.login') @@ -259,11 +260,11 @@ jobs: # Nightly build triggered once a day nightly_build: runs-on: ubuntu-latest - if: github.event.schedule == '0 2 * * *' + if: github.event.schedule == '0 2 * * *' #TODO_before_PR_in_SOFA - remove + ### if: ${{ github.repository_owner == 'sofa-framework' && github.event.schedule == '0 2 * * *' }} #TODO_before_PR_in_SOFA - uncomment strategy: matrix: sofa_version: [master, v24.12] - ### if: ${{ github.repository_owner == 'sofa-framework' }} outputs: SOFA_BRANCH_NAME: ${{ steps.export-vars.outputs.SOFA_BRANCH_NAME }} SOFA_COMMIT_SHA: ${{ steps.export-vars.outputs.SOFA_COMMIT_SHA }} @@ -347,7 +348,8 @@ jobs: # Trigger the build and sharing all parameters from filter_build > outputs call-workflow-passing-data: needs: filter_build - uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master + ### if: ${{ github.repository_owner == 'sofa-framework' }} #TODO_before_PR_in_SOFA - uncomment + uses: bakpaul/sofa/.github/workflows/build-and-test.yml@master #TODO_before_PR_in_SOFA - replace with sofa-framework with: sofa-branch-name: ${{ needs.filter_build.outputs.SOFA_BRANCH_NAME }} sofa-commit-sha: ${{ needs.filter_build.outputs.SOFA_COMMIT_SHA }} diff --git a/scripts/github_CI/checkPRInfoBeforeBuild.py b/scripts/github_CI/checkPRInfoBeforeBuild.py index 27082fd3cf8..3305cb42def 100644 --- a/scripts/github_CI/checkPRInfoBeforeBuild.py +++ b/scripts/github_CI/checkPRInfoBeforeBuild.py @@ -22,7 +22,8 @@ # GitHub API base URL -API_URL = f"https://api.github.com/repos/bakpaul/sofa" +API_URL = f"https://api.github.com/repos/bakpaul/sofa" #TODO_before_PR_in_SOFA - replace with sofa-framework + # Headers for authentication HEADERS = { @@ -128,9 +129,7 @@ def export_pr_info(): env_file.write(f"PR_OWNER_URL={pr_url}\n") env_file.write(f"PR_BRANCH_NAME={pr_branch_name}\n") env_file.write(f"PR_COMMIT_SHA={pr_commit_sha}\n") - - ## TODO : pr_data.get('mergeable', False) could also let us know if it is mergeable - + # ======================================================================== From 2118d079a88d89de288c99a0fa85d2f89c328cc2 Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Tue, 29 Apr 2025 12:06:41 +0200 Subject: [PATCH 58/61] Add log publishing (#31) * Add log publishing * Apply suggestions from code review --------- Co-authored-by: Hugo --- .github/workflows/build-and-test.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e7f2d1ad09c..35c70510f79 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -326,7 +326,7 @@ jobs: ${DOCKER_IMAGE} \ /bin/bash -c "env ; cd /workspace; /bin/bash ./ci/scripts/test.sh /workspace/build/ /workspace/sofa/ /workspace/ci/scripts/ ${NODE_NAME} ${{ inputs.python-version }} ${{ inputs.with-all-tests }}" else - echo "Launching configuration and build" + echo "Launching test suite" bash ${CI_DIR}/scripts/test.sh "${BUILD_DIR}" "${SRC_DIR}" "${CI_DIR}/scripts/" "${NODE_NAME}" "${{ inputs.python-version }}" "${{ inputs.with-all-tests }}" fi @@ -338,9 +338,18 @@ jobs: run: | #TODO - - name: Publish logs + - name: Publish build logs if: always() - shell: bash - run: | - #TODO - #Use new action with workflow call + uses: actions/upload-artifact@v4 + with: + name: build-logs + path: | + $${{ env.BUILD_DIR }}/build/make-output.txt + + - name: Publish tests logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: tests-logs + path: | + $${{ env.BUILD_DIR }}/build/tests_results/ \ No newline at end of file From a465d8ba88fa8787811f652f69fc8244df610a11 Mon Sep 17 00:00:00 2001 From: hugtalbot Date: Tue, 29 Apr 2025 16:28:30 +0200 Subject: [PATCH 59/61] add forgotten build action --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e7f2d1ad09c..b3c2a4a8bb4 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -141,7 +141,7 @@ jobs: fi else echo "This is not a PR: checking out sha ${{ inputs.sofa-commit-sha }} from branch ${{ inputs.sofa-branch-name }}" - git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/bakpaul/sofa ## TODO : REMOVE THIS WITH SOFA_FRAMEWORK @@@@@@@@@@@@@ + git clone -b ${{ inputs.sofa-branch-name }} --single-branch https://www.github.com/bakpaul/sofa ## TODO_before_PR_in_SOFA : REMOVE THIS WITH SOFA_FRAMEWORK @@@@@@@@@@@@@ cd sofa git checkout ${{ inputs.sofa-commit-sha }} fi @@ -149,7 +149,7 @@ jobs: cd $WORKSPACE ## Clone CI and use ci-depends-on structure - ### TODO: UNCOMMENT THIS WHEN PUSHIN TO REAL SOFA + ### TODO_before_PR_in_SOFA : UNCOMMENT THIS WHEN PUSHIN TO REAL SOFA # ci_branch=${{ inputs.sofa-branch-name }} # ci_repo_url="https://www.github.com/sofa-framework/ci" ### AND REMOVE THOSE From 0636656fe9015fa9114b4d736ec4aaf143b6b93c Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Wed, 30 Apr 2025 12:03:25 +0200 Subject: [PATCH 60/61] Fix log publish --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 35c70510f79..a7b8da2f776 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -344,7 +344,7 @@ jobs: with: name: build-logs path: | - $${{ env.BUILD_DIR }}/build/make-output.txt + $${{ env.BUILD_DIR }}/make-output.txt - name: Publish tests logs if: always() @@ -352,4 +352,4 @@ jobs: with: name: tests-logs path: | - $${{ env.BUILD_DIR }}/build/tests_results/ \ No newline at end of file + $${{ env.BUILD_DIR }}/tests_results/ From c3cb6b0f33c8a7ec389c9b952f2c41c111c72eaf Mon Sep 17 00:00:00 2001 From: Paul Baksic <30337881+bakpaul@users.noreply.github.com> Date: Wed, 30 Apr 2025 17:48:16 +0200 Subject: [PATCH 61/61] Update build-and-test.yml --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a7b8da2f776..f4d1858a6cc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -344,7 +344,7 @@ jobs: with: name: build-logs path: | - $${{ env.BUILD_DIR }}/make-output.txt + ${{ env.BUILD_DIR }}/make-output.txt - name: Publish tests logs if: always() @@ -352,4 +352,4 @@ jobs: with: name: tests-logs path: | - $${{ env.BUILD_DIR }}/tests_results/ + ${{ env.BUILD_DIR }}/tests_results/