Skip to content

Commit 8b104c0

Browse files
evertlammertsevertlammerts
authored
Create a PR to bump submodule if nightly is stale (#159)
2 parents 5442569 + 4746ac8 commit 8b104c0

File tree

2 files changed

+173
-16
lines changed

2 files changed

+173
-16
lines changed

‎.github/workflows/release.yml‎

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ on:
2222
options:
2323
- test
2424
- prod
25+
nightly-stale-after-days:
26+
type: string
27+
description: After how many days should nightlies be considered stale
28+
required: true
29+
default: 3
2530
store-s3:
2631
type: boolean
2732
description: Also store test packages in S3 (always true for prod)
@@ -41,6 +46,17 @@ jobs:
4146
duckdb-sha: ${{ inputs.duckdb-sha }}
4247
set-version: ${{ inputs.stable-version }}
4348

49+
submodule_pr:
50+
name: Create or update PR to bump submodule to given SHA
51+
needs: build_sdist
52+
uses: ./.github/workflows/submodule_auto_pr.yml
53+
with:
54+
duckdb-python-sha: ${{ inputs.duckdb-python-sha }}
55+
duckdb-sha: ${{ inputs.duckdb-sha }}
56+
secrets:
57+
# reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206
58+
DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
59+
4460
workflow_state:
4561
name: Set state for the release workflow
4662
needs: build_sdist
@@ -51,23 +67,36 @@ jobs:
5167
runs-on: ubuntu-latest
5268
steps:
5369
- id: index_check
54-
name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI
70+
name: Check version on PyPI
5571
run: |
56-
set -eu
57-
# Check PyPI whether the release we're building is already present
72+
set -ex
5873
pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org
59-
pkg_version=${{ needs.build_sdist.outputs.package-version }}
60-
url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json
61-
http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? )
62-
if [[ $http_status == "200" ]]; then
63-
echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}"
64-
pypi_state=VERSION_FOUND
65-
elif [[ $http_status == 000* ]]; then
66-
echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}"
67-
pypi_state=UNKNOWN
68-
else
69-
echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})"
74+
# install duckdb
75+
curl https://install.duckdb.org | sh
76+
# query pypi
77+
result=$(cat <<EOF | ${HOME}/.duckdb/cli/latest/duckdb | xargs
78+
---- Output lines
79+
.mode line
80+
---- Query that fetches the given version's age, if the version already exists
81+
SELECT
82+
today() - (file.value->>'upload_time_iso_8601')::DATE AS age,
83+
FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd
84+
CROSS JOIN json_each(jd.releases) AS rel(key, value)
85+
CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]')) AS file(value)
86+
WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}'
87+
LIMIT 1;
88+
EOF
89+
)
90+
if [ -z "$result" ]; then
7091
pypi_state=VERSION_NOT_FOUND
92+
else
93+
pypi_state=VERSION_FOUND
94+
fi
95+
if [[ -z "${{ inputs.stable-version }}" ]]; then
96+
age=${result#age = }
97+
if [ "${age}" -ge "${{ inputs.nightly-stale-after-days }}" ]; then
98+
echo "::warning title=Stale nightly for ${{ github.ref_name }}::Nightly is ${age} days old (max=${{ inputs.nightly-stale-after-days }})"
99+
fi
71100
fi
72101
echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT
73102
@@ -96,7 +125,7 @@ jobs:
96125
echo "::notice::S3 upload disabled in inputs, not generating S3 URL"
97126
exit 0
98127
fi
99-
if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then
128+
if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then
100129
echo "::warning::S3 upload disabled because package version already uploaded to PyPI"
101130
exit 0
102131
fi
@@ -110,7 +139,7 @@ jobs:
110139
build_wheels:
111140
name: Build and test releases
112141
needs: workflow_state
113-
if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }}
142+
if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }}
114143
uses: ./.github/workflows/packaging_wheels.yml
115144
with:
116145
minimal: false

‎.github/workflows/submodule_auto_pr.yml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
name: Submodule Auto PR
2+
on:
3+
workflow_call:
4+
inputs:
5+
duckdb-python-sha:
6+
type: string
7+
description: The commit to build against (defaults to latest commit of current ref)
8+
required: false
9+
duckdb-sha:
10+
type: string
11+
description: The DuckDB submodule commit or ref to build against
12+
required: true
13+
auto-land:
14+
type: boolean
15+
description: Immediately merge the PR (placeholder - doesn't work)
16+
default: false
17+
secrets:
18+
DUCKDBLABS_BOT_TOKEN:
19+
description: Github token of the DuckDBLabs bot
20+
required: true
21+
22+
defaults:
23+
run:
24+
shell: bash
25+
26+
jobs:
27+
create_pr:
28+
name: Create PR to bump duckdb submodule to given SHA
29+
runs-on: ubuntu-latest
30+
steps:
31+
- name: Checkout DuckDB Python
32+
uses: actions/checkout@v4
33+
with:
34+
ref: ${{ inputs.duckdb-python-sha }}
35+
fetch-depth: 0
36+
submodules: true
37+
38+
- name: Checkout or Create Needed Branch
39+
run: |
40+
git fetch --all
41+
head_sha=${{ inputs.duckdb-python-sha }}
42+
branch_name="vendoring-${{ github.ref_name }}"
43+
if [[ `git rev-parse --verify ${branch_name} 2>/dev/null` ]]; then
44+
# branch exists
45+
git checkout ${branch_name}
46+
else
47+
# new branch
48+
git checkout -b ${branch_name}
49+
fi
50+
[[ ${head_sha} ]] && git reset --hard ${head_sha} || true
51+
52+
- name: Checkout DuckDB at Given SHA
53+
run: |
54+
cd external/duckdb
55+
git fetch origin
56+
git checkout ${{ inputs.duckdb-sha }}
57+
58+
- name: Determine GH PR Command
59+
id: gh_pr_command
60+
env:
61+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
62+
run: |
63+
pr_url=$( gh pr list --head vendoring-${{ github.ref_name }} --state open --json url --jq '.[].url' )
64+
if [[ $pr_url ]]; then
65+
echo "::notice::Found existing pr, will edit (${pr_url})"
66+
gh_command="edit ${pr_url}"
67+
else
68+
echo "::notice::No existing PR, will create new"
69+
gh_command="create --head vendoring-${{ github.ref_name }} --base ${{ github.ref_name }}"
70+
fi
71+
echo "subcommand=${gh_command}" >> $GITHUB_OUTPUT
72+
73+
- name: Set Git User
74+
run: |
75+
git config --global user.email "[email protected]"
76+
git config --global user.name "DuckDB Labs GitHub Bot"
77+
78+
- name: Create PR to Bump DuckDB Submodule
79+
env:
80+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
81+
run: |
82+
# First commit and push
83+
git add external/duckdb
84+
git commit -m "Bump submodule"
85+
git push --force origin vendoring-${{ github.ref_name }}
86+
# create PR msg
87+
echo "Bump duckdb submodule:" > body.txt
88+
echo "- Target branch: ${{ github.ref_name }}" >> body.txt
89+
echo "- Date: $( date +"%Y-%m-%d %H:%M:%S" )" >> body.txt
90+
echo "- DuckDB SHA: ${{ inputs.duckdb-sha }}" >> body.txt
91+
echo "- Trigger: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt
92+
subcommand="${{ steps.gh_pr_command.outputs.subcommand }}"
93+
gh pr ${subcommand} \
94+
--title "[duckdb-labs bot] Bump DuckDB submodule" \
95+
--body-file body.txt > output.txt 2>&1
96+
success=$?
97+
# Show summary
98+
url=$( [[ $success ]] && gh pr view vendoring-${{ github.ref_name }} --json url --jq .url || true )
99+
echo "## Submodule PR Summary" >> $GITHUB_STEP_SUMMARY
100+
if [[ $success ]]; then
101+
prefix=$( [[ $subcommand == edit* ]] && echo "Created" || echo "Updated" )
102+
echo "### ${prefix} PR: [${url}](${url})" >> $GITHUB_STEP_SUMMARY
103+
else
104+
echo "### Failed to create PR" >> $GITHUB_STEP_SUMMARY
105+
fi
106+
echo '```' >> $GITHUB_STEP_SUMMARY
107+
cat output.txt >> $GITHUB_STEP_SUMMARY
108+
echo '```' >> $GITHUB_STEP_SUMMARY
109+
[[ $success ]] || exit 1
110+
111+
- name: Automerge PR
112+
if: ${{ inputs.auto-land }}
113+
env:
114+
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
115+
run: |
116+
# PLACEHOLDER: DUCKDBLABS_BOT_TOKEN DOES NOT HAVE PERMISSIONS TO MERGE PRS
117+
set -ex
118+
gh pr merge vendoring-${{ github.ref_name }} --rebase > output.txt
119+
success=$?
120+
# Show summary
121+
if [[ $success ]]; then
122+
echo "### PR merged" >> $GITHUB_STEP_SUMMARY
123+
else
124+
echo "### Failed to auto-merge PR" >> $GITHUB_STEP_SUMMARY
125+
fi
126+
echo '```' >> $GITHUB_STEP_SUMMARY
127+
cat output.txt >> $GITHUB_STEP_SUMMARY
128+
echo '```' >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)