From 5c29a7c1d7ce55d8adfe629733fc4a16ece1f41a Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 21 May 2026 16:26:41 -0400 Subject: [PATCH 1/7] feat: import from production ship it instance instead of staging Staging data is a mess; most testing (eg: of product details rebuilds) benefits more from production data. --- api/src/shipit_api/admin/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/shipit_api/admin/cli.py b/api/src/shipit_api/admin/cli.py index f45ce41ea..b5b03e92d 100644 --- a/api/src/shipit_api/admin/cli.py +++ b/api/src/shipit_api/admin/cli.py @@ -114,7 +114,7 @@ def get_taskcluster_headers(request_url, method, content, taskcluster_client_id, @click.command(name="shipit-import") -@click.option("--api-from", default="https://api.shipit.staging.mozilla-releng.net") +@click.option("--api-from", default="https://shipit-api.mozilla-releng.net") @flask.cli.with_appcontext def shipit_import(api_from): configure_logging() From 4927926a7fb14bb32d84b71b8a4b85726a795d79 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 21 May 2026 16:25:10 -0400 Subject: [PATCH 2/7] fix: only fetch shipped releases during import Importing aborted and scheduled releases is really not going to help testing, and it seems to make the production backend return 502s more often than not. --- api/src/shipit_api/admin/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/shipit_api/admin/cli.py b/api/src/shipit_api/admin/cli.py index b5b03e92d..cab09f083 100644 --- a/api/src/shipit_api/admin/cli.py +++ b/api/src/shipit_api/admin/cli.py @@ -121,7 +121,7 @@ def shipit_import(api_from): session = flask.current_app.db.session click.echo("Fetching release list...", nl=False) - req = requests.get(f"{api_from}/releases?status=shipped,aborted,scheduled") + req = requests.get(f"{api_from}/releases?status=shipped") req.raise_for_status() releases = req.json() From 7f10bc29b9dd79ae5eaba495bfa397c8728543f4 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 21 May 2026 16:32:53 -0400 Subject: [PATCH 3/7] fix: db session setup in shipit-import command This currently fails with an error about not being able to find the Flask app. --- api/src/shipit_api/admin/cli.py | 50 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/api/src/shipit_api/admin/cli.py b/api/src/shipit_api/admin/cli.py index cab09f083..889e9a5f2 100644 --- a/api/src/shipit_api/admin/cli.py +++ b/api/src/shipit_api/admin/cli.py @@ -13,14 +13,13 @@ import aiohttp import backoff import click -import flask -import flask.cli import mohawk import requests import sqlalchemy import sqlalchemy.orm from backend_common.log import configure_logging +from shipit_api.admin.flask import flask_app from shipit_api.admin.product_details import rebuild from shipit_api.common.config import BREAKPOINT_VERSION from shipit_api.common.models import Release @@ -115,31 +114,32 @@ def get_taskcluster_headers(request_url, method, content, taskcluster_client_id, @click.command(name="shipit-import") @click.option("--api-from", default="https://shipit-api.mozilla-releng.net") -@flask.cli.with_appcontext def shipit_import(api_from): configure_logging() - session = flask.current_app.db.session - - click.echo("Fetching release list...", nl=False) - req = requests.get(f"{api_from}/releases?status=shipped") - req.raise_for_status() - releases = req.json() - - for release in releases: - r = Release( - product=release["product"], - version=release["version"], - branch=release["branch"], - revision=release["revision"], - build_number=release["build_number"], - release_eta=release.get("release_eta"), - partial_updates=release.get("partials"), - status=release["status"], - ) - r.created = release["created"] - r.completed = release["completed"] or release["created"] - session.add(r) - session.commit() + + with flask_app.app_context(): + session = flask_app.db.session + + click.echo("Fetching release list...", nl=False) + req = requests.get(f"{api_from}/releases?status=shipped") + req.raise_for_status() + releases = req.json() + + for release in releases: + r = Release( + product=release["product"], + version=release["version"], + branch=release["branch"], + revision=release["revision"], + build_number=release["build_number"], + release_eta=release.get("release_eta"), + partial_updates=release.get("partials"), + status=release["status"], + ) + r.created = release["created"] + r.completed = release["completed"] or release["created"] + session.add(r) + session.commit() @click.command(name="trigger-product-details") From 493e0147558ec77081910e4452dc511e13bf7d3b Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 21 May 2026 16:41:05 -0400 Subject: [PATCH 4/7] feat: print release names as they are imported --- api/src/shipit_api/admin/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/shipit_api/admin/cli.py b/api/src/shipit_api/admin/cli.py index 889e9a5f2..d77ebed32 100644 --- a/api/src/shipit_api/admin/cli.py +++ b/api/src/shipit_api/admin/cli.py @@ -120,12 +120,14 @@ def shipit_import(api_from): with flask_app.app_context(): session = flask_app.db.session - click.echo("Fetching release list...", nl=False) + click.echo("Fetching release list...") req = requests.get(f"{api_from}/releases?status=shipped") req.raise_for_status() releases = req.json() for release in releases: + click.echo(f"Importing {release['name']}") + r = Release( product=release["product"], version=release["version"], From 54e5046fc10b0d64be05189af6c89b8f6299c096 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 21 May 2026 16:51:11 -0400 Subject: [PATCH 5/7] add support for importing versions --- api/src/shipit_api/admin/cli.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/api/src/shipit_api/admin/cli.py b/api/src/shipit_api/admin/cli.py index d77ebed32..e81ed7842 100644 --- a/api/src/shipit_api/admin/cli.py +++ b/api/src/shipit_api/admin/cli.py @@ -22,7 +22,7 @@ from shipit_api.admin.flask import flask_app from shipit_api.admin.product_details import rebuild from shipit_api.common.config import BREAKPOINT_VERSION -from shipit_api.common.models import Release +from shipit_api.common.models import Release, Version def coroutine(f): @@ -143,6 +143,23 @@ def shipit_import(api_from): session.add(r) session.commit() + # Import Versions + # only import the two entries we need to rebuild product details for now + click.echo("Importing Version information...") + for product, channel in (("firefox", "nightly"), ("thunderbird", "nightly")): + click.echo(f"Importing {product} {channel} Version information") + req = requests.get(f"{api_from}/versions/{product}/{channel}") + req.raise_for_status() + version = req.json() + + if v := session.query(Version).filter(Version.product_name == product, Version.product_channel == channel).first(): + v.current_version = version + else: + v = Version(product_name=product, product_channel=channel, current_version=version) + + session.add(v) + session.commit() + @click.command(name="trigger-product-details") @click.option("--base-url", default="https://shipit-api.mozilla-releng.net") From 2d873bc53721eaf7122302ddfda85245c7848fa3 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 21 May 2026 16:38:20 -0400 Subject: [PATCH 6/7] feat: add support for limiting the number of releases imported Importing everything takes ages, and in most cases is unnecessary. --- api/src/shipit_api/admin/cli.py | 55 +++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/api/src/shipit_api/admin/cli.py b/api/src/shipit_api/admin/cli.py index e81ed7842..93ff0faac 100644 --- a/api/src/shipit_api/admin/cli.py +++ b/api/src/shipit_api/admin/cli.py @@ -9,6 +9,7 @@ import json import os import typing +from datetime import datetime import aiohttp import backoff @@ -114,34 +115,42 @@ def get_taskcluster_headers(request_url, method, content, taskcluster_client_id, @click.command(name="shipit-import") @click.option("--api-from", default="https://shipit-api.mozilla-releng.net") -def shipit_import(api_from): +@click.option("--limit-releases", default=100, help="Only import N releases. Use -1 to import all releases, 0 to skip importing releases entirely.") +def shipit_import(api_from, limit_releases): configure_logging() with flask_app.app_context(): session = flask_app.db.session - click.echo("Fetching release list...") - req = requests.get(f"{api_from}/releases?status=shipped") - req.raise_for_status() - releases = req.json() - - for release in releases: - click.echo(f"Importing {release['name']}") - - r = Release( - product=release["product"], - version=release["version"], - branch=release["branch"], - revision=release["revision"], - build_number=release["build_number"], - release_eta=release.get("release_eta"), - partial_updates=release.get("partials"), - status=release["status"], - ) - r.created = release["created"] - r.completed = release["completed"] or release["created"] - session.add(r) - session.commit() + if limit_releases != 0: + # Import releases + click.echo("Fetching release list...") + req = requests.get(f"{api_from}/releases?status=shipped") + req.raise_for_status() + releases = req.json() + # Pull the most recent N releases when limiting; they're more important + # for testing in most cases. + releases.sort(key=lambda r: datetime.fromisoformat(r["created"]), reverse=True) + if limit_releases == -1: + limit_releases = releases.len() + + for release in releases[:limit_releases]: + click.echo(f"Importing {release['name']}") + + r = Release( + product=release["product"], + version=release["version"], + branch=release["branch"], + revision=release["revision"], + build_number=release["build_number"], + release_eta=release.get("release_eta"), + partial_updates=release.get("partials"), + status=release["status"], + ) + r.created = release["created"] + r.completed = release["completed"] or release["created"] + session.add(r) + session.commit() # Import Versions # only import the two entries we need to rebuild product details for now From a2b68bcd341af173c0d398e58d2365e34630b987 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 21 May 2026 16:44:25 -0400 Subject: [PATCH 7/7] fix: don't import releases that already exist This makes imports idempotent. --- api/src/shipit_api/admin/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/shipit_api/admin/cli.py b/api/src/shipit_api/admin/cli.py index 93ff0faac..9f9406d8d 100644 --- a/api/src/shipit_api/admin/cli.py +++ b/api/src/shipit_api/admin/cli.py @@ -135,6 +135,10 @@ def shipit_import(api_from, limit_releases): limit_releases = releases.len() for release in releases[:limit_releases]: + if session.query(Release).filter(Release.name == release["name"]).first(): + click.echo(f"{release['name']} already exists, skipping...") + continue + click.echo(f"Importing {release['name']}") r = Release(