Framework: BuildManifest Migration, Firmware Range Refactor, and Admin INFO Resync Tools (#159)

Author: mreid-tt

migrations/versions/30bc0c0455f8_split_build_firmware_and_add_build_.py

@@ -0,0 +1,177 @@
+"""Split Build Firmware and add Build Manifest
+
+Revision ID: 30bc0c0455f8
+Revises: f95855ce9471
+Create Date: 2025-11-05 20:43:32.593098
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "30bc0c0455f8"
+down_revision = "f95855ce9471"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    bind = op.get_bind()
+    meta = sa.MetaData()
+
+    # 1) Add new columns (nullable for backfill), add FKs
+    op.add_column("build", sa.Column("firmware_min_id", sa.Integer(), nullable=True))
+    op.add_column("build", sa.Column("firmware_max_id", sa.Integer(), nullable=True))
+    op.create_foreign_key(None, "build", "firmware", ["firmware_min_id"], ["id"])
+    op.create_foreign_key(None, "build", "firmware", ["firmware_max_id"], ["id"])
+
+    # 2) Backfill firmware_min_id from legacy firmware_id (still present right now)
+    op.execute("UPDATE build SET firmware_min_id = firmware_id")
+
+    # 3) Make firmware_min_id NOT NULL
+    op.alter_column(
+        "build", "firmware_min_id", existing_type=sa.Integer(), nullable=False
+    )
+
+    # 4) Drop old FK and legacy column firmware_id
+    op.drop_constraint(op.f("build_firmware_id_fkey"), "build", type_="foreignkey")
+    op.drop_column("build", "firmware_id")
+
+    # 5) Create buildmanifest table
+    op.create_table(
+        "buildmanifest",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("build_id", sa.Integer(), nullable=False, unique=True),
+        sa.Column("dependencies", sa.Unicode(255)),
+        sa.Column("conf_dependencies", sa.UnicodeText()),
+        sa.Column("conflicts", sa.Unicode(255)),
+        sa.Column("conf_conflicts", sa.UnicodeText()),
+        sa.Column("conf_privilege", sa.UnicodeText()),
+        sa.Column("conf_resource", sa.UnicodeText()),
+        sa.ForeignKeyConstraint(["build_id"], ["build.id"]),
+    )
+
+    # 6) Reflect
+    build = sa.Table("build", meta, autoload_with=bind)
+    version = sa.Table("version", meta, autoload_with=bind)
+    buildmanifest = sa.Table("buildmanifest", meta, autoload_with=bind)
+
+    # 7) Backfill buildmanifest: one row per build using its version’s fields
+    sel = sa.select(
+        build.c.id.label("build_id"),
+        version.c.dependencies,
+        version.c.conf_dependencies,
+        version.c.conflicts,
+        version.c.conf_conflicts,
+        version.c.conf_privilege,
+        version.c.conf_resource,
+    ).select_from(build.join(version, build.c.version_id == version.c.id))
+
+    rows = bind.execute(sel).fetchall()
+    if rows:
+        bind.execute(
+            buildmanifest.insert(),
+            [
+                {
+                    "build_id": r.build_id,
+                    "dependencies": r.dependencies,
+                    "conf_dependencies": r.conf_dependencies,
+                    "conflicts": r.conflicts,
+                    "conf_conflicts": r.conf_conflicts,
+                    "conf_privilege": r.conf_privilege,
+                    "conf_resource": r.conf_resource,
+                }
+                for r in rows
+            ],
+        )
+
+    # 8) Drop moved columns from version
+    op.drop_column("version", "conf_resource")
+    op.drop_column("version", "conf_dependencies")
+    op.drop_column("version", "conf_conflicts")
+    op.drop_column("version", "dependencies")
+    op.drop_column("version", "conf_privilege")
+    op.drop_column("version", "conflicts")
+
+
+def downgrade():
+    bind = op.get_bind()
+    meta = sa.MetaData()
+
+    # 1) Recreate columns on version
+    op.add_column("version", sa.Column("conflicts", sa.Unicode(255)))
+    op.add_column("version", sa.Column("conf_privilege", sa.UnicodeText()))
+    op.add_column("version", sa.Column("dependencies", sa.Unicode(255)))
+    op.add_column("version", sa.Column("conf_conflicts", sa.UnicodeText()))
+    op.add_column("version", sa.Column("conf_dependencies", sa.UnicodeText()))
+    op.add_column("version", sa.Column("conf_resource", sa.UnicodeText()))
+
+    # Reflect
+    version = sa.Table("version", meta, autoload_with=bind)
+    build = sa.Table("build", meta, autoload_with=bind)
+    buildmanifest = sa.Table("buildmanifest", meta, autoload_with=bind)
+
+    # 2) Copy manifest back to version using the first build (lowest build.id) per version
+    first_build_cte = (
+        sa.select(
+            build.c.version_id,
+            sa.func.min(build.c.id).label("first_build_id"),
+        )
+        .group_by(build.c.version_id)
+        .cte("first_build")
+    )
+
+    sel = sa.select(
+        version.c.id.label("version_id"),
+        buildmanifest.c.dependencies,
+        buildmanifest.c.conf_dependencies,
+        buildmanifest.c.conflicts,
+        buildmanifest.c.conf_conflicts,
+        buildmanifest.c.conf_privilege,
+        buildmanifest.c.conf_resource,
+    ).select_from(
+        version.join(first_build_cte, first_build_cte.c.version_id == version.c.id)
+        .join(build, build.c.id == first_build_cte.c.first_build_id)
+        .join(buildmanifest, buildmanifest.c.build_id == build.c.id)
+    )
+
+    rows = bind.execute(sel).fetchall()
+    for r in rows:
+        bind.execute(
+            version.update()
+            .where(version.c.id == r.version_id)
+            .values(
+                dependencies=r.dependencies,
+                conf_dependencies=r.conf_dependencies,
+                conflicts=r.conflicts,
+                conf_conflicts=r.conf_conflicts,
+                conf_privilege=r.conf_privilege,
+                conf_resource=r.conf_resource,
+            )
+        )
+
+    # 3) Drop buildmanifest
+    op.drop_table("buildmanifest")
+
+    # 4) Restore legacy firmware_id and backfill from firmware_min_id
+    op.add_column("build", sa.Column("firmware_id", sa.Integer(), nullable=True))
+
+    op.execute("UPDATE build SET firmware_id = firmware_min_id")
+
+    op.alter_column("build", "firmware_id", existing_type=sa.Integer(), nullable=False)
+    op.create_foreign_key(
+        op.f("build_firmware_id_fkey"), "build", "firmware", ["firmware_id"], ["id"]
+    )
+
+    # 5) Drop FKs on firmware_min_id / firmware_max_id (names may be auto-generated)
+    bind = op.get_bind()
+    insp = sa.inspect(bind)
+    for fk in insp.get_foreign_keys("build"):
+        cols = tuple(fk.get("constrained_columns") or ())
+        if "firmware_min_id" in cols or "firmware_max_id" in cols:
+            op.drop_constraint(fk["name"], "build", type_="foreignkey")
+
+    # 6) Drop the new columns
+    op.drop_column("build", "firmware_max_id")
+    op.drop_column("build", "firmware_min_id")

spkrepo/app.py

@@ -37,6 +37,9 @@ def create_app(config=None, register_blueprints=True, init_admin=True):
     # Enable or disable Flask’s subdomain routing per config
     app.subdomain_matching = app.config.get("SUBDOMAIN_MATCHING", False)
 
+    # Disable strict slashes
+    app.url_map.strict_slashes = False
+
     if config is not None:
         app.config.from_object(config)
 

spkrepo/cli.py

@@ -30,9 +30,21 @@ def create_user(username, email, password):
 @with_appcontext
 def populate_db():
     """Populate the database with some packages."""
-    from spkrepo.models import Architecture
+    from spkrepo.models import Architecture, BuildManifest
     from spkrepo.tests.common import BuildFactory, PackageFactory, VersionFactory
 
+    def attach_manifest(build, *, dependencies=None):
+        """Attach a simple manifest to a build with optional dependencies."""
+        manifest = BuildManifest(
+            dependencies=dependencies,
+            conf_dependencies=None,
+            conflicts=None,
+            conf_conflicts=None,
+            conf_privilege=None,
+            conf_resource=None,
+        )
+        build.buildmanifest = manifest
+
     with db.session.no_autoflush:
         # nzbget
         nzbget_package = PackageFactory(name="nzbget")
@@ -41,7 +53,6 @@ def populate_db():
                 package=nzbget_package,
                 upstream_version="12.0",
                 version=10,
-                dependencies=None,
                 report_url=None,
                 install_wizard=True,
                 upgrade_wizard=False,
@@ -50,15 +61,16 @@ def populate_db():
                 package=nzbget_package,
                 upstream_version="13.0",
                 version=11,
-                dependencies=None,
                 report_url=None,
                 install_wizard=True,
                 upgrade_wizard=False,
             ),
         ]
         nzbget_builds = []
         for version in nzbget_versions:
-            builds = BuildFactory.create_batch(2, version=version, active=True)
+            builds = BuildFactory.create_batch(
+                2, version=version, active=True, buildmanifest=False
+            )
             nzbget_builds.extend(builds)
 
         # sickbeard
@@ -68,8 +80,6 @@ def populate_db():
                 package=sickbeard_package,
                 upstream_version="20140528",
                 version=3,
-                dependencies="git",
-                service_dependencies=[],
                 report_url=None,
                 install_wizard=False,
                 upgrade_wizard=False,
@@ -79,8 +89,6 @@ def populate_db():
                 package=sickbeard_package,
                 upstream_version="20140702",
                 version=4,
-                dependencies="git",
-                service_dependencies=[],
                 report_url=None,
                 install_wizard=False,
                 upgrade_wizard=False,
@@ -94,8 +102,10 @@ def populate_db():
                     version=version,
                     architectures=[Architecture.find("noarch")],
                     active=True,
+                    buildmanifest=False,
                 )
             )
+            attach_manifest(sickbeard_builds[-1], dependencies="git")
 
         # git
         git_package = PackageFactory(name="git")
@@ -104,8 +114,6 @@ def populate_db():
                 package=git_package,
                 upstream_version="1.8.4",
                 version=3,
-                dependencies=None,
-                service_dependencies=[],
                 report_url=None,
                 install_wizard=False,
                 upgrade_wizard=False,
@@ -115,8 +123,6 @@ def populate_db():
                 package=git_package,
                 upstream_version="2.1.2",
                 version=4,
-                dependencies=None,
-                service_dependencies=[],
                 report_url=None,
                 install_wizard=False,
                 upgrade_wizard=False,
@@ -125,7 +131,9 @@ def populate_db():
         ]
         git_builds = []
         for version in git_versions:
-            builds = BuildFactory.create_batch(3, version=version, active=True)
+            builds = BuildFactory.create_batch(
+                3, version=version, active=True, buildmanifest=False
+            )
             git_builds.extend(builds)
 
         # bitlbee
@@ -135,8 +143,6 @@ def populate_db():
                 package=bitlbee_package,
                 upstream_version="3.2.2",
                 version=9,
-                dependencies=None,
-                service_dependencies=[],
                 report_url=None,
                 install_wizard=False,
                 upgrade_wizard=False,
@@ -146,8 +152,6 @@ def populate_db():
                 package=bitlbee_package,
                 upstream_version="3.2.3",
                 version=10,
-                dependencies=None,
-                service_dependencies=[],
                 report_url=None,
                 install_wizard=False,
                 upgrade_wizard=False,
@@ -157,16 +161,16 @@ def populate_db():
                 package=bitlbee_package,
                 upstream_version="3.3.0",
                 version=11,
-                dependencies=None,
-                service_dependencies=[],
                 install_wizard=False,
                 upgrade_wizard=False,
                 startable=True,
             ),
         ]
         bitlbee_builds = []
         for version in bitlbee_versions:
-            builds = BuildFactory.create_batch(3, version=version, active=True)
+            builds = BuildFactory.create_batch(
+                3, version=version, active=True, buildmanifest=False
+            )
             bitlbee_builds.extend(builds)
     db.session.commit()