Refactor release related files and scripts (#483)

* Renames and moves `gen_version_info_rc.py`.
* Harmonizes the naming of version files:
  In IREE a `version.json` file is used to compute a `version_local.json`.
  With this patch, the same is applied here to reduce maintenance efforts.
* Refactors `write_requirements.py`:
  Refactors the script to generate the requirements file in memory and
  write only once.

.github/workflows/build_packages.yml

@@ -37,15 +37,15 @@ jobs:
       - name: Generate release candidate versions
         id: version_rc
         run: |
-          sharktank_package_version=$(python3 build_tools/gen_version_info_rc.py sharktank)
-          shortfin_package_version=$(python3 build_tools/gen_version_info_rc.py shortfin)
-      - name: Upload version_info_rc.json
+          sharktank_package_version=$(python3 build_tools/python_deploy/compute_local_version.py sharktank)
+          shortfin_package_version=$(python3 build_tools/python_deploy/compute_local_version.py shortfin)
+      - name: Upload version_local.json
         uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
-          name: version_info_rc
+          name: version_local
           path: |
-            sharktank/version_info_rc.json
-            shortfin/version_info_rc.json
+            sharktank/version_local.json
+            shortfin/version_local.json
 
   build_packages:
     name: "${{ matrix.package }} :: ${{ matrix.platform }} :: ${{ matrix.python-version }}"
@@ -91,10 +91,10 @@ jobs:
         path: "c" # Windows can hit path length limits, so use a short path.
         submodules: false
 
-    - name: Download version_info_rc.json
+    - name: Download version_local.json
       uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
       with:
-        name: version_info_rc
+        name: version_local
         path: ./c/
         merge-multiple: true
 

.gitignore

@@ -30,6 +30,9 @@ wheelhouse
 *.whl
 *.venv
 
+# Local-only config options
+version_local.json
+
 #Model artifacts
 *.pt
 *.safetensors

build_tools/python_deploy/compute_common_version.py

@@ -36,8 +36,8 @@
 THIS_DIR = Path(__file__).parent.resolve()
 REPO_ROOT = THIS_DIR.parent.parent
 
-VERSION_FILE_SHARKTANK = REPO_ROOT / "sharktank/version_info.json"
-VERSION_FILE_SHORTFIN = REPO_ROOT / "shortfin/version_info.json"
+VERSION_FILE_SHARKTANK = REPO_ROOT / "sharktank/version.json"
+VERSION_FILE_SHORTFIN = REPO_ROOT / "shortfin/version.json"
 VERSION_FILE_LOCAL = REPO_ROOT / "shark-ai/version_local.json"
 
 

build_tools/gen_version_info_rc.py → build_tools/python_deploy/compute_local_version.py

@@ -1,12 +1,13 @@
+#!/usr/bin/env python3
 # Copyright 2024 Advanced Micro Devices, Inc.
 #
 # Licensed under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 # This scripts grabs the X.Y.Z[.dev]` version identifier from a
-# `version_info.json` and writes the corresponding
-# `X.Y.ZrcYYYYMMDD` version identifier to `version_rc_info.json`.
+# `version.json` and writes the corresponding
+# `X.Y.ZrcYYYYMMDD` version identifier to `version_local.json`.
 
 import argparse
 from pathlib import Path
@@ -20,29 +21,31 @@
 parser.add_argument("path", type=Path)
 args = parser.parse_args()
 
-VERSION_INFO_FILE = args.path / "version_info.json"
-VERSION_INFO_RC_FILE = args.path / "version_info_rc.json"
+VERSION_FILE = args.path / "version.json"
+VERSION_FILE_LOCAL = args.path / "version_local.json"
 
 
 def load_version_info():
-    with open(VERSION_INFO_FILE, "rt") as f:
+    with open(VERSION_FILE, "rt") as f:
         return json.load(f)
 
 
 def write_version_info():
-    with open(VERSION_INFO_RC_FILE, "w") as f:
-        json.dump(version_info_rc, f, indent=2)
+    with open(VERSION_FILE_LOCAL, "w") as f:
+        json.dump(version_local, f, indent=2)
         f.write("\n")
 
 
 version_info = load_version_info()
 
 PACKAGE_VERSION = version_info.get("package-version")
 PACKAGE_BASE_VERSION = Version(PACKAGE_VERSION).base_version
-PACKAGE_RC_VERSION = PACKAGE_BASE_VERSION + "rc" + datetime.today().strftime("%Y%m%d")
+PACKAGE_LOCAL_VERSION = (
+    PACKAGE_BASE_VERSION + "rc" + datetime.today().strftime("%Y%m%d")
+)
 
-version_info_rc = {"package-version": PACKAGE_RC_VERSION}
+version_local = {"package-version": PACKAGE_LOCAL_VERSION}
 
 write_version_info()
 
-print(PACKAGE_RC_VERSION)
+print(PACKAGE_LOCAL_VERSION)

build_tools/python_deploy/write_requirements.py

@@ -33,8 +33,8 @@
 THIS_DIR = Path(__file__).parent.resolve()
 REPO_ROOT = THIS_DIR.parent.parent
 
-VERSION_FILE_SHARKTANK = REPO_ROOT / "sharktank/version_info.json"
-VERSION_FILE_SHORTFIN = REPO_ROOT / "shortfin/version_info.json"
+VERSION_FILE_SHARKTANK = REPO_ROOT / "sharktank/version_local.json"
+VERSION_FILE_SHORTFIN = REPO_ROOT / "shortfin/version_local.json"
 VERSION_FILE_LOCAL = REPO_ROOT / "shark-ai/version_local.json"
 REQUIREMENTS_TXT = REPO_ROOT / "shark-ai/requirements.txt"
 
@@ -44,18 +44,9 @@ def load_version_info(version_file):
         return json.load(f)
 
 
-def write_requirements(package_list, package_version):
+def write_requirements(requirements):
     with open(REQUIREMENTS_TXT, "w") as f:
-        for package in package_list:
-            PINNED_PACKAGE = package + "==" + package_version
-            f.write("%s\n" % PINNED_PACKAGE)
-
-
-def append_requirements(package_list, package_version):
-    with open(REQUIREMENTS_TXT, "a") as f:
-        for package in package_list:
-            PINNED_PACKAGE = package + "==" + package_version
-            f.write("%s\n" % PINNED_PACKAGE)
+        f.write("%s\n" % requirements)
 
 
 metapackage_version = load_version_info(VERSION_FILE_LOCAL)
@@ -70,20 +61,34 @@ def append_requirements(package_list, package_version):
 stable_packages_list = ["iree-base-compiler", "iree-base-runtime", "iree-turbine"]
 
 if Version(PACKAGE_VERSION).is_prerelease:
-    write_requirements(
-        ["sharktank"],
-        Version(SHARKTANK_PACKAGE_VERSION).base_version + "rc" + args.version_suffix,
+    requirements = (
+        "sharktank=="
+        + Version(SHARKTANK_PACKAGE_VERSION).base_version
+        + "rc"
+        + args.version_suffix
+        + "\n"
     )
-    append_requirements(
-        ["shortfin"],
-        Version(SHORTFIN_PACKAGE_VERSION).base_version + "rc" + args.version_suffix,
+    requirements += (
+        "shortfin=="
+        + Version(SHORTFIN_PACKAGE_VERSION).base_version
+        + "rc"
+        + args.version_suffix
     )
+
+    write_requirements(requirements)
+
 else:
     MAJOR_VERSION = Version(PACKAGE_VERSION).major
     MINOR_VERSION = Version(PACKAGE_VERSION).minor
 
-    write_requirements(
-        stable_packages_list, str(MAJOR_VERSION) + "." + str(MINOR_VERSION) + ".*"
+    STABLE_VERSION_TO_PIN = str(MAJOR_VERSION) + "." + str(MINOR_VERSION) + ".*"
+
+    requirements = ""
+    for package in stable_packages_list:
+        requirements += package + "==" + STABLE_VERSION_TO_PIN + "\n"
+    requirements += (
+        "sharktank==" + Version(SHARKTANK_PACKAGE_VERSION).base_version + "\n"
     )
-    append_requirements(["sharktank"], Version(SHARKTANK_PACKAGE_VERSION).base_version)
-    append_requirements(["shortfin"], Version(SHORTFIN_PACKAGE_VERSION).base_version)
+    requirements += "shortfin==" + Version(SHORTFIN_PACKAGE_VERSION).base_version
+
+    write_requirements(requirements)

shark-ai/.gitignore

@@ -1,3 +1,2 @@
 # Local-only config options
-version_local.json
 requirements.txt

sharktank/setup.py

@@ -13,8 +13,8 @@
 SETUPPY_DIR = os.path.realpath(os.path.dirname(__file__))
 
 # Setup and get version information.
-VERSION_INFO_FILE = os.path.join(SETUPPY_DIR, "version_info.json")
-VERSION_INFO_RC_FILE = os.path.join(SETUPPY_DIR, "version_info_rc.json")
+VERSION_FILE = os.path.join(SETUPPY_DIR, "version.json")
+VERSION_FILE_LOCAL = os.path.join(SETUPPY_DIR, "version_local.json")
 
 
 def load_version_info(version_file):
@@ -23,10 +23,10 @@ def load_version_info(version_file):
 
 
 try:
-    version_info = load_version_info(VERSION_INFO_RC_FILE)
+    version_info = load_version_info(VERSION_FILE_LOCAL)
 except FileNotFoundError:
-    print("version_info_rc.json not found. Default to dev build")
-    version_info = load_version_info(VERSION_INFO_FILE)
+    print("version_local.json not found. Default to dev build")
+    version_info = load_version_info(VERSION_FILE)
 
 PACKAGE_VERSION = version_info.get("package-version")
 print(f"Using PACKAGE_VERSION: '{PACKAGE_VERSION}'")

shortfin/CMakeLists.txt

@@ -14,7 +14,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR)
 endif()
 
 # Get version number from file
-file(READ ${CMAKE_CURRENT_SOURCE_DIR}/version_info.json VERSION_JSON_STRING)
+file(READ ${CMAKE_CURRENT_SOURCE_DIR}/version.json VERSION_JSON_STRING)
 string(JSON PACKAGE_VERSION GET ${VERSION_JSON_STRING} package-version)
 string(REGEX MATCH "(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*" BASE_VERSION ${PACKAGE_VERSION})
 

shortfin/setup.py

@@ -141,8 +141,8 @@ def copy_extensions_to_source(self, *args, **kwargs):
 
 
 # Setup and get version information.
-VERSION_INFO_FILE = os.path.join(REL_SOURCE_DIR, "version_info.json")
-VERSION_INFO_RC_FILE = os.path.join(REL_SOURCE_DIR, "version_info_rc.json")
+VERSION_FILE = os.path.join(REL_SOURCE_DIR, "version.json")
+VERSION_FILE_LOCAL = os.path.join(REL_SOURCE_DIR, "version_local.json")
 
 
 def load_version_info(version_file):
@@ -151,10 +151,10 @@ def load_version_info(version_file):
 
 
 try:
-    version_info = load_version_info(VERSION_INFO_RC_FILE)
+    version_info = load_version_info(VERSION_FILE_LOCAL)
 except FileNotFoundError:
-    print("version_info_rc.json not found. Default to dev build")
-    version_info = load_version_info(VERSION_INFO_FILE)
+    print("version_local.json not found. Default to dev build")
+    version_info = load_version_info(VERSION_FILE)
 
 PACKAGE_VERSION = version_info.get("package-version")
 print(f"Using PACKAGE_VERSION: '{PACKAGE_VERSION}'")