tools: Add scripts and supporting files to generate os packge repo

.github/workflows/generate-osrepo-site.yml

@@ -0,0 +1,81 @@
+name: generate-site
+on:
+  workflow_dispatch:
+
+jobs:
+  gen-site:
+    runs-on: ubuntu-latest
+    env:
+      SiteRoot: _site
+
+    name: Generate index and site assets
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install packaging tools
+        # RPM tools are available on ubuntu
+        run: sudo apt install -y rpm gpg createrepo-c dpkg-dev reprepro
+
+      - name: Setup requirements
+        working-directory: tools/packaging/osrepos
+        run: pip install -r requirements.txt
+
+      - name: Download packages
+        working-directory: tools/packaging/osrepos
+        run: python scripts/fetch-releases.py $SiteRoot
+
+      - name: Import GPG key
+        id: gpg-import
+        uses: crazy-max/ghaction-import-gpg@v6
+        with:
+          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+
+      - name: Sign RPMs
+        shell: sh
+        working-directory: tools/packaging/osrepos
+        run: sh scripts/sign-rpms.sh ${{ steps.gpg-import.outputs.fingerprint }}
+
+      - name: Create YUM repository
+        # Creates metadata for YUM/DNF repository, the files were copied in the download step
+        shell: sh
+        working-directory: tools/packaging/osrepos
+        run: createrepo_c -v $SiteRoot/rpm
+
+      - name: Sign YUM repository
+        shell: sh
+        working-directory: tools/packaging/osrepos
+        run: gpg --armor --detach-sign $SiteRoot/rpm/repodata/repomd.xml
+
+      - name: Create APT repository
+        # The configuration for apt repo is in tools/packaging/osrepos/reprepro-config,
+        # which ensures the same GPG key used elsewhere in this action is used to sign
+        # the repository
+        shell: sh
+        working-directory: tools/packaging/osrepos
+        run: sh -x scripts/generate-apt-repo.sh
+
+      - name: Prepare assets
+        working-directory: tools/packaging/osrepos
+        run: |
+          cp -aRv dragonfly.repo pgp-key.public dragonfly.sources $SiteRoot/
+          rm -rf $SiteRoot/deb/conf
+
+      - name: Generate Directory Listings
+        working-directory: tools/packaging/osrepos
+        run: python scripts/generate-index.py $SiteRoot
+
+      - name: Authenticate
+        uses: 'google-github-actions/auth@v3'
+        with:
+          project_id: 'dragonflydb'
+          credentials_json: ${{ secrets.GCP_BUCKET_CREDENTIALS }}
+
+      - name: GCloud setup
+        uses: 'google-github-actions/setup-gcloud@v3'
+
+      - name: Deploy site
+        working-directory: tools/packaging/osrepos
+        run: |
+          gcloud storage rm ${{ secrets.GCP_PACKAGES_BUCKET }}/**
+          gcloud storage rsync $SiteRoot ${{ secrets.GCP_PACKAGES_BUCKET }} --recursive --delete-unmatched-destination-objects

tests/dragonfly/conftest.py

@@ -433,10 +433,11 @@ def pytest_runtest_makereport(item, call):
     if report.when == "teardown":
         call_outcome = getattr(item, "call_outcome", None)
         log_dir = item.funcargs.get("df_log_dir")
-        if report.failed:
-            copy_failed_logs(log_dir, report)
-        if call_outcome and call_outcome.failed:
-            copy_failed_logs(log_dir, call_outcome)
+        if log_dir:
+            if report.failed:
+                copy_failed_logs(log_dir, report)
+            if call_outcome and call_outcome.failed:
+                copy_failed_logs(log_dir, call_outcome)
 
 
 @pytest.fixture(scope="function")

tests/dragonfly/package_install_test.py

@@ -0,0 +1,33 @@
+from testcontainers.core.container import DockerContainer
+
+
+def check(container: DockerContainer, cmd: str):
+    result = container.exec(cmd)
+    assert result.exit_code == 0, f"command {cmd} failed with result {result}"
+
+
+async def test_install_package_on_fedora():
+    with DockerContainer(image="fedora:latest", tty=True) as fedora:
+        check(
+            fedora,
+            "dnf config-manager addrepo --from-repofile=https://packages.dragonflydb.io/dragonfly.repo",
+        )
+        check(fedora, "dnf -y install dragonfly")
+        check(fedora, "dragonfly --version")
+
+
+async def test_install_package_on_ubuntu():
+    with DockerContainer(image="ubuntu:latest", tty=True) as ubuntu:
+        check(ubuntu, "apt update")
+        check(ubuntu, "apt install -y curl")
+        check(
+            ubuntu,
+            "curl -Lo /usr/share/keyrings/dragonfly-keyring.public https://packages.dragonflydb.io/pgp-key.public",
+        )
+        check(
+            ubuntu,
+            "curl -Lo /etc/apt/sources.list.d/dragonfly.sources https://packages.dragonflydb.io/dragonfly.sources",
+        )
+        check(ubuntu, "apt update")
+        check(ubuntu, "apt install -y dragonfly")
+        check(ubuntu, "dragonfly --version")

tools/packaging/osrepos/README.md

@@ -0,0 +1,74 @@
+# Package repositories for rpm and debian packages
+
+This directory contains scripts and definitions for setting up YUM and apt repositories for Linux users to install
+dragonfly packages.
+
+The repositories are served as static websites. The generate-site workflow is used to set up and deploy the sites using
+scripts and definitions included here.
+
+The workflow does the following tasks:
+
+* Download the latest 5 releases from dragonfly releases page, specifically deb and rpm assets
+    * for deb files, only the latest package is downloaded and present (see note below)
+* Set up a directory structure separating deb and rpm files into version specific paths
+* Sign the packages (see note on GPG)
+* Deploy the assets prepared, along with the public GPG key and repo definitions for apt and rpm tooling
+
+## Using the YUM repository
+
+Add the repository using:
+
+```shell
+sudo dnf config-manager addrepo --from-repofile=https://packages.dragonflydb.io/dragonfly.repo
+```
+
+Then install dragonfly as usual, or a specific version:
+
+```shell
+sudo dnf -y install dragonfly-0:v1.33.1-1.fc30.x86_64
+```
+
+## Using the APT repository
+
+First download the public GPG key to an appropriate location:
+
+```shell
+sudo curl -Lo /usr/share/keyrings/dragonfly-keyring.public https://packages.dragonflydb.io/pgp-key.public
+```
+
+Then add the sources file:
+
+```shell
+sudo curl -Lo /etc/apt/sources.list.d/dragonfly.sources https://packages.dragonflydb.io/dragonfly.sources
+```
+
+Finally install dragonfly using apt
+
+```shell
+sudo apt update && sudo apt install dragonfly
+```
+
+#### Versions in APT repository
+
+Unlike the yum repo, the apt repo only has the latest version. The reason for this is the tool, `reprepro` supplied by
+debian to build repositories only supports multiple
+versions in version 5.4 onwards, and the github runner using ubuntu-latest does not have this version.
+
+Another option would be to use the components feature of apt repositories in the sources file we ask users to install,
+but then the versions would need
+to be hardcoded in the sources file and the user would have
+to update the file with each new release which makes for a bad user experience. As of now users wanting older packages
+should download them directly.
+
+### Signing packages
+
+The packages are signed using the GPG key imported from the secret GPG_PRIVATE_KEY in this repository.
+
+The corresponding public key is served with site assets, so the apt/yum/dnf based tooling can consume the public key to
+verify package integrity.
+
+### TODO
+
+- [X] debian packages signing (not required? release file is signed)
+- [X] debian repo metadata setup
+- [ ] tests asserting that packages are installable?

tools/packaging/osrepos/dragonfly.repo

@@ -0,0 +1,6 @@
+[dragonfly]
+name=Dragonfly Packages
+baseurl=https://packages.dragonflydb.io/rpm/
+enabled=1
+gpgcheck=1
+gpgkey=https://packages.dragonflydb.io/pgp-key.public

tools/packaging/osrepos/dragonfly.sources

@@ -0,0 +1,5 @@
+Types: deb
+URIs: https://packages.dragonflydb.io/deb
+Suites: noble
+Components: main
+Signed-By: /usr/share/keyrings/dragonfly-keyring.public

tools/packaging/osrepos/pgp-key.public

@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGjkpygBEADuvzXdOXChr/e4Uh2UBne60NPjmuhpjmArfMfqySeRezJ1Nuvd
+AvKNuYRyCw+zsh0Zc/sSANpIdAeKPqrfZJgfEIJI0f8WVjfqsCKi+yWB7Bx0GjQ9
+y/xoFLKkT7p0P/F4yRlb8kQq2KVP9UvcZBETJY96TpQIJM4N3XoG+8DsELW5HYF2
+6sbhgmaNUsxm9oH5UqHcBc7TTgUp10GmZFR4dTeB1IffD/eLMVDMQ8ygzmVxkJPQ
+zEKfpFFzseTVyreQlZ5U4GDR8FiB0mY4gZxbCywNqZRycyMM7v4EHuUO0fOgRHdl
+5dseF+H1aEG/00JRo6zjiIbgMga0x9wYmVWvTU4wLnGoomukEMCkEQxlil1QjUlK
+XI0EltU03DuGki5uhYc9dSS1h74ku2xWePaMsvmxrTphRo1WQBDutzVXSIZ6NBc3
+BN+VBHcumVvif9aRrsfsj2CXhnOB61AW+VWk3fk0evW9cceXZDA0NgGdyeTfS7EI
+pioaWtmE3Uv3AfHTlNbMytxG7d7k7oAT2xV6z2IygyQZ5LI1tvSJJ+I5kZHKeruj
+k2bFp6H9FGi+g4kA+z9QWgkt+0UXYbjKZAs5Es1uGrRk6o1rAyVTKBKz62F0YQbK
+j8Q49Z6iSobaKeQG8naCVkALSM49i4Zpw3x1jUpd7k8/KhpJObq3rewqIQARAQAB
+tCREcmFnb25mbHkgPHBhY2thZ2luZ0BkcmFnb25mbHlkYi5pbz6JAlIEEwEKADwW
+IQRgvYPC7oTdikxvMGcSMEAYvD0qugUCaOSnKAMbLwQFCwkIBwICIgIGFQoJCAsC
+BBYCAwECHgcCF4AACgkQEjBAGLw9KrpGbw//VH2zUjaoSh7SnKGdDOA7A95o2EET
+ZvChxImyb6xNKfUoMajPnKcJFg514aPFKLuJl4qJmikxdqBF/bYkznCQSJcLQhsT
+pvkqanUh/XwBqbJye1QjBq1o0qXLgeY/Ciz2nqupwLQdzvGHO6+2Yk04T89pnZEo
+CDSoZKkacu8TpalStqzqDlumryXZzdZ35hAu9OT0fVc2wtcMiY3pznLG1iawNk8I
+bzme0ezGA/fk7xEptEbGlb1OtUV5+iG/SFEVvic8GTNf1yLQNCVK3QzD1ciL3MzR
+OTH8a04ov2bMxjl8bIefKE/dFBeCSKbvkfTSMAEgqUAuRp7gvoO7uHO05A5AHU2i
+y4agskGkgQR9u1yqUXyYIM9kkpuUqqAkwRqg1pw55LG686Xe35QYH4zbpgvr45/Q
+JRPFjCbLzR1ZcNyrecHgrq2M9WNlk6dtdWBSJuc7L0M8KJqfrPxQmMpMm/KR43Ey
+um0FCgb2J+ceO2W4GrE/DHHoNTt2iio2gMcmRXM7XTmVupsigbYk7AqGncLIQ60B
+94jtv16ggXIeA5sPqmyssARXtweTM+EzLLs4K79be4K5j/yyg3CxxvZcq5CZNwoi
+fbQgGVNb4SS+nv2r1mVe9XNSonmVVrAqSIFpptH5ahqgaRDUnmy0Lzk7qiHv02OW
+PjbSiwQGHDHwq98=
+=SOT5
+-----END PGP PUBLIC KEY BLOCK-----

tools/packaging/osrepos/reprepro-config/distributions

@@ -0,0 +1,8 @@
+Codename: noble
+Suite: stable
+Architectures: amd64 arm64
+Components: main
+Origin: Dragonfly
+Label: Dragonfly
+Description: Dragonfly APT repository
+SignWith: 60BD83C2EE84DD8A4C6F306712304018BC3D2ABA

tools/packaging/osrepos/reprepro-config/options

@@ -0,0 +1 @@
+verbose

tools/packaging/osrepos/requirements.txt

@@ -0,0 +1,5 @@
+certifi>=2025.10.5
+charset-normalizer>=3.4.3
+idna>=3.10
+requests>=2.32.5
+urllib3>=2.5.0

tools/packaging/osrepos/scripts/fetch-releases.py

@@ -0,0 +1,106 @@
+import dataclasses
+import enum
+import os.path
+import time
+
+import requests
+
+"""
+Fetches the latest five releases for RPM and the single latest release for DEB.
+RPM files are placed in the destination folder where the DNF repo will expect them.
+DEB files are placed in a temporary location from where they will be copied by the
+reprepro tool.
+"""
+
+RELEASE_URL = "https://api.github.com/repos/dragonflydb/dragonfly/releases"
+
+
+class AssetKind(enum.Enum):
+    RPM = 1
+    DEB = 2
+
+
+@dataclasses.dataclass
+class Package:
+    kind: AssetKind
+    download_url: str
+    version: str
+    filename: str
+    arch: str
+
+    @staticmethod
+    def from_url(url: str) -> "Package":
+        tokens = url.split("/")
+        filename = tokens[-1]
+        kind = AssetKind.RPM if filename.endswith(".rpm") else AssetKind.DEB
+        if kind == AssetKind.DEB:
+            arch = filename.split(".")[0].split("_")[1]
+        else:
+            arch = filename.split(".")[1]
+        return Package(
+            kind=kind, download_url=url, version=tokens[-2], filename=filename, arch=arch
+        )
+
+    def storage_path(self, root: str) -> str:
+        match self.kind:
+            case AssetKind.RPM:
+                return os.path.join(root, "rpm", self.version)
+            case AssetKind.DEB:
+                # Debian packages are stored in a temporary path.
+                # The reprepro tool will copy them later to the final path.
+                return os.path.join("deb_tmp", self.arch, self.version)
+
+
+def collect_download_urls() -> list[Package]:
+    packages = []
+    # TODO retry logic
+    response = requests.get(RELEASE_URL)
+    releases = response.json()
+    for release in releases[:5]:
+        for asset in release["assets"]:
+            if asset["name"].endswith(".rpm") or asset["name"].endswith(".deb"):
+                packages.append(Package.from_url(asset["browser_download_url"]))
+    return packages
+
+
+def download_packages(root: str, packages: list[Package]):
+    # The debian repository building tool, reprepo, only supports a single package per version by default.
+    # The ability to support multiple versions has been added but is not present in ubuntu-latest on
+    # github action runners yet. So we only download one package, the latest, for ubuntu.
+    # The rest of the scripts work on a set of packages, so that when the Limit parameter is supported,
+    # we can remove this flag and start hosting more than the latest versions.
+    # Another alternative would be to use the components feature of reprepo, but it would involve updating
+    # the repository definition itself for each release, which is a bad experience for end users.
+    deb_done = False
+    for package in packages:
+        if package.kind == AssetKind.DEB and deb_done:
+            continue
+
+        print(f"Downloading {package.download_url}")
+        path = package.storage_path(root)
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+        target = os.path.join(path, package.filename)
+        # TODO retry logic
+        response = requests.get(package.download_url)
+        with open(target, "wb") as f:
+            f.write(response.content)
+        print(f"Downloaded {package.download_url}")
+        time.sleep(0.5)
+        if package.kind == AssetKind.DEB:
+            deb_done = True
+
+
+def main(root: str):
+    packages = collect_download_urls()
+    download_packages(root, packages)
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) == 1:
+        print(f"Usage: {sys.argv[0]} <site folder>")
+        sys.exit(1)
+    main(sys.argv[1])

tools/packaging/osrepos/scripts/generate-apt-repo.sh

@@ -0,0 +1,16 @@
+set -e
+
+METADATA_ROOT=_site/deb
+mkdir -pv ${METADATA_ROOT}/conf
+
+cp -av reprepro-config/* ${METADATA_ROOT}/conf
+
+reprepro -b ${METADATA_ROOT} createsymlinks
+reprepro -b ${METADATA_ROOT} export
+
+for file in $(find deb_tmp -type f -name "*.deb"); do
+  reprepro -b ${METADATA_ROOT} includedeb noble "${file}"
+done
+
+# reprepro copied files to the destination, the temporary files can be removed now
+rm -rf deb_tmp