wip soci-snapshotter

.buildkite/pipeline.yml

@@ -1,26 +1,24 @@
 common_plugins:
-  hermit: &hermit
-    elastic/hermit#v1.0.2
+  hermit: &hermit elastic/hermit#v1.0.2
   gcp: &gcp_auth
     gcp-workload-identity-federation#v1.4.0:
       audience: $GCP_CLOUD_AUDIENCE
       service-account: $GCP_CLOUD_SERVICE_ACCOUNT
   gcr: &gcr_auth
     planetscale/docker-login-gcr#v0.0.1:
-      registries: [ "us-docker.pkg.dev", "us-west1-docker.pkg.dev" ]
-
+      registries: ["us-docker.pkg.dev", "us-west1-docker.pkg.dev"]
 steps:
-  - group: "CI"
-    key: "ci"
-    # Build must be triggered by a valid tag (e.g., v1.9.5-groq1). This is because extensions-validator uses the tag to generate a version, but when 
-    # TAG is a git SHA, the version is incompatible with the following validation:
-    # https://github.com/siderolabs/extensions-validator/blob/7d4395d34c8275b8a1c98707e85cf08b381d4d63/cmd/extensions-validator/cmd/validate.go#L31
-    if: build.tag != null
-    steps:
-      - id: "build"
-        label: "Build"
-        commands: "make iscsi-tools PUSH=true"
-        plugins:
-          - *hermit
-          - *gcp_auth
-          - *gcr_auth
+- group: "CI"
+  key: "ci"
+  # Build must be triggered by a valid tag (e.g., v1.9.5-groq1). This is because extensions-validator uses the tag to generate a version, but when 
+  # TAG is a git SHA, the version is incompatible with the following validation:
+  # https://github.com/siderolabs/extensions-validator/blob/7d4395d34c8275b8a1c98707e85cf08b381d4d63/cmd/extensions-validator/cmd/validate.go#L31
+  if: build.tag != null
+  steps:
+  - id: "build"
+    label: "Build"
+    commands: "make iscsi-tools soci-snapshotter PUSH=true"
+    plugins:
+    - *hermit
+    - *gcp_auth
+    - *gcr_auth

Makefile

@@ -91,6 +91,7 @@ TARGETS += qemu-guest-agent
 TARGETS += qlogic-firmware
 TARGETS += realtek-firmware
 TARGETS += spin
+TARGETS += soci-snapshotter
 TARGETS += stargz-snapshotter
 TARGETS += tailscale
 TARGETS += thunderbolt

container-runtime/nydus-snapshotter/10-nydus-snapshotter.part

@@ -0,0 +1,11 @@
+# Enable nydus snapshotter for CRI
+[plugins.'io.containerd.cri.v1.images']
+  snapshotter = "nydus"
+  disable_snapshot_annotations = false
+  discard_unpacked_layers = false
+
+# Plug nydus snapshotter into containerd
+[proxy_plugins]
+  [proxy_plugins.nydus]
+    type = "snapshot"
+    address = "/run/containerd-nydus-grpc/containerd-nydus-grpc.sock"

container-runtime/nydus-snapshotter/README.md

@@ -0,0 +1,45 @@
+# Nydus Snapshotter extension
+
+## Installation
+
+See [Installing Extensions](https://github.com/siderolabs/extensions#installing-extensions).
+
+## Configuration
+
+### Registry Mirrors
+
+By default, snapshotter configuration sets `remote.mirrors_config.dir` to the directory `/etc/cri/conf.d/hosts`.
+
+The `/etc/cri/conf.d/hosts` file is where Talos writes its CRI-compatible mirror configuration. Fortunately,
+the snapshotter supports overriding nydusd mirrors given you configure it with the same directory that
+containerd uses. This makes mirror configuration transparent.
+
+### Snapshotter or Nydusd
+
+Configuraton is broken into two parts: snapshotter and nydusd configuration. You can modify the
+Talos machine configuraton to overwrite or append the respective configuration files as needed:
+
+ - `/usr/local/etc/containerd-nydus-grpc/snapshotter-config.toml`
+ - `/usr/local/etc/containerd-nydus-grpc/nydusd-config.json`
+
+For example:
+```
+machine:
+  files:
+    - content: |
+        [log]
+        level = "debug"
+      path: /usr/local/etc/containerd-nydus-grpc/snapshotter-config.toml
+      op: append
+    - content: |
+        {
+          "fs_prefetch": {
+            "enable": false,
+            "threads_count": 8,
+            "merging_size": 1048576,
+            "prefetch_all": true
+          }
+        }
+      path: /usr/local/etc/containerd-nydus-grpc/nydusd-config.json
+      op: overwrite
+```

container-runtime/nydus-snapshotter/manifest.yaml

@@ -0,0 +1,10 @@
+version: v1alpha1
+metadata:
+  name: nydus-snapshotter
+  version: "$VERSION"
+  author: Groq
+  description: |
+    This system extension provides Nydus Snapshotter using containerd's runtime handler.
+  compatibility:
+    talos:
+      version: ">= v1.9.0"

container-runtime/nydus-snapshotter/nydus-snapshotter.yaml

@@ -0,0 +1,83 @@
+name: nydus-snapshotter
+depends:
+  - service: cri
+container:
+  environment:
+  - PATH=/usr/local/bin
+  entrypoint: containerd-nydus-grpc  
+  args:
+    - --address=/var/run/containerd-nydus-grpc/containerd-nydus-grpc.sock
+    - --config=/etc/containerd-nydus-grpc/snapshotter-config.toml
+    - --nydusd-config=/etc/containerd-nydus-grpc/nydusd-config.json
+    - --root=/var/lib/containerd/io.containerd.snapshotter.v1.nydus
+    - --daemon-mode=multiple
+    - --log-to-stdout
+  security:
+    rootfsPropagation: shared
+  mounts:
+    - source: /etc/ssl/certs/ca-certificates.crt
+      destination: /etc/ssl/certs/ca-certificates.crt
+      type: bind
+      options:
+        - bind
+        - ro
+    # share registry mirror config with with nydus
+    - source: /etc/cri/conf.d/hosts
+      destination: /etc/cri/conf.d/hosts
+      type: bind
+      options:
+        - rshared
+        - rbind
+        - ro
+    # location of snapshotter and nydusd config on the host
+    - source: /usr/local/etc/containerd-nydus-grpc
+      destination: /etc/containerd-nydus-grpc
+      type: bind
+      options:
+        - rbind
+        - ro
+    # location of nydusd and containerd-nydus-grpc binaries
+    - source: /usr/local/bin
+      destination: /usr/local/bin
+      type: bind
+      options:
+        - bind
+        - ro
+    # required for snapshotter to store state
+    - source: /var/lib/containerd/io.containerd.snapshotter.v1.nydus
+      destination: /var/lib/containerd/io.containerd.snapshotter.v1.nydus
+      type: bind
+      options:
+        - rshared
+        - rbind
+        - rw
+    # required containerd to proxy to snapshotter
+    - source: /var/run/containerd-nydus-grpc
+      destination: /var/run/containerd-nydus-grpc
+      type: bind
+      options:
+        - bind
+        - rw
+    # libs
+    - source: /lib
+      destination: /lib
+      type: bind
+      options:
+        - bind
+        - ro
+    # more libs
+    - source: /usr/lib
+      destination: /usr/lib
+      type: bind
+      options:
+        - bind
+        - ro
+    # required for nydusd fuse mounts
+    - source: /dev
+      destination: /dev
+      type: bind
+      options:
+        - rshared
+        - rbind
+        - rw
+restart: always

container-runtime/nydus-snapshotter/nydusd-config.json

@@ -0,0 +1,29 @@
+{
+    "device": {
+      "backend": {
+        "type": "registry",
+        "config": {
+          "mirrors": [],
+          "timeout": 5,
+          "connect_timeout": 5,
+          "retry_limit": 2
+        }
+      },
+      "cache": {
+        "type": "blobcache",
+        "config": {
+          "work_dir": "/var/lib/nydus/cache/"
+        }
+      }
+    },
+    "mode": "direct",
+    "digest_validate": false,
+    "iostats_files": false,
+    "enable_xattr": true,
+    "fs_prefetch": {
+      "enable": false,
+      "threads_count": 8,
+      "merging_size": 1048576,
+      "prefetch_all": false
+    }
+  }

container-runtime/nydus-snapshotter/pkg.yaml

@@ -0,0 +1,67 @@
+name: nydus-snapshotter
+variant: scratch
+shell: /toolchain/bin/bash
+dependencies:
+  - stage: base
+steps:
+  - sources:
+      - url: https://github.com/containerd/nydus-snapshotter/archive/refs/tags/{{ .NYDUS_SNAPSHOTTER_VERSION }}.tar.gz
+        destination: nydus-snapshotter.tar.gz
+        sha256: 3ecc747fd05096472a2d661b6d944d644f5ffa665b2e0b3eda1c3b67e88820d3
+        sha512: d557bcc743631818a68b58dc08597ff775aaf41279fd91fb200b451157978ac7740c388c7a803966b42154e5a4c92ca046abf35a500c89486e0ca86a27ff43cc
+      # {{ if eq .ARCH "aarch64" }} This in fact is YAML comment, but Go templating instruction is evaluated by bldr
+      - url: https://github.com/dragonflyoss/nydus/releases/download/{{ .NYDUS_VERSION }}/nydus-static-{{ .NYDUS_VERSION }}-linux-arm64.tgz
+        destination: nydus-static.tar.gz
+        sha256: a222238dfcd205e9c667c4ae51867bda981a51c6de4177a014abb0711be232d8
+        sha512: c1078f5f74dfe4f329c89897c0403a0152dd8e89453a313c85886187bd37ef75341dfb80439fbc13ac0803b81a1f8055b3200ac1216165c044a90dd8635c57e2
+      # {{ else }} This in fact is YAML comment, but Go templating instruction is evaluated by bldr
+      - url: https://github.com/dragonflyoss/nydus/releases/download/{{ .NYDUS_VERSION }}/nydus-static-{{ .NYDUS_VERSION }}-linux-amd64.tgz
+        destination: nydus-static.tar.gz
+        sha256: bd0cac0785572b2d8af1b4f97f9272713356fbc0c8e0770b23cfc8ea0e3920ae
+        sha512: c8331c991749321001e200be649534ef00ca41e5bd90313e6468c94ddadaa849037d430aec4a7cfa21ed63597c8bda703cae0f0dc1535f43af9f961889670e75
+      # {{ end }} This in fact is YAML comment, but Go templating instruction is evaluated by bldr
+    cachePaths:
+      - /.cache/go-build
+      - /go/pkg
+    prepare:
+      - |
+        sed -i 's#$VERSION#{{ .VERSION }}#' /pkg/manifest.yaml
+      - |
+        mkdir -p nydus-snapshotter
+        tar -xzf nydus-snapshotter.tar.gz --strip-components=1 -C nydus-snapshotter
+      - | 
+        tar -xzf nydus-static.tar.gz --strip-components=1
+    build:
+      - |
+        export PATH=${PATH}:${TOOLCHAIN}/go/bin
+        cd nydus-snapshotter
+        make static
+    install:
+      - |
+        mkdir -p /rootfs/usr/local/bin
+        cp -p nydusd /rootfs/usr/local/bin
+        cp -p nydus-snapshotter/bin/containerd-nydus-grpc /rootfs/usr/local/bin
+        cp -p nydus-snapshotter/bin/nydus-overlayfs /rootfs/usr/local/bin
+      - |
+        mkdir -p /rootfs/etc/cri/conf.d
+        cp /pkg/10-nydus-snapshotter.part /rootfs/etc/cri/conf.d/10-nydus-snapshotter.part
+
+        mkdir -p /rootfs/usr/local/etc/containerd-nydus-grpc
+        cp /pkg/snapshotter-config.toml /rootfs/usr/local/etc/containerd-nydus-grpc/snapshotter-config.toml
+        cp /pkg/nydusd-config.json /rootfs/usr/local/etc/containerd-nydus-grpc/nydusd-config.json
+
+        mkdir -p /rootfs/usr/local/lib/containers/nydus-snapshotter
+
+        mkdir -p /rootfs/usr/local/etc/containers
+        cp /pkg/nydus-snapshotter.yaml /rootfs/usr/local/etc/containers/
+    test:
+      - |
+        mkdir -p /extensions-validator-rootfs
+        cp -r /rootfs/ /extensions-validator-rootfs/rootfs
+        cp /pkg/manifest.yaml /extensions-validator-rootfs/manifest.yaml
+        /extensions-validator validate --rootfs=/extensions-validator-rootfs --pkg-name="${PKG_NAME}"
+finalize:
+  - from: /rootfs
+    to: /rootfs
+  - from: /pkg/manifest.yaml
+    to: /

container-runtime/nydus-snapshotter/snapshotter-config.toml

@@ -0,0 +1,4 @@
+version = 1
+
+[remote.mirrors_config]
+dir = "/etc/cri/conf.d/hosts"

container-runtime/nydus-snapshotter/vars.yaml

@@ -0,0 +1,2 @@
+VERSION: "{{ .NYDUS_SNAPSHOTTER_VERSION }}"
+NYDUS_VERSION: "{{ .NYDUS_VERSION }}"

container-runtime/soci-snapshotter/10-soci-snapshotter.part

@@ -0,0 +1,10 @@
+[proxy_plugins.soci]
+type = "snapshot"
+address = "/run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock"
+[proxy_plugins.soci.exports]
+  root = "/var/lib/soci-snapshotter-grpc"
+
+[plugins."io.containerd.cri.v1.images"]
+  snapshotter = "soci"
+  # This line is required for containerd to send information about how to lazily load the image to the snapshotter
+  disable_snapshot_annotations = false

container-runtime/soci-snapshotter/README.md

@@ -0,0 +1,45 @@
+# Nydus Snapshotter extension
+
+## Installation
+
+See [Installing Extensions](https://github.com/siderolabs/extensions#installing-extensions).
+
+## Configuration
+
+### Registry Mirrors
+
+By default, snapshotter configuration sets `remote.mirrors_config.dir` to the directory `/etc/cri/conf.d/hosts`.
+
+The `/etc/cri/conf.d/hosts` file is where Talos writes its CRI-compatible mirror configuration. Fortunately,
+the snapshotter supports overriding nydusd mirrors given you configure it with the same directory that
+containerd uses. This makes mirror configuration transparent.
+
+### Snapshotter or Nydusd
+
+Configuraton is broken into two parts: snapshotter and nydusd configuration. You can modify the
+Talos machine configuraton to overwrite or append the respective configuration files as needed:
+
+ - `/usr/local/etc/containerd-nydus-grpc/snapshotter-config.toml`
+ - `/usr/local/etc/containerd-nydus-grpc/nydusd-config.json`
+
+For example:
+```
+machine:
+  files:
+    - content: |
+        [log]
+        level = "debug"
+      path: /usr/local/etc/containerd-nydus-grpc/snapshotter-config.toml
+      op: append
+    - content: |
+        {
+          "fs_prefetch": {
+            "enable": false,
+            "threads_count": 8,
+            "merging_size": 1048576,
+            "prefetch_all": true
+          }
+        }
+      path: /usr/local/etc/containerd-nydus-grpc/nydusd-config.json
+      op: overwrite
+```

container-runtime/soci-snapshotter/manifest.yaml

@@ -0,0 +1,10 @@
+version: v1alpha1
+metadata:
+  name: soci-snapshotter
+  version: "$VERSION"
+  author: Groq
+  description: |
+    This system extension provides Soci Snapshotter using containerd's runtime handler.
+  compatibility:
+    talos:
+      version: ">= v1.9.0"