Harden zot registry, pt 1 (#231)

## Summary
- Enable OIDC + API key authentication on zot with anonymous pull preserved
- Enforce tag immutability for version tags
- Adopt commit-SHA-based container image tagging

Details in the [[harden-zot-registry]] Mikado chain (`mise run docs-mikado harden-zot-registry`).

## Test plan
- [ ] Anonymous pull still works
- [ ] Unauthenticated push fails (401)
- [ ] CI container builds pass with new auth and tagging
- [ ] `mise run services-check` passes

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/231

.dagger/src/blumeops_ci/main.py

@@ -70,6 +70,57 @@ class BlumeopsCi:
             .file(f"/docs-{version}.tar.gz")
         )
 
+    @function
+    async def build_nix(
+        self, src: dagger.Directory, container_name: str
+    ) -> dagger.File:
+        """Build a nix container from containers/<name>/default.nix.
+
+        Returns the docker-archive tarball that can be loaded with
+        `docker load` or pushed with `skopeo copy`.
+        """
+        nix_file = f"containers/{container_name}/default.nix"
+        # Resolve nixpkgs store path from flake registry, then build.
+        # Uses nix-instantiate to parse JSON (avoids needing jq).
+        resolve_and_build = (
+            "set -e; "
+            "nix --extra-experimental-features 'nix-command flakes' "
+            "flake metadata nixpkgs --json > /tmp/nixpkgs.json; "
+            "NIXPKGS_PATH=$(nix-instantiate --eval -E "
+            '"(builtins.fromJSON (builtins.readFile /tmp/nixpkgs.json)).path" '
+            "| tr -d '\"'); "
+            'export NIX_PATH="nixpkgs=$NIXPKGS_PATH"; '
+            'echo "NIX_PATH=$NIX_PATH"; '
+            'nix-build "$1" -o /result'
+        )
+        return await (
+            dag.container()
+            .from_(NIX_IMAGE)
+            .with_directory("/workspace", src)
+            .with_workdir("/workspace")
+            .with_exec(["sh", "-c", resolve_and_build, "_", nix_file])
+            .file("/result")
+        )
+
+    @function
+    async def nix_version(self, package: str) -> str:
+        """Extract the version of a nixpkgs package. Returns version string."""
+        return await (
+            dag.container()
+            .from_(NIX_IMAGE)
+            .with_exec(
+                [
+                    "nix",
+                    "--extra-experimental-features",
+                    "nix-command flakes",
+                    "eval",
+                    "--raw",
+                    f"nixpkgs#{package}.version",
+                ]
+            )
+            .stdout()
+        )
+
     @function
     async def flake_lock(
         self, src: dagger.Directory, flake_path: str = "nixos/ringtail"

.pre-commit-config.yaml

@@ -89,6 +89,16 @@ repos:
         args: ['-config-file', '.github/actionlint.yaml']
         files: ^\.forgejo/workflows/
 
+  # Container version consistency
+  - repo: local
+    hooks:
+      - id: container-version-check
+        name: container-version-check
+        entry: mise run container-version-check
+        language: system
+        files: ^(containers/|service-versions\.yaml)
+        pass_filenames: false
+
   # Documentation validation
   - repo: local
     hooks:

containers/cv/Dockerfile

@@ -6,6 +6,8 @@
 #
 # The container downloads the tarball on startup, extracts it, and serves with nginx.
 
+ARG CONTAINER_APP_VERSION=1.0.3
+
 FROM nginx:alpine
 
 # Install curl for downloading release assets

containers/devpi/Dockerfile

@@ -1,7 +1,15 @@
+ARG CONTAINER_APP_VERSION=6.19.1
+
 FROM python:3.12-slim
 
+ARG CONTAINER_APP_VERSION
+ARG DEVPI_SERVER_VERSION=${CONTAINER_APP_VERSION}
+ARG DEVPI_WEB_VERSION=5.0.1
+
 # Install devpi-server and devpi-web
-RUN pip install --no-cache-dir devpi-server devpi-web
+RUN pip install --no-cache-dir \
+    devpi-server==${DEVPI_SERVER_VERSION} \
+    devpi-web==${DEVPI_WEB_VERSION}
 
 # Create non-root user
 RUN useradd -r -u 1000 devpi && mkdir -p /devpi && chown devpi:devpi /devpi

containers/forgejo-runner/Dockerfile

@@ -9,9 +9,13 @@
 # Usage: Configure runner with label like:
 #   docker:docker://registry.ops.eblu.me/blumeops/forgejo-runner:latest
 
+ARG CONTAINER_APP_VERSION=0.19.11
+
 FROM debian:bookworm-slim
 
 ARG TARGETARCH
+ARG CONTAINER_APP_VERSION
+ARG DAGGER_VERSION=${CONTAINER_APP_VERSION}
 
 # Install base dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -51,7 +55,6 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \
     && argocd version --client
 
 # Install Dagger CLI (for running Dagger CI pipelines)
-ARG DAGGER_VERSION=0.19.11
 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \
     && curl -fsSL -o /tmp/dagger.tar.gz \
     "https://dl.dagger.io/dagger/releases/${DAGGER_VERSION}/dagger_v${DAGGER_VERSION}_linux_${ARCH}.tar.gz" \

containers/homepage/Dockerfile

@@ -1,7 +1,8 @@
 # Homepage - self-hosted services dashboard
 # Two-stage build: Node.js build, Alpine runtime
 
-ARG HOMEPAGE_VERSION=v1.10.1
+ARG CONTAINER_APP_VERSION=v1.10.1
+ARG HOMEPAGE_VERSION=${CONTAINER_APP_VERSION}
 
 FROM node:24-slim AS builder
 

containers/kiwix-serve/Dockerfile

@@ -1,10 +1,13 @@
 # kiwix-serve container
 # Downloads pre-built binary from kiwix mirror
 
+ARG CONTAINER_APP_VERSION=3.8.1
+
 FROM alpine:3.22
 
 ARG TARGETPLATFORM
-ARG KIWIX_VERSION=3.8.1
+ARG CONTAINER_APP_VERSION
+ARG KIWIX_VERSION=${CONTAINER_APP_VERSION}
 
 RUN set -e && \
     apk --no-cache add dumb-init curl && \

containers/kubectl/Dockerfile

@@ -1,10 +1,13 @@
 # Minimal kubectl container
 # Multi-arch build: downloads correct binary for target platform
 
+ARG CONTAINER_APP_VERSION=v1.34.4
+
 FROM alpine:3.22 AS downloader
 
 ARG TARGETARCH
-ARG KUBECTL_VERSION=v1.34.4
+ARG CONTAINER_APP_VERSION
+ARG KUBECTL_VERSION=${CONTAINER_APP_VERSION}
 
 RUN apk add --no-cache curl && \
     # Detect architecture - use TARGETARCH if set, otherwise detect from uname

containers/miniflux/Dockerfile

@@ -1,7 +1,8 @@
 # Miniflux RSS feed reader
 # Based on upstream packaging/docker/alpine/Dockerfile
 
-ARG MINIFLUX_VERSION=2.2.17
+ARG CONTAINER_APP_VERSION=2.2.17
+ARG MINIFLUX_VERSION=${CONTAINER_APP_VERSION}
 
 FROM golang:alpine3.22 AS build
 

containers/navidrome/Dockerfile

@@ -1,7 +1,8 @@
 # Navidrome music server
 # Three-stage build: UI (Node), backend (Go+taglib), runtime (Alpine)
 
-ARG NAVIDROME_VERSION=v0.60.3
+ARG CONTAINER_APP_VERSION=v0.60.3
+ARG NAVIDROME_VERSION=${CONTAINER_APP_VERSION}
 
 FROM node:22-alpine AS ui-build
 

containers/nettest/Dockerfile

@@ -4,6 +4,8 @@
 # - Docker on indri (during CI build)
 # - Minikube pods (manual testing)
 
+ARG CONTAINER_APP_VERSION=0.1.0
+
 FROM alpine:3.22
 
 RUN apk add --no-cache \

containers/ntfy/Dockerfile

@@ -1,7 +1,8 @@
 # ntfy push notification server
 # Three-stage build: Web UI (Node), server (Go+SQLite), runtime (Alpine)
 
-ARG NTFY_VERSION=v2.17.0
+ARG CONTAINER_APP_VERSION=v2.17.0
+ARG NTFY_VERSION=${CONTAINER_APP_VERSION}
 ARG NTFY_COMMIT=a03a37feb1869e84e3af0dd6190bdc7183f211ec
 
 FROM node:22-alpine AS web-build

containers/ntfy/default.nix

@@ -1,20 +1,80 @@
 # Nix-built ntfy push notification server
-# Replaces the multi-stage Dockerfile (Node + Go + Alpine) with nixpkgs ntfy-sh
+# Builds v2.17.0 from forge mirror (nixpkgs has 2.15.0)
 # Built with dockerTools.buildLayeredImage for efficient layer caching
 { pkgs ? import <nixpkgs> { } }:
 
+let
+  version = "2.17.0";
+
+  src = pkgs.fetchgit {
+    url = "https://forge.ops.eblu.me/eblume/ntfy.git";
+    rev = "v${version}";
+    hash = "sha256-/dxILAkye1HwYcybnx1WrMRK2jXZMrxal2ZKm6y2bWc=";
+  };
+
+  ui = pkgs.buildNpmPackage {
+    inherit src version;
+    pname = "ntfy-sh-ui";
+    npmDepsHash = "sha256-d73rymqCKalsjAwHSJshEovmUHJStfGt8wcZYN49sHY=";
+
+    prePatch = ''
+      cd web/
+    '';
+
+    installPhase = ''
+      runHook preInstall
+      mv build/index.html build/app.html
+      rm build/config.js
+      mkdir -p $out
+      mv build/ $out/site
+      runHook postInstall
+    '';
+  };
+
+  ntfy = pkgs.buildGoModule {
+    inherit src version;
+    pname = "ntfy-sh";
+    vendorHash = "sha256-/mQ+UwBYz78mPVVwYgsSYatE00ce2AKXJdx+nl6oT8E=";
+
+    doCheck = false;
+
+    ldflags = [
+      "-s"
+      "-w"
+      "-X main.version=${version}"
+    ];
+
+    postPatch = ''
+      sed -i 's# /bin/echo# echo#' Makefile
+    '';
+
+    # Copy pre-built web UI; skip docs (create placeholder for go:embed)
+    preBuild = ''
+      cp -r ${ui}/site/ server/
+      mkdir -p server/docs && touch server/docs/placeholder
+    '';
+
+    meta = with pkgs.lib; {
+      description = "Send push notifications to your phone or desktop via PUT/POST";
+      homepage = "https://ntfy.sh";
+      license = licenses.asl20;
+      mainProgram = "ntfy";
+    };
+  };
+in
+
 pkgs.dockerTools.buildLayeredImage {
   name = "blumeops/ntfy";
   tag = "latest";
 
   contents = [
-    pkgs.ntfy-sh
+    ntfy
     pkgs.cacert
     pkgs.tzdata
   ];
 
   config = {
-    Entrypoint = [ "${pkgs.ntfy-sh}/bin/ntfy" ];
+    Entrypoint = [ "${ntfy}/bin/ntfy" ];
     Env = [
       "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
       "TZDIR=${pkgs.tzdata}/share/zoneinfo"

containers/quartz/Dockerfile

@@ -6,7 +6,10 @@
 #
 # The container downloads the tarball on startup, extracts it, and serves with nginx.
 
-FROM nginx:alpine
+ARG CONTAINER_APP_VERSION=1.28.2
+ARG NGINX_VERSION=${CONTAINER_APP_VERSION}
+
+FROM nginx:${NGINX_VERSION}-alpine
 
 # Install curl for downloading release assets
 RUN apk add --no-cache curl

containers/teslamate/Dockerfile

@@ -1,7 +1,8 @@
 # TeslaMate - Tesla data logger
 # Based on upstream Dockerfile
 
-ARG TESLAMATE_VERSION=v2.2.0
+ARG CONTAINER_APP_VERSION=v2.2.0
+ARG TESLAMATE_VERSION=${CONTAINER_APP_VERSION}
 
 FROM elixir:1.18-otp-26 AS builder
 

containers/transmission/Dockerfile

@@ -1,9 +1,12 @@
 # Transmission BitTorrent daemon
 # Simpler alternative to linuxserver image
 
+ARG CONTAINER_APP_VERSION=4.0.6-r4
+
 FROM alpine:3.22
 
-ARG TRANSMISSION_VERSION=4.0.6-r4
+ARG CONTAINER_APP_VERSION
+ARG TRANSMISSION_VERSION=${CONTAINER_APP_VERSION}
 
 RUN apk add --no-cache \
     transmission-daemon=${TRANSMISSION_VERSION} \

docs/changelog.d/harden-zot-mikado-cards.ai.md

@@ -1 +1 @@
-Create C2 Mikado cards for harden-zot-registry: root goal and three prerequisite cards (register-zot-oidc-client, wire-ci-registry-auth, enforce-tag-immutability).
+Expand harden-zot-registry Mikado chain: add prereqs for container version sync check, pin container versions, and Dagger nix build function.

docs/how-to/authentik/build-authentik-container.md

@@ -16,7 +16,7 @@ Discovered while attempting [[deploy-authentik]]: the deployment references `reg
 
 ## What to Do
 
-1. Verify `containers/authentik/default.nix` builds on ringtail (the Nix builder runs there)
+1. Verify `containers/authentik/default.nix` builds — locally via Dagger (`dagger call build-nix --src=. --container-name=authentik`) or on ringtail (the CI nix builder runs there)
 2. The `ak` entrypoint needs bash (included via `bashInteractive`) and orchestrates both `server` and `worker` subcommands
 3. Tag and release: `mise run container-tag-and-release authentik v1.0.0`
 4. Verify the `-nix` tagged image appears in the registry

docs/how-to/deployment/build-container-image.md

@@ -1,6 +1,6 @@
 ---
 title: Build Container Image
-modified: 2026-02-19
+modified: 2026-02-20
 last-reviewed: 2026-02-15
 tags:
   - how-to
@@ -38,7 +38,13 @@ A container can have one or both build files. The directory name becomes the ima
 dagger call build --src=. --container-name=<name>
 ```
 
-**Nix** — test with nix-build (requires nix, e.g. on [[ringtail]]):
+**Nix** — test with Dagger (no local nix required):
+
+```bash
+dagger call build-nix --src=. --container-name=<name> export --path=./<name>.tar.gz
+```
+
+Or with nix-build directly (requires nix, e.g. on [[ringtail]]):
 
 ```bash
 nix-build containers/<name>/default.nix -o result

docs/how-to/how-to.md

@@ -73,6 +73,10 @@ Mikado chain for hardening the zot registry. Track progress with `mise run docs-
 - [[wire-ci-registry-auth]]
 - [[enforce-tag-immutability]]
 - [[adopt-commit-based-container-tags]]
+- [[add-container-version-sync-check]]
+- [[pin-container-versions]]
+- [[add-dagger-nix-build]]
+- [[fix-ntfy-nix-version]]
 
 ## Authentik
 

docs/how-to/zot/add-container-version-sync-check.md

@@ -0,0 +1,82 @@
+---
+title: Add Container Version Sync Check
+modified: 2026-02-20
+requires:
+  - pin-container-versions
+  - add-dagger-nix-build
+  - fix-ntfy-nix-version
+tags:
+  - how-to
+  - containers
+  - ci
+  - zot
+---
+
+# Add Container Version Sync Check
+
+Add a pre-commit check that validates version consistency across the three places container versions are declared: Dockerfile ARGs, `service-versions.yaml`, and nix derivations. No VERSION files needed — the existing sources are the source of truth, and the check enforces they agree.
+
+## Context
+
+Discovered during analysis of [[adopt-commit-based-container-tags]]: the new commit-SHA-based image tags need a reliable version source (`vX.Y.Z-<sha>`). Versions are currently scattered across Dockerfile ARGs (varying naming conventions), `service-versions.yaml` entries (many still `null`), and nix derivations (implicit from nixpkgs). A sync check ensures these stay consistent without adding a redundant fourth source.
+
+## What Was Done
+
+### 1. Created `mise run container-version-check` task
+
+A typer-based uv-script that iterates over `containers/*/` and validates five rules per container:
+
+1. Any Dockerfile must declare `ARG CONTAINER_APP_VERSION=<value>`
+2. Any `default.nix` must produce a version via `dagger call nix-version`
+3. At least one build file must exist (Dockerfile or default.nix)
+4. A matching `service-versions.yaml` entry must exist with non-null `current-version`
+5. All resolved versions from (1), (2), and (4) must agree (v-prefix stripped for comparison)
+
+Scoping: by default only checks containers changed vs main. `--all-files` checks everything. If `service-versions.yaml` itself changed, all containers are checked.
+
+Blacklisted containers (utility images, not tracked services): `kubectl`, `nettest`.
+
+Container-to-service name mapping: `quartz` → `docs`, `kiwix-serve` → `kiwix`.
+
+### 2. Added pre-commit hook
+
+```yaml
+- id: container-version-check
+  name: container-version-check
+  entry: mise run container-version-check
+  language: system
+  files: ^(containers/|service-versions\.yaml)
+  pass_filenames: false
+```
+
+### 3. Populated `service-versions.yaml`
+
+Filled in `current-version` for all hybrid services: navidrome (v0.60.3), miniflux (2.2.17), teslamate (v2.2.0), transmission (4.0.6-r4), kiwix (3.8.1), forgejo-runner (0.19.11). Added authentik (2025.10.1) as a new hybrid entry.
+
+### ntfy nix version skew (resolved)
+
+The check discovered that ntfy's Dockerfile pins v2.17.0 but nixpkgs has ntfy-sh 2.15.0. This was resolved in [[fix-ntfy-nix-version]] by building a custom nix derivation from the forge mirror. The version check now extracts the version from local nix files via regex, falling back to Dagger for unmodified nixpkgs packages.
+
+## Key Files
+
+| File | Change |
+|------|--------|
+| `mise-tasks/container-version-check` | New: typer CLI sync validation script |
+| `.pre-commit-config.yaml` | Add `container-version-check` hook |
+| `service-versions.yaml` | Populate `current-version` for all hybrid services + authentik |
+
+## Verification
+
+- [x] `mise run container-version-check --all-files` passes with no errors
+- [x] Intentionally changing a Dockerfile ARG without updating `service-versions.yaml` fails the check
+- [x] `service-versions.yaml` has `current-version` populated for all hybrid services
+- [x] Nix-only container versions (authentik) checked via Dagger
+- [x] ntfy nix version resolved via [[fix-ntfy-nix-version]]
+
+## Related
+
+- [[pin-container-versions]] — Prereq: containers need parseable version ARGs first
+- [[add-dagger-nix-build]] — Prereq: nix version extraction
+- [[fix-ntfy-nix-version]] — Prereq: ntfy nix derivation version skew
+- [[adopt-commit-based-container-tags]] — Parent: CI uses the same version extraction at build time
+- [[harden-zot-registry]] — Root goal

docs/how-to/zot/add-dagger-nix-build.md

@@ -0,0 +1,97 @@
+---
+title: Add Dagger Nix Build Function
+modified: 2026-02-20
+status:
+tags:
+  - how-to
+  - containers
+  - ci
+  - dagger
+  - zot
+---
+
+# Add Dagger Nix Build Function
+
+Add Dagger functions for building nix container images and extracting version info from nix derivations. This enables local nix container evaluation and provides the version extraction mechanism needed by [[add-container-version-sync-check]].
+
+## Context
+
+Discovered during analysis of [[adopt-commit-based-container-tags]]: nix containers (authentik, ntfy, nettest) derive their bundled app version from the nixpkgs pin, not from an explicit declaration. To validate that a VERSION file matches the actual nix-built version, we need a way to query the version from nix.
+
+Currently, nix containers can only be built on ringtail (the `nix-container-builder` runner). There is no local build path for developers — the only option is to push and wait for CI. Adding a Dagger-based nix build gives both local evaluation and version extraction.
+
+## What to Do
+
+### 1. Add `build_nix` Dagger function
+
+A new function in `.dagger/src/blumeops_ci/main.py` that builds a nix container inside a `nixos/nix` container:
+
+```python
+@function
+async def build_nix(
+    self, src: dagger.Directory, container_name: str
+) -> dagger.File:
+    """Build a nix container from containers/<name>/default.nix. Returns the image tarball."""
+    # Uses NIX_IMAGE (nixos/nix:2.33.3) — already defined in the module
+    # Runs nix-build inside the container
+    # Returns the docker-archive tarball
+```
+
+This mirrors the existing `build` function (Dockerfile) but for nix. The result is a docker-archive tarball that can be loaded with `docker load` or pushed with `skopeo`.
+
+### 2. Add `nix_version` Dagger function
+
+A function that extracts the version of a specific nix package from the nixpkgs pin:
+
+```python
+@function
+async def nix_version(
+    self, src: dagger.Directory, package: str
+) -> str:
+    """Extract the version of a nixpkgs package. Returns version string."""
+    # nix eval --raw nixpkgs#<package>.version
+```
+
+This lets the version sync check run `dagger call nix-version --src=. --package=authentik` to get the actual version that would be built.
+
+### 3. Add `publish_nix` Dagger function (optional)
+
+If useful, a combined build-and-push that mirrors `publish` but for nix images:
+
+```python
+@function
+async def publish_nix(
+    self, src: dagger.Directory, container_name: str, version: str,
+    registry: str = "registry.ops.eblu.me",
+) -> str:
+    """Build nix container and push to registry via skopeo."""
+```
+
+This would give a `dagger call publish-nix` path parallel to the existing `dagger call publish`.
+
+## Nix in Dagger
+
+The `flake_lock` function already demonstrates running nix inside Dagger using `nixos/nix:2.33.3`. The nix build function follows the same pattern but needs:
+
+- `NIX_PATH` set to resolved nixpkgs (same as the CI workflow does)
+- `--extra-experimental-features "nix-command flakes"` for `nix eval`
+- The full repo source mounted (nix files may reference other files like `test-connectivity.sh`)
+
+## Key Files
+
+| File | Change |
+|------|--------|
+| `.dagger/src/blumeops_ci/main.py` | Add `build_nix`, `nix_version`, optionally `publish_nix` |
+
+## Verification
+
+- [ ] `dagger call build-nix --src=. --container-name=nettest` produces a valid docker-archive tarball
+- [ ] `dagger call nix-version --src=. --package=ntfy-sh` returns the correct version string
+- [ ] `dagger call nix-version --src=. --package=authentik` returns the Authentik version
+- [ ] Tarball from `build-nix` can be loaded with `docker load` and run locally
+
+## Related
+
+- [[add-container-version-sync-check]] — Parent: needs nix version extraction for sync check
+- [[adopt-commit-based-container-tags]] — Grandparent goal
+- [[dagger]] — Dagger reference

docs/how-to/zot/adopt-commit-based-container-tags.md

@@ -2,6 +2,8 @@
 title: Adopt Commit-Based Container Tags
 modified: 2026-02-20
 status: active
+requires:
+  - add-container-version-sync-check
 tags:
   - how-to
   - containers
@@ -35,7 +37,12 @@ Both the Dockerfile and Nix workflows fire for each trigger, each bailing out if
 
 ### Version Source
 
-Each container declares the version of its primary bundled app. The mechanism for declaring this (e.g., a `VERSION` file, parsing a Dockerfile `ARG`, or a convention per container) should be determined during implementation.
+Each container's version is extracted at build time from existing declarations — no separate VERSION file:
+
+- **Dockerfile builds**: parsed from `ARG CONTAINER_APP_VERSION=<value>` in the Dockerfile
+- **Nix builds**: extracted via `dagger call nix-version` or `nix eval`
+
+The [[add-container-version-sync-check]] pre-commit check ensures these declarations stay in sync with `service-versions.yaml`. See [[pin-container-versions]] for the work to ensure every container has a parseable version.
 
 ### Image Tag Format
 

docs/how-to/zot/fix-ntfy-nix-version.md

@@ -0,0 +1,41 @@
+---
+title: Fix ntfy Nix Version
+modified: 2026-02-20
+tags:
+  - how-to
+  - containers
+  - nix
+  - zot
+---
+
+# Fix ntfy Nix Version
+
+Override the nixpkgs ntfy-sh derivation to build v2.17.0 from the forge mirror, aligning the nix-built container with the Dockerfile version.
+
+## Context
+
+Discovered during [[add-container-version-sync-check]]: the ntfy container has both a Dockerfile and a `default.nix`. The Dockerfile builds v2.17.0 from `forge.ops.eblu.me/eblume/ntfy.git`, but the nix derivation uses `pkgs.ntfy-sh` from nixpkgs which is pinned at 2.15.0. The version sync check currently excludes ntfy from nix version validation as a workaround.
+
+## What Was Done
+
+Replaced the nixpkgs `pkgs.ntfy-sh` reference in `containers/ntfy/default.nix` with a custom derivation that builds v2.17.0 from the forge mirror using `fetchgit`, `buildNpmPackage` (web UI), and `buildGoModule` (server). Docs are skipped (placeholder for `go:embed`, matching the Dockerfile approach).
+
+The `container-version-check` script was updated to extract versions from local nix files via regex (`version = "X.Y.Z"`) before falling back to the Dagger `nix-version` function for unmodified nixpkgs packages. This avoids the issue where `nix eval nixpkgs#ntfy-sh.version` returns the upstream 2.15.0 instead of our overridden 2.17.0.
+
+## Key Files
+
+| File | Change |
+|------|--------|
+| `containers/ntfy/default.nix` | Custom derivation building v2.17.0 from forge |
+| `mise-tasks/container-version-check` | Regex-based local nix version extraction |
+
+## Verification
+
+- [x] `dagger call build-nix --src=. --container-name=ntfy` produces a working image
+- [x] Version extractable from local `default.nix` via regex (2.17.0)
+- [x] `mise run container-version-check --all-files` passes with ntfy included
+
+## Related
+
+- [[add-container-version-sync-check]] — Parent: needs ntfy in NIX_PACKAGE_MAP
+- [[harden-zot-registry]] — Root goal

docs/how-to/zot/pin-container-versions.md

@@ -0,0 +1,53 @@
+---
+title: Pin Container Versions
+modified: 2026-02-20
+tags:
+  - how-to
+  - containers
+  - ci
+  - zot
+---
+
+# Pin Container Versions
+
+Ensure every container has an explicit, parseable version declaration so that [[add-container-version-sync-check]] has something to validate against.
+
+## Context
+
+Discovered during analysis of [[adopt-commit-based-container-tags]]: containers needed a uniform, parseable version declaration for the sync check. Most containers already had version ARGs (miniflux, navidrome, ntfy, etc.), but with inconsistent naming (`NAVIDROME_VERSION`, `MINIFLUX_VERSION`, etc.), and several containers (devpi, cv, quartz, nettest) had none.
+
+## What Was Done
+
+Every container Dockerfile now declares `ARG CONTAINER_APP_VERSION=X.Y.Z` as its first ARG, providing a uniform parsing target. Containers that use the version in build commands chain it to a semantic ARG:
+
+```dockerfile
+ARG CONTAINER_APP_VERSION=v0.60.3
+ARG NAVIDROME_VERSION=${CONTAINER_APP_VERSION}
+```
+
+Specific changes:
+- **devpi**: Pinned devpi-server==6.19.1 and devpi-web==5.0.1
+- **cv**: `CONTAINER_APP_VERSION=1.0.3` (matches latest Forgejo package release)
+- **quartz**: `CONTAINER_APP_VERSION=1.28.2` (pinned nginx:1.28.2-alpine base)
+- **nettest**: `CONTAINER_APP_VERSION=0.1.0` (internal, no upstream)
+- **All others**: Existing versions carried forward with new uniform ARG pattern
+
+## Key Files
+
+| File | Change |
+|------|--------|
+| `containers/*/Dockerfile` | Add `ARG CONTAINER_APP_VERSION` to all 13 containers |
+| `service-versions.yaml` | Populate `current-version` for devpi, cv, docs |
+
+## Verification
+
+- [x] Every container Dockerfile has `ARG CONTAINER_APP_VERSION=X.Y.Z`
+- [x] ARG chaining tested with Docker build (nginx:1.28.2-alpine)
+- [x] devpi container pins pip package versions
+- [x] cv version matches Forgejo package release (1.0.3)
+- [x] quartz pins nginx base image to stable (1.28.2)
+
+## Related
+
+- [[add-container-version-sync-check]] — Parent: needs parseable versions for sync check
+- [[adopt-commit-based-container-tags]] — Grandparent goal

docs/reference/tools/dagger.md

@@ -1,6 +1,6 @@
 ---
 title: Dagger
-modified: 2026-02-12
+modified: 2026-02-20
 tags:
   - reference
   - ci-cd
@@ -27,7 +27,10 @@ Build engine for BlumeOps CI/CD pipelines. Replaces shell-based build scripts wi
 |----------|-----------|-------------|
 | `build` | `(src, container_name) → Container` | Build a container from `containers/<name>/Dockerfile` |
 | `publish` | `(src, container_name, version, registry?) → str` | Build and push to registry (default: `registry.ops.eblu.me`) |
+| `build_nix` | `(src, container_name) → File` | Build a nix container from `containers/<name>/default.nix`, return docker-archive tarball |
+| `nix_version` | `(package) → str` | Extract the version of a nixpkgs package |
 | `build_docs` | `(src, version) → File` | Build Quartz docs site, return docs tarball |
+| `flake_lock` | `(src, flake_path?) → File` | Resolve flake inputs, return updated `flake.lock` |
 
 ## CLI Examples
 
@@ -44,6 +47,12 @@ dagger call --interactive build --src=. --container-name=devpi
 # Publish a container to zot
 dagger call publish --src=. --container-name=devpi --version=v1.1.0
 
+# Build a nix container (no local nix required)
+dagger call build-nix --src=. --container-name=nettest export --path=./nettest.tar.gz
+
+# Check a nixpkgs package version
+dagger call nix-version --package=authentik
+
 # Build docs tarball locally
 dagger call build-docs --src=. --version=dev export --path=./docs-dev.tar.gz
 

mise-tasks/container-version-check

@@ -0,0 +1,255 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["pyyaml>=6.0", "rich>=13.0.0", "typer>=0.15.0"]
+# ///
+#MISE description="Validate container version consistency across Dockerfiles, nix derivations, and service-versions.yaml"
+#USAGE flag "--all-files" help="Check all containers, not just changed ones"
+"""Validate that container versions are consistent across all declaration sites.
+
+For each container directory under containers/, checks:
+1. Any Dockerfile must declare ARG CONTAINER_APP_VERSION=<value>
+2. Any default.nix must produce a version (via dagger call nix-version)
+3. At least one build file (Dockerfile or default.nix) must exist
+4. A matching entry in service-versions.yaml must exist with non-null current-version
+5. All resolved versions from (1), (2), and (4) must agree
+
+By default, only checks containers whose files differ from main.
+Pass --all-files to check every container.
+
+Usage:
+  mise run container-version-check              # changed containers only
+  mise run container-version-check --all-files  # all containers
+"""
+
+import re
+import shutil
+import subprocess
+from pathlib import Path
+
+import typer
+import yaml
+from rich.console import Console
+from rich.table import Table
+
+REPO_ROOT = Path(__file__).parent.parent
+CONTAINERS_DIR = REPO_ROOT / "containers"
+SERVICE_VERSIONS_FILE = REPO_ROOT / "service-versions.yaml"
+
+# Containers that are utility/test images, not tracked services
+BLACKLIST = {"kubectl", "nettest"}
+
+# Container dir name → service-versions.yaml name (when they differ)
+CONTAINER_TO_SERVICE = {
+    "quartz": "docs",
+    "kiwix-serve": "kiwix",
+}
+
+# Container dir name → nixpkgs package name for dagger nix-version.
+# Used for containers that use an unmodified nixpkgs package (version matches upstream).
+# Containers with local overrides (e.g. ntfy) declare version in default.nix
+# and are detected automatically via NIX_VERSION_PATTERN.
+NIX_PACKAGE_MAP = {
+    "authentik": "authentik",
+}
+
+VERSION_ARG_PATTERN = re.compile(r"^ARG\s+CONTAINER_APP_VERSION=(\S+)", re.MULTILINE)
+NIX_VERSION_PATTERN = re.compile(r'^\s*version\s*=\s*"([^"]+)"\s*;', re.MULTILINE)
+
+app = typer.Typer()
+console = Console()
+
+
+def strip_v(version: str) -> str:
+    """Strip leading 'v' prefix for comparison."""
+    return version.lstrip("v")
+
+
+def changed_containers() -> set[str] | None:
+    """Return container names with changes vs main, or None on git failure."""
+    result = subprocess.run(
+        ["git", "diff", "--name-only", "main...HEAD"],
+        capture_output=True,
+        text=True,
+        cwd=REPO_ROOT,
+    )
+    if result.returncode != 0:
+        return None
+
+    names: set[str] = set()
+    sv_changed = False
+    for line in result.stdout.splitlines():
+        if line.startswith("containers/"):
+            parts = line.split("/")
+            if len(parts) >= 2:
+                names.add(parts[1])
+        if line == "service-versions.yaml":
+            sv_changed = True
+
+    # If service-versions.yaml changed, check all containers
+    if sv_changed:
+        return None
+
+    return names
+
+
+def get_nix_version(container_name: str, nix_file: Path) -> str | None:
+    """Extract nix package version. Tries local nix file first, then dagger."""
+    # Try extracting version declared directly in the nix file (local overrides)
+    match = NIX_VERSION_PATTERN.search(nix_file.read_text())
+    if match:
+        return match.group(1)
+
+    # Fall back to dagger for unmodified nixpkgs packages
+    pkg = NIX_PACKAGE_MAP.get(container_name)
+    if pkg is None:
+        return None
+
+    if not shutil.which("dagger"):
+        return None
+
+    result = subprocess.run(
+        ["dagger", "-m", ".dagger", "call", "nix-version", f"--package={pkg}"],
+        capture_output=True,
+        text=True,
+        cwd=REPO_ROOT,
+    )
+    if result.returncode != 0:
+        return None
+
+    return result.stdout.strip().splitlines()[-1].strip()
+
+
+@app.command()
+def main(
+    all_files: bool = typer.Option(False, "--all-files", help="Check all containers, not just changed ones"),
+) -> None:
+    """Validate container version consistency."""
+    # Determine which containers to check
+    if all_files:
+        scope = None  # check all
+    else:
+        scope = changed_containers()  # None means check all (fallback)
+
+    # Load service versions
+    data = yaml.safe_load(SERVICE_VERSIONS_FILE.read_text())
+    services = {svc["name"]: svc for svc in data.get("services", [])}
+
+    errors: list[tuple[str, str]] = []
+    results: list[dict] = []
+
+    for container_dir in sorted(CONTAINERS_DIR.iterdir()):
+        if not container_dir.is_dir():
+            continue
+
+        name = container_dir.name
+        if name in BLACKLIST:
+            continue
+        if scope is not None and name not in scope:
+            continue
+
+        dockerfile = container_dir / "Dockerfile"
+        nix_file = container_dir / "default.nix"
+        has_dockerfile = dockerfile.exists()
+        has_nix = nix_file.exists()
+
+        versions: dict[str, str] = {}
+        entry = {
+            "name": name,
+            "has_dockerfile": has_dockerfile,
+            "has_nix": has_nix,
+            "versions": versions,
+        }
+        results.append(entry)
+
+        # Rule 3: at least one build file
+        if not has_dockerfile and not has_nix:
+            errors.append((name, "No Dockerfile or default.nix found"))
+            continue
+
+        # Rule 1: Dockerfile must declare CONTAINER_APP_VERSION
+        if has_dockerfile:
+            match = VERSION_ARG_PATTERN.search(dockerfile.read_text())
+            if match:
+                versions["dockerfile"] = match.group(1)
+            else:
+                errors.append((name, "Dockerfile missing ARG CONTAINER_APP_VERSION"))
+
+        # Rule 2: nix derivation must produce a version
+        if has_nix:
+            nix_ver = get_nix_version(name, nix_file)
+            if nix_ver is not None:
+                versions["nix"] = nix_ver
+            elif name in NIX_PACKAGE_MAP:
+                errors.append((name, "Failed to extract nix version via dagger"))
+
+        # Rule 4: service-versions.yaml entry with non-null version
+        svc_name = CONTAINER_TO_SERVICE.get(name, name)
+        svc = services.get(svc_name)
+        if svc is None:
+            errors.append((name, f"No entry '{svc_name}' in service-versions.yaml"))
+        elif svc.get("current-version") is None:
+            errors.append((name, f"Null current-version for '{svc_name}' in service-versions.yaml"))
+        else:
+            versions["service-versions"] = str(svc["current-version"])
+
+        # Rule 5: all resolved versions must match
+        if len(versions) >= 2:
+            normalized = {src: strip_v(v) for src, v in versions.items()}
+            unique = set(normalized.values())
+            if len(unique) > 1:
+                detail = ", ".join(f"{src}={v}" for src, v in sorted(versions.items()))
+                errors.append((name, f"Version mismatch: {detail}"))
+
+    # Output
+    console.print("[bold]Container Version Sync Check[/bold]")
+    if scope is not None:
+        console.print(f"Scope: {len(scope)} container(s) changed vs main")
+    else:
+        console.print("Scope: all containers")
+    console.print()
+
+    if results:
+        table = Table(show_header=True, header_style="bold")
+        table.add_column("Container")
+        table.add_column("Build")
+        table.add_column("Versions")
+        table.add_column("Status")
+
+        for entry in results:
+            name = entry["name"]
+            build_parts = []
+            if entry["has_dockerfile"]:
+                build_parts.append("dockerfile")
+            if entry["has_nix"]:
+                build_parts.append("nix")
+
+            ver_parts = [f"{src}={v}" for src, v in sorted(entry["versions"].items())]
+            has_error = any(e[0] == name for e in errors)
+            status = "[red]FAIL[/red]" if has_error else "[green]OK[/green]"
+
+            table.add_row(
+                name,
+                "+".join(build_parts),
+                ", ".join(ver_parts) or "—",
+                status,
+            )
+
+        console.print(table)
+        console.print()
+
+    if errors:
+        console.print(f"[bold red]{len(errors)} error(s):[/bold red]")
+        for name, msg in errors:
+            console.print(f"  {name}: {msg}")
+        console.print()
+        raise typer.Exit(code=1)
+
+    if not results:
+        console.print("[dim]No containers to check.[/dim]")
+    else:
+        console.print("[bold green]All container versions are consistent![/bold green]")
+
+
+if __name__ == "__main__":
+    app()

service-versions.yaml

@@ -136,60 +136,66 @@ services:
 
   # --- Hybrid (custom container + ArgoCD) ---
 
+  - name: authentik
+    type: hybrid
+    last-reviewed: null
+    current-version: "2025.10.1"
+    upstream-source: https://github.com/goauthentik/authentik/releases
+
   - name: navidrome
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "v0.60.3"
     upstream-source: https://github.com/navidrome/navidrome/releases
 
   - name: miniflux
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "2.2.17"
     upstream-source: https://github.com/miniflux/v2/releases
 
   - name: teslamate
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "v2.2.0"
     upstream-source: https://github.com/teslamate-org/teslamate/releases
 
   - name: transmission
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "4.0.6-r4"
     upstream-source: https://github.com/transmission/transmission/releases
 
   - name: kiwix
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "3.8.1"
     upstream-source: https://github.com/kiwix/kiwix-tools/releases
 
   - name: devpi
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "6.19.1"
     upstream-source: https://github.com/devpi/devpi/releases
 
   - name: cv
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "1.0.3"
     upstream-source: null
     notes: Personal static site, no upstream
 
   - name: docs
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "1.28.2"
     upstream-source: https://github.com/jackyzha0/quartz/releases
-    notes: Quartz static site generator
+    notes: Quartz static site generator; container version tracks nginx base
 
   - name: forgejo-runner
     type: hybrid
     last-reviewed: null
-    current-version: null
+    current-version: "0.19.11"
     upstream-source: https://code.forgejo.org/forgejo/runner/releases
 
   # --- Ansible native ---