Add custom Kingfisher container built from sporked feature branches
- Dockerfile: deterministic build from pinned CONTAINER_APP_VERSION + FEATURES - Merges named feature branches at specific SHAs for reproducibility - Switch CronJob to custom image with --clone-url-base and --all-organizations - Add kingfisher to service-versions.yaml (version tracks upstream main SHA) - Document spork container builds in new how-to card - Document spork workflow in CLAUDE.md - Update kingfisher service docs for custom image Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
99a1a49175
commit
5cd32f8885
12 changed files with 10232 additions and 8 deletions
12
CLAUDE.md
12
CLAUDE.md
|
|
@ -121,6 +121,18 @@ from upstream.
|
|||
|
||||
Ask user to mirror on forge first, then clone to `~/code/3rd/<project>/`.
|
||||
|
||||
### Sporked Projects
|
||||
|
||||
Some mirrored projects are "sporked" — a floating-branch soft-fork strategy
|
||||
where local patches are continuously rebased on top of upstream. See
|
||||
[[spork-strategy]] and [[create-a-spork]] for the full methodology.
|
||||
|
||||
Sporked projects live in `~/code/3rd/<project>/` with three remotes:
|
||||
`origin` (eblume/ fork on forge), `mirror` (mirrors/ on forge), `upstream`
|
||||
(canonical). The `blumeops` branch is the default; `deploy` merges everything.
|
||||
|
||||
Create a new spork: `mise run spork-create <mirror-name>`
|
||||
|
||||
## Task Discovery
|
||||
|
||||
```fish
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ spec:
|
|||
targetRevision: main
|
||||
path: argocd/manifests/kingfisher
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
server: https://ringtail.tail8d86e.ts.net:6443
|
||||
namespace: kingfisher
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ spec:
|
|||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: kingfisher
|
||||
image: ghcr.io/mongodb/kingfisher:kustomized
|
||||
image: registry.ops.eblu.me/blumeops/kingfisher:kustomized
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
|
|
@ -28,7 +28,9 @@ spec:
|
|||
|
||||
kingfisher scan gitea \
|
||||
--api-url https://forge.ops.eblu.me/api/v1/ \
|
||||
--clone-url-base https://forge.ops.eblu.me/ \
|
||||
--user eblume \
|
||||
--all-organizations \
|
||||
--repo-type all \
|
||||
--no-update-check \
|
||||
--tls-mode lax \
|
||||
|
|
|
|||
|
|
@ -11,5 +11,5 @@ resources:
|
|||
- cronjob.yaml
|
||||
|
||||
images:
|
||||
- name: ghcr.io/mongodb/kingfisher
|
||||
newTag: "1.91.0"
|
||||
- name: registry.ops.eblu.me/blumeops/kingfisher
|
||||
newTag: kustomized
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
# NFS PersistentVolume for Kingfisher secret scan reports
|
||||
# Reuses the same sifaka:/volume1/reports share as Prowler
|
||||
# NFS rules already configured for indri
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
|
|
|
|||
10002
containers/kingfisher/Cargo.lock
generated
Normal file
10002
containers/kingfisher/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
114
containers/kingfisher/default.nix
Normal file
114
containers/kingfisher/default.nix
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
# Nix-built Kingfisher secret scanner
|
||||
# Built from upstream main + sporked feature branches applied as patches.
|
||||
# Runs on ringtail (amd64) via nix-container-builder runner.
|
||||
#
|
||||
# How it works:
|
||||
# 1. builtins.fetchGit fetches upstream and feature branches at eval time
|
||||
# 2. diff generates patches from upstream→feature in a sandboxed derivation
|
||||
# 3. buildRustPackage applies patches to the upstream source and builds
|
||||
#
|
||||
# To update:
|
||||
# 1. Update upstreamRev to the new main SHA
|
||||
# 2. Rebase feature branches onto new main (mirror-sync does this daily)
|
||||
# 3. Update feature revs to the new rebased SHAs
|
||||
# 4. Update Cargo.lock if dependencies changed
|
||||
#
|
||||
# The upstream rev must be an ancestor of each feature rev.
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
|
||||
let
|
||||
version = "165768b";
|
||||
repoUrl = "https://forge.ops.eblu.me/eblume/kingfisher.git";
|
||||
|
||||
upstreamRev = "165768b5ca9a85c2e8c64bed19bb197e82b45360";
|
||||
|
||||
features = [
|
||||
{
|
||||
name = "clone-url-base";
|
||||
ref = "feature/upstream/clone-url-base";
|
||||
rev = "4d5ce57a12650ec54c41b909f8623a1d395aa0a9";
|
||||
}
|
||||
];
|
||||
|
||||
# Fetch upstream source at the pinned rev (eval-time, network access)
|
||||
upstreamSrc = builtins.fetchGit {
|
||||
url = repoUrl;
|
||||
ref = "main";
|
||||
rev = upstreamRev;
|
||||
};
|
||||
|
||||
# Fetch each feature branch source and generate a patch against upstream
|
||||
featurePatches = map (f:
|
||||
let
|
||||
featureSrc = builtins.fetchGit {
|
||||
url = repoUrl;
|
||||
ref = f.ref;
|
||||
rev = f.rev;
|
||||
};
|
||||
in
|
||||
pkgs.runCommand "spork-${f.name}.patch" {
|
||||
nativeBuildInputs = [ pkgs.diffutils pkgs.gnused ];
|
||||
} ''
|
||||
diff -ruN --no-dereference ${upstreamSrc} ${featureSrc} \
|
||||
| sed -e 's|${upstreamSrc}/|a/|g' -e 's|${featureSrc}/|b/|g' \
|
||||
> $out || true
|
||||
''
|
||||
) features;
|
||||
|
||||
kingfisher = pkgs.rustPlatform.buildRustPackage {
|
||||
pname = "kingfisher";
|
||||
inherit version;
|
||||
src = upstreamSrc;
|
||||
|
||||
patches = featurePatches;
|
||||
|
||||
# Cargo.lock is not committed upstream; we vendor a copy alongside default.nix
|
||||
cargoLock.lockFile = ./Cargo.lock;
|
||||
|
||||
# Patch the source to include Cargo.lock (buildRustPackage needs it in-tree)
|
||||
postPatch = ''
|
||||
cp ${./Cargo.lock} Cargo.lock
|
||||
chmod +w Cargo.lock
|
||||
'';
|
||||
|
||||
nativeBuildInputs = with pkgs; [
|
||||
cmake
|
||||
pkg-config
|
||||
python3
|
||||
];
|
||||
|
||||
buildInputs = with pkgs; [
|
||||
boost
|
||||
openssl
|
||||
];
|
||||
|
||||
# Don't run tests — they need network access for wiremock
|
||||
doCheck = false;
|
||||
|
||||
meta = with pkgs.lib; {
|
||||
description = "Secret detection and live validation tool";
|
||||
homepage = "https://github.com/mongodb/kingfisher";
|
||||
license = licenses.asl20;
|
||||
mainProgram = "kingfisher";
|
||||
};
|
||||
};
|
||||
in
|
||||
|
||||
pkgs.dockerTools.buildLayeredImage {
|
||||
name = "blumeops/kingfisher";
|
||||
contents = [
|
||||
kingfisher
|
||||
pkgs.cacert
|
||||
pkgs.git
|
||||
pkgs.tzdata
|
||||
];
|
||||
|
||||
config = {
|
||||
Entrypoint = [ "${kingfisher}/bin/kingfisher" ];
|
||||
Env = [
|
||||
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||
"TZDIR=${pkgs.tzdata}/share/zoneinfo"
|
||||
];
|
||||
User = "65534";
|
||||
};
|
||||
}
|
||||
1
docs/changelog.d/feature-kingfisher-container.feature.md
Normal file
1
docs/changelog.d/feature-kingfisher-container.feature.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
Build custom Kingfisher container from sporked deploy branch, replacing upstream image with locally-built version including --clone-url-base patch.
|
||||
|
|
@ -60,6 +60,7 @@ Note that a cron-triggered workflow is especially dangerous: it requires no user
|
|||
|
||||
- [[create-a-spork]] — initial setup with `mise run spork-create`
|
||||
- [[manage-spork-branches]] — feature branches, the deploy branch, handling rebase conflicts
|
||||
- [[build-spork-container]] — building reproducible containers from pinned SHAs
|
||||
|
||||
## See also
|
||||
|
||||
|
|
|
|||
86
docs/how-to/configuration/build-spork-container.md
Normal file
86
docs/how-to/configuration/build-spork-container.md
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
---
|
||||
title: Build a Spork Container
|
||||
modified: 2026-03-29
|
||||
last-reviewed: 2026-03-29
|
||||
tags:
|
||||
- how-to
|
||||
- containers
|
||||
- git
|
||||
---
|
||||
|
||||
# Build a Spork Container
|
||||
|
||||
How to build a container image from a [[spork-strategy|sporked]] project with fully-pinned, reproducible inputs.
|
||||
|
||||
## Why not use the `deploy` branch directly?
|
||||
|
||||
The `deploy` branch is force-pushed on every mirror-sync. Building from `deploy` is not reproducible — the same Dockerfile run a week later gives different code. Instead, spork containers build their own merge tree from explicit inputs:
|
||||
|
||||
- **`CONTAINER_APP_VERSION`** — the commit on `main` to base on (the upstream version)
|
||||
- **`FEATURES`** — space-separated `branch=sha` pairs to merge on top
|
||||
|
||||
This makes builds reproducible regardless of when they run.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Sporked project set up (see [[create-a-spork]])
|
||||
- Container build tooling (`mise run container-build-and-release`)
|
||||
|
||||
## Get the SHAs
|
||||
|
||||
```fish
|
||||
cd ~/code/3rd/kingfisher
|
||||
git fetch origin
|
||||
|
||||
# Upstream SHA (what main is based on)
|
||||
git rev-parse --short origin/main
|
||||
# e.g., 1d37d29
|
||||
|
||||
# Feature branch SHAs
|
||||
git rev-parse --short origin/feature/upstream/clone-url-base
|
||||
# e.g., 677c7a5
|
||||
```
|
||||
|
||||
## Build the container
|
||||
|
||||
```fish
|
||||
# The version in service-versions.yaml is the upstream SHA
|
||||
mise run container-build-and-release kingfisher 1d37d29 \
|
||||
--build-arg CONTAINER_APP_VERSION=1d37d29 \
|
||||
--build-arg FEATURES="feature/upstream/clone-url-base=677c7a5"
|
||||
```
|
||||
|
||||
The container tag will be `1d37d29-<blumeops-commit>`.
|
||||
|
||||
## Update the deployment
|
||||
|
||||
1. Update `argocd/manifests/kingfisher/kustomization.yaml` with the new tag
|
||||
2. Update `service-versions.yaml` if the upstream SHA changed
|
||||
3. Sync the ArgoCD app
|
||||
|
||||
## How the Dockerfile works
|
||||
|
||||
The build stage:
|
||||
|
||||
1. Clones the sporked repo from forge
|
||||
2. Checks out `main` at `CONTAINER_APP_VERSION`
|
||||
3. For each entry in `FEATURES`, fetches the branch and merges at the pinned SHA
|
||||
4. Builds from source with `cargo build --release`
|
||||
|
||||
If any merge conflicts, the build fails loudly.
|
||||
|
||||
The runtime stage is minimal: debian-slim + git + the binary.
|
||||
|
||||
## Note on `CONTAINER_APP_VERSION`
|
||||
|
||||
For most blumeops containers, `CONTAINER_APP_VERSION` is an upstream release version like `5.22.0` or `v2.19.2`. For sporked containers it's a git SHA — the upstream commit the build is based on. This is a deliberate abuse of the naming convention to satisfy the `container-version-check` hook. Don't confuse it with an upstream release number.
|
||||
|
||||
## Reproducibility guarantee
|
||||
|
||||
Given the same `CONTAINER_APP_VERSION` and `FEATURES`, the build produces identical source code regardless of what `deploy`, `blumeops`, or `main` currently look like on forge. The only external dependency is the Rust/Boost toolchain version in the `FROM` line.
|
||||
|
||||
## See also
|
||||
|
||||
- [[create-a-spork]] — initial spork setup
|
||||
- [[manage-spork-branches]] — feature branch workflow
|
||||
- [[kingfisher]] — first sporked project
|
||||
|
|
@ -16,7 +16,7 @@ Secret detection and live validation scanner for Forgejo repositories, using Mon
|
|||
| Property | Value |
|
||||
|----------|-------|
|
||||
| **Namespace** | `kingfisher` |
|
||||
| **Image** | `ghcr.io/mongodb/kingfisher` (see `argocd/manifests/kingfisher/kustomization.yaml` for current tag) |
|
||||
| **Image** | `registry.ops.eblu.me/blumeops/kingfisher` (see `argocd/manifests/kingfisher/kustomization.yaml` for current tag) |
|
||||
| **Schedule** | Sunday 4am (after Prowler k8s scan at 3am) |
|
||||
| **Reports** | `sifaka:/volume1/reports/kingfisher/` (NFS) |
|
||||
| **Manifests** | `argocd/manifests/kingfisher/` |
|
||||
|
|
@ -24,7 +24,7 @@ Secret detection and live validation scanner for Forgejo repositories, using Mon
|
|||
|
||||
## What it does
|
||||
|
||||
Runs as a weekly CronJob that scans all repositories in the `eblume` user on Forgejo for leaked secrets, API keys, and credentials. Produces timestamped HTML and JSON reports on the sifaka NFS share.
|
||||
Runs as a weekly CronJob that scans all Forgejo repos (eblume + all orgs) for leaked secrets, API keys, and credentials. Produces timestamped HTML reports on the sifaka NFS share. Uses `--clone-url-base` to route git clones via the internal tailnet instead of the public Fly.io proxy.
|
||||
|
||||
Uses the Forgejo/Gitea API to enumerate repos, then clones and scans each one. Validation is enabled (secrets are tested against their respective APIs to confirm they're live). Reports are HTML only.
|
||||
|
||||
|
|
@ -46,7 +46,7 @@ kubectl logs -f job/kingfisher-manual -n kingfisher --context=minikube-indri
|
|||
|
||||
## Limitations
|
||||
|
||||
- Clone URLs come from Forgejo's API response using the instance's public `ROOT_URL` (`forge.eblu.me`), so clones roundtrip through Fly.io. Mirror/org scanning is excluded for now to avoid unnecessary external bandwidth. A clone URL rewrite option would need an upstream contribution.
|
||||
- Built from a [[spork-strategy|sporked]] fork with a local `--clone-url-base` patch. See [[build-spork-container]] for the build process.
|
||||
- Only one output format per invocation. Currently producing HTML only.
|
||||
|
||||
## See also
|
||||
|
|
|
|||
|
|
@ -285,6 +285,13 @@ services:
|
|||
upstream-source: https://github.com/prowler-cloud/prowler/releases
|
||||
notes: CIS Kubernetes Benchmark scanner; weekly CronJob on minikube-indri
|
||||
|
||||
- name: kingfisher
|
||||
type: argocd
|
||||
last-reviewed: 2026-03-29
|
||||
current-version: "165768b"
|
||||
upstream-source: https://github.com/mongodb/kingfisher/releases
|
||||
notes: Secret scanner; sporked from upstream with --clone-url-base patch. Version is upstream main SHA.
|
||||
|
||||
- name: forgejo
|
||||
type: ansible
|
||||
last-reviewed: 2026-03-28
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue