Build custom Kingfisher container from sporked deploy branch #318
12 changed files with 10247 additions and 8 deletions
12
CLAUDE.md
12
CLAUDE.md
|
|
@ -121,6 +121,18 @@ from upstream.
|
|||
|
||||
Ask user to mirror on forge first, then clone to `~/code/3rd/<project>/`.
|
||||
|
||||
### Sporked Projects
|
||||
|
||||
Some mirrored projects are "sporked" — a floating-branch soft-fork strategy
|
||||
where local patches are continuously rebased on top of upstream. See
|
||||
[[spork-strategy]] and [[create-a-spork]] for the full methodology.
|
||||
|
||||
Sporked projects live in `~/code/3rd/<project>/` with three remotes:
|
||||
`origin` (eblume/ fork on forge), `mirror` (mirrors/ on forge), `upstream`
|
||||
(canonical). The `blumeops` branch is the default; `deploy` merges everything.
|
||||
|
||||
Create a new spork: `mise run spork-create <mirror-name>`
|
||||
|
||||
## Task Discovery
|
||||
|
||||
```fish
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ spec:
|
|||
targetRevision: main
|
||||
path: argocd/manifests/kingfisher
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
server: https://ringtail.tail8d86e.ts.net:6443
|
||||
namespace: kingfisher
|
||||
syncPolicy:
|
||||
syncOptions:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ spec:
|
|||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: kingfisher
|
||||
image: ghcr.io/mongodb/kingfisher:kustomized
|
||||
image: registry.ops.eblu.me/blumeops/kingfisher:kustomized
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
|
|
@ -28,7 +28,9 @@ spec:
|
|||
|
||||
kingfisher scan gitea \
|
||||
--api-url https://forge.ops.eblu.me/api/v1/ \
|
||||
--clone-url-base https://forge.ops.eblu.me/ \
|
||||
--user eblume \
|
||||
--all-organizations \
|
||||
--repo-type all \
|
||||
--no-update-check \
|
||||
--tls-mode lax \
|
||||
|
|
|
|||
|
|
@ -11,5 +11,5 @@ resources:
|
|||
- cronjob.yaml
|
||||
|
||||
images:
|
||||
- name: ghcr.io/mongodb/kingfisher
|
||||
newTag: "1.91.0"
|
||||
- name: registry.ops.eblu.me/blumeops/kingfisher
|
||||
newTag: v165768b-5cd32f8-nix
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
# NFS PersistentVolume for Kingfisher secret scan reports
|
||||
# Reuses the same sifaka:/volume1/reports share as Prowler
|
||||
# NFS rules already configured for indri
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
|
|
|
|||
10002
containers/kingfisher/Cargo.lock
generated
Normal file
10002
containers/kingfisher/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
114
containers/kingfisher/default.nix
Normal file
114
containers/kingfisher/default.nix
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
# Nix-built Kingfisher secret scanner
|
||||
# Built from upstream main + sporked feature branches applied as patches.
|
||||
# Runs on ringtail (amd64) via nix-container-builder runner.
|
||||
#
|
||||
# How it works:
|
||||
# 1. builtins.fetchGit fetches upstream and feature branches at eval time
|
||||
# 2. diff generates patches from upstream→feature in a sandboxed derivation
|
||||
# 3. buildRustPackage applies patches to the upstream source and builds
|
||||
#
|
||||
# To update:
|
||||
# 1. Update upstreamRev to the new main SHA
|
||||
# 2. Rebase feature branches onto new main (mirror-sync does this daily)
|
||||
# 3. Update feature revs to the new rebased SHAs
|
||||
# 4. Update Cargo.lock if dependencies changed
|
||||
#
|
||||
# The upstream rev must be an ancestor of each feature rev.
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
|
||||
let
|
||||
version = "165768b";
|
||||
repoUrl = "https://forge.ops.eblu.me/eblume/kingfisher.git";
|
||||
|
||||
upstreamRev = "165768b5ca9a85c2e8c64bed19bb197e82b45360";
|
||||
|
||||
features = [
|
||||
{
|
||||
name = "clone-url-base";
|
||||
ref = "feature/upstream/clone-url-base";
|
||||
rev = "4d5ce57a12650ec54c41b909f8623a1d395aa0a9";
|
||||
}
|
||||
];
|
||||
|
||||
# Fetch upstream source at the pinned rev (eval-time, network access)
|
||||
upstreamSrc = builtins.fetchGit {
|
||||
url = repoUrl;
|
||||
ref = "main";
|
||||
rev = upstreamRev;
|
||||
};
|
||||
|
||||
# Fetch each feature branch source and generate a patch against upstream
|
||||
featurePatches = map (f:
|
||||
let
|
||||
featureSrc = builtins.fetchGit {
|
||||
url = repoUrl;
|
||||
ref = f.ref;
|
||||
rev = f.rev;
|
||||
};
|
||||
in
|
||||
pkgs.runCommand "spork-${f.name}.patch" {
|
||||
nativeBuildInputs = [ pkgs.diffutils pkgs.gnused ];
|
||||
} ''
|
||||
diff -ruN --no-dereference ${upstreamSrc} ${featureSrc} \
|
||||
| sed -e 's|${upstreamSrc}/|a/|g' -e 's|${featureSrc}/|b/|g' \
|
||||
> $out || true
|
||||
''
|
||||
) features;
|
||||
|
||||
kingfisher = pkgs.rustPlatform.buildRustPackage {
|
||||
pname = "kingfisher";
|
||||
inherit version;
|
||||
src = upstreamSrc;
|
||||
|
||||
patches = featurePatches;
|
||||
|
||||
# Cargo.lock is not committed upstream; we vendor a copy alongside default.nix
|
||||
cargoLock.lockFile = ./Cargo.lock;
|
||||
|
||||
# Patch the source to include Cargo.lock (buildRustPackage needs it in-tree)
|
||||
postPatch = ''
|
||||
cp ${./Cargo.lock} Cargo.lock
|
||||
chmod +w Cargo.lock
|
||||
'';
|
||||
|
||||
nativeBuildInputs = with pkgs; [
|
||||
cmake
|
||||
pkg-config
|
||||
python3
|
||||
];
|
||||
|
||||
buildInputs = with pkgs; [
|
||||
boost
|
||||
openssl
|
||||
];
|
||||
|
||||
# Don't run tests — they need network access for wiremock
|
||||
doCheck = false;
|
||||
|
||||
meta = with pkgs.lib; {
|
||||
description = "Secret detection and live validation tool";
|
||||
homepage = "https://github.com/mongodb/kingfisher";
|
||||
license = licenses.asl20;
|
||||
mainProgram = "kingfisher";
|
||||
};
|
||||
};
|
||||
in
|
||||
|
||||
pkgs.dockerTools.buildLayeredImage {
|
||||
name = "blumeops/kingfisher";
|
||||
contents = [
|
||||
kingfisher
|
||||
pkgs.cacert
|
||||
pkgs.git
|
||||
pkgs.tzdata
|
||||
];
|
||||
|
||||
config = {
|
||||
Entrypoint = [ "${kingfisher}/bin/kingfisher" ];
|
||||
Env = [
|
||||
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||
"TZDIR=${pkgs.tzdata}/share/zoneinfo"
|
||||
];
|
||||
User = "65534";
|
||||
};
|
||||
}
|
||||
1
docs/changelog.d/feature-kingfisher-container.feature.md
Normal file
1
docs/changelog.d/feature-kingfisher-container.feature.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
Build custom Kingfisher container from sporked deploy branch, replacing upstream image with locally-built version including --clone-url-base patch.
|
||||
|
|
@ -60,6 +60,7 @@ Note that a cron-triggered workflow is especially dangerous: it requires no user
|
|||
|
||||
- [[create-a-spork]] — initial setup with `mise run spork-create`
|
||||
- [[manage-spork-branches]] — feature branches, the deploy branch, handling rebase conflicts
|
||||
- [[build-spork-container]] — building reproducible containers from pinned SHAs
|
||||
|
||||
## See also
|
||||
|
||||
|
|
|
|||
101
docs/how-to/configuration/build-spork-container.md
Normal file
101
docs/how-to/configuration/build-spork-container.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
---
|
||||
title: Build a Spork Container
|
||||
modified: 2026-03-29
|
||||
last-reviewed: 2026-03-29
|
||||
tags:
|
||||
- how-to
|
||||
- containers
|
||||
- git
|
||||
---
|
||||
|
||||
# Build a Spork Container
|
||||
|
||||
How to build a container image from a [[spork-strategy|sporked]] project with fully-pinned, reproducible inputs.
|
||||
|
||||
## Why not use the `deploy` branch directly?
|
||||
|
||||
The `deploy` branch is force-pushed on every mirror-sync. Building from `deploy` is not reproducible — the same build a week later fetches different code (or fails because the old commit was garbage collected).
|
||||
|
||||
Instead, spork containers use Nix to fetch upstream `main` at a pinned SHA and generate patches from feature branches at pinned SHAs. Both upstream and feature commits are on stable branches that are never force-pushed.
|
||||
|
||||
## How the Nix build works
|
||||
|
||||
The `default.nix` uses `builtins.fetchGit` (eval-time, network access) to fetch two source trees:
|
||||
|
||||
1. **Upstream source** at the pinned `upstreamRev` on `main`
|
||||
2. **Feature branch source** at the pinned `rev` for each feature
|
||||
|
||||
Then a sandboxed `diff -ruN` generates a patch from upstream→feature for each feature branch. `buildRustPackage` applies the patches to the upstream source and builds.
|
||||
|
||||
This means:
|
||||
- The upstream rev persists forever (main only fast-forwards)
|
||||
- Feature revs are on your branches (you control them)
|
||||
- No dependency on the `deploy` branch
|
||||
- Fully reproducible given the same revs
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Sporked project set up (see [[create-a-spork]])
|
||||
- Nix build runs on ringtail (`nix-container-builder` runner)
|
||||
|
||||
## Get the SHAs
|
||||
|
||||
```fish
|
||||
cd ~/code/3rd/kingfisher
|
||||
git fetch origin
|
||||
|
||||
# Upstream SHA (main branch)
|
||||
git rev-parse origin/main
|
||||
# e.g., 1d37d2983cd4a58c12663dd8df0e79dfe89a5d75
|
||||
|
||||
# Feature branch SHAs
|
||||
git rev-parse origin/feature/upstream/clone-url-base
|
||||
# e.g., 677c7a5d5fc42b655d38fbf95dc8b814d89ceabb
|
||||
```
|
||||
|
||||
## Update `default.nix`
|
||||
|
||||
Edit `containers/kingfisher/default.nix`:
|
||||
|
||||
- `version` — short upstream SHA (for container tag)
|
||||
- `upstreamRev` — full upstream main SHA
|
||||
- `features[].rev` — full feature branch SHA
|
||||
|
||||
If dependencies changed, update `Cargo.lock` too:
|
||||
|
||||
```fish
|
||||
cd ~/code/3rd/kingfisher
|
||||
git checkout origin/main
|
||||
cargo update
|
||||
cp Cargo.lock ~/code/personal/blumeops/containers/kingfisher/Cargo.lock
|
||||
```
|
||||
|
||||
## Build and push
|
||||
|
||||
The build is triggered via the standard container build workflow on ringtail's `nix-container-builder` runner, or manually:
|
||||
|
||||
```fish
|
||||
mise run container-build-and-release kingfisher
|
||||
```
|
||||
|
||||
## Update the deployment
|
||||
|
||||
1. Update `argocd/manifests/kingfisher/kustomization.yaml` with the new tag
|
||||
2. Update `service-versions.yaml` if the upstream SHA changed
|
||||
3. Sync the ArgoCD app
|
||||
|
||||
## Note on `CONTAINER_APP_VERSION`
|
||||
|
||||
The `default.nix` includes `version` which maps to `CONTAINER_APP_VERSION` for the `container-version-check` hook. For sporked containers this is a git SHA, not a release version. Don't confuse it with an upstream release number.
|
||||
|
||||
## Reproducibility
|
||||
|
||||
The upstream rev must be an ancestor of each feature rev. If you bump the upstream rev without rebasing your feature branches, the generated patch will conflict and the build fails — which is the correct behavior.
|
||||
|
||||
The invariant: **feature revs are descendants of the upstream rev**. Mirror-sync maintains this automatically. You just need to update the revs in `default.nix` after an upgrade.
|
||||
|
||||
## See also
|
||||
|
||||
- [[create-a-spork]] — initial spork setup
|
||||
- [[manage-spork-branches]] — feature branch workflow
|
||||
- [[kingfisher]] — first sporked project
|
||||
|
|
@ -16,7 +16,7 @@ Secret detection and live validation scanner for Forgejo repositories, using Mon
|
|||
| Property | Value |
|
||||
|----------|-------|
|
||||
| **Namespace** | `kingfisher` |
|
||||
| **Image** | `ghcr.io/mongodb/kingfisher` (see `argocd/manifests/kingfisher/kustomization.yaml` for current tag) |
|
||||
| **Image** | `registry.ops.eblu.me/blumeops/kingfisher` (see `argocd/manifests/kingfisher/kustomization.yaml` for current tag) |
|
||||
| **Schedule** | Sunday 4am (after Prowler k8s scan at 3am) |
|
||||
| **Reports** | `sifaka:/volume1/reports/kingfisher/` (NFS) |
|
||||
| **Manifests** | `argocd/manifests/kingfisher/` |
|
||||
|
|
@ -24,7 +24,7 @@ Secret detection and live validation scanner for Forgejo repositories, using Mon
|
|||
|
||||
## What it does
|
||||
|
||||
Runs as a weekly CronJob that scans all repositories in the `eblume` user on Forgejo for leaked secrets, API keys, and credentials. Produces timestamped HTML and JSON reports on the sifaka NFS share.
|
||||
Runs as a weekly CronJob that scans all Forgejo repos (eblume + all orgs) for leaked secrets, API keys, and credentials. Produces timestamped HTML reports on the sifaka NFS share. Uses `--clone-url-base` to route git clones via the internal tailnet instead of the public Fly.io proxy.
|
||||
|
||||
Uses the Forgejo/Gitea API to enumerate repos, then clones and scans each one. Validation is enabled (secrets are tested against their respective APIs to confirm they're live). Reports are HTML only.
|
||||
|
||||
|
|
@ -46,7 +46,7 @@ kubectl logs -f job/kingfisher-manual -n kingfisher --context=minikube-indri
|
|||
|
||||
## Limitations
|
||||
|
||||
- Clone URLs come from Forgejo's API response using the instance's public `ROOT_URL` (`forge.eblu.me`), so clones roundtrip through Fly.io. Mirror/org scanning is excluded for now to avoid unnecessary external bandwidth. A clone URL rewrite option would need an upstream contribution.
|
||||
- Built from a [[spork-strategy|sporked]] fork with a local `--clone-url-base` patch. See [[build-spork-container]] for the build process.
|
||||
- Only one output format per invocation. Currently producing HTML only.
|
||||
|
||||
## See also
|
||||
|
|
|
|||
|
|
@ -285,6 +285,13 @@ services:
|
|||
upstream-source: https://github.com/prowler-cloud/prowler/releases
|
||||
notes: CIS Kubernetes Benchmark scanner; weekly CronJob on minikube-indri
|
||||
|
||||
- name: kingfisher
|
||||
type: argocd
|
||||
last-reviewed: 2026-03-29
|
||||
current-version: "165768b"
|
||||
upstream-source: https://github.com/mongodb/kingfisher/releases
|
||||
notes: Secret scanner; sporked from upstream with --clone-url-base patch. Version is upstream main SHA.
|
||||
|
||||
- name: forgejo
|
||||
type: ansible
|
||||
last-reviewed: 2026-03-28
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue