eblume/kingfisher
1
0
Fork 0
forked from mirrors/kingfisher
Code Pull requests Activity Actions

Fixed broken windows build script

Browse source
This commit is contained in:
Mick Grove 2025-07-29 10:12:40 -07:00
commit d648756a03
5 changed files with 131 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

26
README.md
View file

@ -43,19 +43,33 @@ brew install kingfisher
Pre-built binaries are also available on the [Releases](https://github.com/mongodb/kingfisher/releases) section of this page.
You can also install using [ubi](https://github.com/houseabsolute/ubi), which downloads the correct binary for your platform:
```bash
# Linux, macOS
curl --silent --location \
https://raw.githubusercontent.com/houseabsolute/ubi/master/bootstrap/bootstrap-ubi.sh | \
sh && \
ubi --project mongodb/kingfisher --in "$HOME/bin"
```
```powershell
# Windows
powershell -exec bypass -c "Invoke-WebRequest -URI 'https://raw.githubusercontent.com/houseabsolute/ubi/master/bootstrap/bootstrap-ubi.ps1' -UseBasicParsing | Invoke-Expression" && ubi --project mongodb/kingfisher --in .
```
This installs `ubi` and then places the `kingfisher` executable in `~/bin` on Unix-like
systems (or the current directory on Windows).
Or you may compile for your platform via `make`:
```bash
# NOTE: Requires Docker
make linux
```
```bash
# macOS
# macOS --- must build from a macOS host
make darwin
```
```bash
# Windows x64 --- requires building from a Windows host with Visual Studio installed
./buildwin.bat -force
```
@ -67,10 +81,8 @@ make darwin-all # builds both x64 and arm64
make all # builds for every OS and architecture supported
```
### Run Kingfisher in Docker
Run the dockerized Kingfisher container:
```bash
# GitHub Container Registry

123
buildwin.bat
View file

@ -1,42 +1,42 @@
@echo off
REM ---------------------------------------------------------------------------
REM Build a Windows-x64 release of Kingfisher and package it with checksums.
REM This script builds a Windows x64 release binary and creates a tarball with checksum.
REM It requires vcpkg to be installed at root of C: drive (https://github.com/microsoft/vcpkg).
REM This script will install Rust (using chocolatey) if it is not already installed.
REM
REM Call with -force to clone and bootstrap vcpkg if it is not found
REM
REM • Installs Hyperscan statically via vcpkg so vectorscan-rs-sys can link
REM against hs.lib.
REM • Installs Rust (via Chocolatey) if missing.
REM • Call with -force to clone & bootstrap vcpkg if it isn’t found.
REM ---------------------------------------------------------------------------
setlocal enabledelayedexpansion
setlocal
REM ── Project name ────────────────────────────────────────────────────────────
REM Set your Cargo project name manually here if desired:
set "PROJECT_NAME=kingfisher"
REM ── Require Windows ─────────────────────────────────────────────────────────
REM Optional check for OS:
if NOT "%OS%"=="Windows_NT" (
echo This script must be run on Windows.
exit /b 1
)
REM ── Locate MSVC toolchain ───────────────────────────────────────────────────
if "%VCINSTALLDIR%"=="" (
echo VCINSTALLDIR not set — attempting auto-detection...
echo VCINSTALLDIR not set - attempting auto-detection…
for %%P in (
"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC"
"C:\Program Files\Microsoft Visual Studio\2022\Professional\VC"
"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC"
"C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC"
) do if exist "%%~P\Auxiliary\Build\vcvars64.bat" (
set "VCINSTALLDIR=%%~P"
echo Found Visual C++ Build Tools at: %%~P
goto :vc_found
) do (
if exist "%%~P\Auxiliary\Build\vcvars64.bat" (
set "VCINSTALLDIR=%%~P"
echo Found Visual C++ Build Tools at: %%~P
goto :vc_found
)
)
echo ERROR: Could not find a suitable Visual Studio installation.
echo Install “Desktop development with C++” or set VCINSTALLDIR.
exit /b 1
)
:vc_found
REM Strip trailing backslash if present
if "%VCINSTALLDIR:~-1%"=="\" set "VCINSTALLDIR=%VCINSTALLDIR:~0,-1%"
echo Initialising MSVC environment…
@ -45,95 +45,96 @@ call "%VCINSTALLDIR%\Auxiliary\Build\vcvars64.bat" || (
exit /b 1
)
REM ── Locate or bootstrap vcpkg ───────────────────────────────────────────────
where vcpkg.exe >nul 2>&1
REM Locate vcpkg.exe
where vcpkg.exe >nul 2>nul
if %ERRORLEVEL% NEQ 0 (
if exist "%HOMEDRIVE%\vcpkg\vcpkg.exe" (
set "VCPKG_EXE=%HOMEDRIVE%\vcpkg\vcpkg.exe"
echo Found vcpkg at: %VCPKG_EXE%
) else if "%~1"=="-force" (
echo Cloning and bootstrapping vcpkg…
if exist "%HOMEDRIVE%\vcpkg" rmdir /s /q "%HOMEDRIVE%\vcpkg"
git clone https://github.com/microsoft/vcpkg.git "%HOMEDRIVE%\vcpkg"
pushd "%HOMEDRIVE%\vcpkg"
call .\bootstrap-vcpkg.bat || (echo ERROR: vcpkg bootstrap failed.&exit /b 1)
set "VCPKG_EXE=%CD%\vcpkg.exe"
popd
) else (
echo ERROR: vcpkg not found. Install it or rerun with -force.
exit /b 1
if "%~1"=="-force" (
echo Cloning and bootstrapping vcpkg...
if exist "%HOMEDRIVE%\vcpkg" (
rmdir /s /q "%HOMEDRIVE%\vcpkg"
)
git clone https://github.com/microsoft/vcpkg.git "%HOMEDRIVE%\vcpkg"
pushd "%HOMEDRIVE%\vcpkg"
dir
call .\bootstrap-vcpkg.bat
set "VCPKG_EXE=%CD%\vcpkg.exe"
popd
echo Installed vcpkg at: %VCPKG_EXE%
) else (
echo ERROR: vcpkg not found. Please install it or re-run script with -force.
exit /b 1
)
)
) else (
for /f "tokens=*" %%i in ('where vcpkg.exe') do set "VCPKG_EXE=%%i"
for /f "tokens=*" %%i in ('where vcpkg.exe') do (
set "VCPKG_EXE=%%i"
goto :found_vcpkg
)
:found_vcpkg
echo Found vcpkg at: %VCPKG_EXE%
)
REM ── Ensure LOCALAPPDATA has a drive letter (GitHub Actions quirk) ───────────
REM Check if LOCALAPPDATA starts with a drive letter, if not set it to APPDATA
if /I not "%LOCALAPPDATA:~1,1%"==":" (
echo LOCALAPPDATA lacks drive letter; pointing it at APPDATA.
echo LOCALAPPDATA does not start with a drive letter. Setting it to APPDATA.
set "LOCALAPPDATA=%APPDATA%"
)
REM ── Install Hyperscan statically ────────────────────────────────────────────
set "VCPKG_TRIPLET=x64-windows-static"
echo Installing Hyperscan (%VCPKG_TRIPLET%) via vcpkg…
call "%VCPKG_EXE%" install hyperscan:%VCPKG_TRIPLET% --clean-after-build || (
echo ERROR: vcpkg install failed.
exit /b 1
)
echo Installing hyperscan via vcpkg...
set
"%HOMEDRIVE%\vcpkg\vcpkg.exe" install hyperscan:x64-windows
set "LIBHS_NO_PKG_CONFIG=1"
REM Path hints for vectorscan-rs-sys
set "HYPERSCAN_ROOT=%HOMEDRIVE%\vcpkg\installed\%VCPKG_TRIPLET%"
REM Point vectorscan-rs-sys at the Hyperscan install from vcpkg
set "HYPERSCAN_ROOT=%HOMEDRIVE%\vcpkg\installed\x64-windows"
set "LIB=%HYPERSCAN_ROOT%\lib;%LIB%"
set "INCLUDE=%HYPERSCAN_ROOT%\include;%INCLUDE%"
REM Fallback: rename vectorscan.lib -> hs.lib if vcpkg changed the name
if not exist "%HYPERSCAN_ROOT%\lib\hs.lib" if exist "%HYPERSCAN_ROOT%\lib\vectorscan.lib" (
copy "%HYPERSCAN_ROOT%\lib\vectorscan.lib" "%HYPERSCAN_ROOT%\lib\hs.lib" >nul
)
REM ── Install Rust toolchain if absent ────────────────────────────────────────
where rustc.exe >nul 2>&1
REM Check for Rust, install if missing
where rustc.exe >nul 2>nul
if %ERRORLEVEL% NEQ 0 (
echo Installing Rust via Chocolatey…
choco install rust-ms -y || exit /b 1
choco install cmake -y --installargs "ADD_CMAKE_TO_PATH=System" || exit /b 1
echo Installing Rust...
choco install rust-ms -y
choco install cmake -y --installargs "ADD_CMAKE_TO_PATH=System"
call refreshenv
) else (
echo Rust is already installed.
)
REM ── Build ───────────────────────────────────────────────────────────────────
echo Building for Windows x64…
echo Building for Windows x64...
cargo build --release --target x86_64-pc-windows-msvc || (
echo ERROR: Cargo build failed.
echo Cargo build failed.
exit /b 1
)
REM ── Package & checksum ──────────────────────────────────────────────────────
echo Generating CHECKSUM.txt…
echo Generating CHECKSUM.txt...
powershell -Command ^
"Get-FileHash .\target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe -Algorithm SHA256 | Out-File .\target\x86_64-pc-windows-msvc\release\CHECKSUM.txt"
if not exist "target\release" mkdir "target\release"
copy /Y "target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe" "target\release\" >nul
copy /Y "target\x86_64-pc-windows-msvc\release\CHECKSUM.txt" "target\release\CHECKSUM-windows-x64.txt" >nul
copy /Y "target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe" "target\release\" >nul
copy /Y "target\x86_64-pc-windows-msvc\release\CHECKSUM.txt" "target\release\CHECKSUM-windows-x64.txt" >nul
pushd target\release
cd target\release
echo Creating archive: %PROJECT_NAME%-windows-x64.zip
if exist "%PROJECT_NAME%-windows-x64.zip" del /f /q "%PROJECT_NAME%-windows-x64.zip"
powershell -Command "Compress-Archive -Path '%PROJECT_NAME%.exe','CHECKSUM-windows-x64.txt' -DestinationPath '%PROJECT_NAME%-windows-x64.zip' -Force"
if exist "%PROJECT_NAME%-windows-x64.zip" (
REM -- append the ZIP’s SHA-256 to the existing checksum file ----
certutil -hashfile "%PROJECT_NAME%-windows-x64.zip" SHA256 >> "CHECKSUM-windows-x64.txt"
echo Created: %PROJECT_NAME%-windows-x64.zip
) else (
echo ERROR: Archive not created.
)
echo Archives in target\release:
dir /b *.zip 2>nul || echo None found.
popd
endlocal
exit /b 0
exit /b 0

19
install-precommit-hook.sh
View file

@ -1,8 +1,9 @@
#!/usr/bin/env bash
#
# Install a Git pre-commit hook that runs `kingfisher scan`.
# --global → install once for all repos using core.hooksPath
# --force → overwrite an existing pre-commit hook
# Install a Git pre‑commit hook that runs `kingfisher scan`.
#
# --global → install once for all repos via core.hooksPath
# --force → overwrite an existing pre‑commit hook
#
set -euo pipefail
@ -48,7 +49,7 @@ fi
cat >"$HOOK_PATH" <<'HOOK'
#!/usr/bin/env bash
# Git pre-commit hook to run Kingfisher on staged changes
# Git pre‑commit hook to run Kingfisher on staged changes
set -euo pipefail
if ! command -v kingfisher >/dev/null 2>&1; then
@ -59,6 +60,14 @@ fi
git diff --cached --name-only -z | \
xargs -0 --no-run-if-empty kingfisher scan --only-valid --no-update-check
status=$?
# ────────────────────────────────────────────────────────────────
# Treat Kingfisher exit‑code 200 as success (map → 0)
# ────────────────────────────────────────────────────────────────
if [[ $status -eq 200 ]]; then
status=0
fi
if [[ $status -ne 0 ]]; then
echo "Kingfisher detected secrets in staged files. Commit aborted." >&2
exit $status
@ -66,4 +75,4 @@ fi
HOOK
chmod +x "$HOOK_PATH"
echo "Pre-commit hook installed to $HOOK_PATH ($MODE mode)"
echo "Pre‑commit hook installed to $HOOK_PATH ($MODE mode)"

46
src/decompress.rs
View file

@ -91,16 +91,28 @@ fn handle_tar_archive_streaming(
let out_path = base_dir.join(&path_in_tar);
if let Some(parent) = out_path.parent() {
fs::create_dir_all(parent)?;
if let Err(e) = fs::create_dir_all(parent) {
tracing::debug!("failed to create directory {}: {}", parent.display(), e);
continue;
}
}
if !is_safe_extract_path(&out_path) {
tracing::warn!("unsafe tar path: {}", out_path.display());
continue;
}
let mut out_file = fs::File::create(&out_path)?;
std::io::copy(&mut entry, &mut out_file)?;
entries_on_disk.push((logical_path, out_path));
match fs::File::create(&out_path) {
Ok(mut out_file) => {
if let Err(e) = std::io::copy(&mut entry, &mut out_file) {
tracing::debug!("failed to extract {}: {}", out_path.display(), e);
continue;
}
entries_on_disk.push((logical_path, out_path));
}
Err(e) => {
tracing::debug!("failed to create file {}: {}", out_path.display(), e);
continue;
}
}
}
}
Ok(CompressedContent::ArchiveFiles(entries_on_disk))
@ -122,16 +134,28 @@ fn handle_zip_archive_streaming(
let out_path = base_dir.join(&name_in_zip);
if let Some(parent) = out_path.parent() {
fs::create_dir_all(parent)?;
if let Err(e) = fs::create_dir_all(parent) {
println!("****************failed to create directory {}: {}", parent.display(), e);
continue;
}
}
if !is_safe_extract_path(&out_path) {
tracing::warn!("unsafe zip path: {}", out_path.display());
println!("****************unsafe zip path: {}", out_path.display());
continue;
}
let mut out_file = fs::File::create(&out_path)?;
std::io::copy(&mut zipped_file, &mut out_file)?;
entries_on_disk.push((logical_path, out_path));
match fs::File::create(&out_path) {
Ok(mut out_file) => {
if let Err(e) = std::io::copy(&mut zipped_file, &mut out_file) {
println!("****************failed to extract {}: {}", out_path.display(), e);
continue;
}
entries_on_disk.push((logical_path, out_path));
}
Err(e) => {
println!("****************failed to create file {}: {}", out_path.display(), e);
continue;
}
}
}
}
Ok(CompressedContent::ArchiveFiles(entries_on_disk))

2
src/scanner/docker.rs
View file

@ -238,7 +238,7 @@ impl Docker {
_ => "bin",
};
let digest = layer.sha256_digest();
let file_name = format!("layer_{digest}.{ext}");
let file_name = format!("layer_{}.{}", digest.replace(':', "_"), ext);
let tmp_path = out_dir.join(file_name);
let mut tmp = std::fs::File::create(&tmp_path)?;
tmp.write_all(&layer.data)?;