From d3fbffcb84a8fd622d1f38b7348f9f0e098e4f25 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 29 Jul 2025 10:12:40 -0700 Subject: [PATCH] Fixed broken windows build script --- README.md | 26 +++++--- buildwin.bat | 123 +++++++++++++++++++------------------- install-precommit-hook.sh | 19 ++++-- src/decompress.rs | 46 ++++++++++---- src/scanner/docker.rs | 2 +- 5 files changed, 131 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 6b3728d..ad65c78 100644 --- a/README.md +++ b/README.md @@ -43,19 +43,33 @@ brew install kingfisher Pre-built binaries are also available on the [Releases](https://github.com/mongodb/kingfisher/releases) section of this page. +You can also install using [ubi](https://github.com/houseabsolute/ubi), which downloads the correct binary for your platform: + +```bash +# Linux, macOS +curl --silent --location \ + https://raw.githubusercontent.com/houseabsolute/ubi/master/bootstrap/bootstrap-ubi.sh | \ + sh && \ + ubi --project mongodb/kingfisher --in "$HOME/bin" +``` + +```powershell +# Windows +powershell -exec bypass -c "Invoke-WebRequest -URI 'https://raw.githubusercontent.com/houseabsolute/ubi/master/bootstrap/bootstrap-ubi.ps1' -UseBasicParsing | Invoke-Expression" && ubi --project mongodb/kingfisher --in . +``` + +This installs `ubi` and then places the `kingfisher` executable in `~/bin` on Unix-like +systems (or the current directory on Windows). + Or you may compile for your platform via `make`: ```bash # NOTE: Requires Docker make linux -``` -```bash -# macOS +# macOS --- must build from a macOS host make darwin -``` -```bash # Windows x64 --- requires building from a Windows host with Visual Studio installed ./buildwin.bat -force ``` @@ -67,10 +81,8 @@ make darwin-all # builds both x64 and arm64 make all # builds for every OS and architecture supported ``` - ### Run Kingfisher in Docker - Run the dockerized Kingfisher container: ```bash # GitHub Container Registry diff --git a/buildwin.bat b/buildwin.bat index 70762ec..baa2167 100644 --- a/buildwin.bat +++ b/buildwin.bat @@ -1,42 +1,42 @@ @echo off -REM --------------------------------------------------------------------------- -REM Build a Windows-x64 release of Kingfisher and package it with checksums. +REM This script builds a Windows x64 release binary and creates a tarball with checksum. +REM It requires vcpkg to be installed at root of C: drive (https://github.com/microsoft/vcpkg). +REM This script will install Rust (using chocolatey) if it is not already installed. +REM +REM Call with -force to clone and bootstrap vcpkg if it is not found REM -REM • Installs Hyperscan statically via vcpkg so vectorscan-rs-sys can link -REM against hs.lib. -REM • Installs Rust (via Chocolatey) if missing. -REM • Call with -force to clone & bootstrap vcpkg if it isn’t found. -REM --------------------------------------------------------------------------- -setlocal enabledelayedexpansion +setlocal -REM ── Project name ──────────────────────────────────────────────────────────── +REM Set your Cargo project name manually here if desired: set "PROJECT_NAME=kingfisher" -REM ── Require Windows ───────────────────────────────────────────────────────── +REM Optional check for OS: if NOT "%OS%"=="Windows_NT" ( echo This script must be run on Windows. exit /b 1 ) - -REM ── Locate MSVC toolchain ─────────────────────────────────────────────────── if "%VCINSTALLDIR%"=="" ( - echo VCINSTALLDIR not set — attempting auto-detection... + echo VCINSTALLDIR not set - attempting auto-detection… for %%P in ( "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC" "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC" "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC" - ) do if exist "%%~P\Auxiliary\Build\vcvars64.bat" ( - set "VCINSTALLDIR=%%~P" - echo Found Visual C++ Build Tools at: %%~P - goto :vc_found + ) do ( + if exist "%%~P\Auxiliary\Build\vcvars64.bat" ( + set "VCINSTALLDIR=%%~P" + echo Found Visual C++ Build Tools at: %%~P + goto :vc_found + ) ) echo ERROR: Could not find a suitable Visual Studio installation. echo Install “Desktop development with C++” or set VCINSTALLDIR. exit /b 1 ) :vc_found + +REM Strip trailing backslash if present if "%VCINSTALLDIR:~-1%"=="\" set "VCINSTALLDIR=%VCINSTALLDIR:~0,-1%" echo Initialising MSVC environment… @@ -45,95 +45,96 @@ call "%VCINSTALLDIR%\Auxiliary\Build\vcvars64.bat" || ( exit /b 1 ) -REM ── Locate or bootstrap vcpkg ─────────────────────────────────────────────── -where vcpkg.exe >nul 2>&1 +REM Locate vcpkg.exe +where vcpkg.exe >nul 2>nul if %ERRORLEVEL% NEQ 0 ( if exist "%HOMEDRIVE%\vcpkg\vcpkg.exe" ( set "VCPKG_EXE=%HOMEDRIVE%\vcpkg\vcpkg.exe" echo Found vcpkg at: %VCPKG_EXE% - ) else if "%~1"=="-force" ( - echo Cloning and bootstrapping vcpkg… - if exist "%HOMEDRIVE%\vcpkg" rmdir /s /q "%HOMEDRIVE%\vcpkg" - git clone https://github.com/microsoft/vcpkg.git "%HOMEDRIVE%\vcpkg" - pushd "%HOMEDRIVE%\vcpkg" - call .\bootstrap-vcpkg.bat || (echo ERROR: vcpkg bootstrap failed.&exit /b 1) - set "VCPKG_EXE=%CD%\vcpkg.exe" - popd ) else ( - echo ERROR: vcpkg not found. Install it or rerun with -force. - exit /b 1 + if "%~1"=="-force" ( + echo Cloning and bootstrapping vcpkg... + if exist "%HOMEDRIVE%\vcpkg" ( + rmdir /s /q "%HOMEDRIVE%\vcpkg" + ) + git clone https://github.com/microsoft/vcpkg.git "%HOMEDRIVE%\vcpkg" + pushd "%HOMEDRIVE%\vcpkg" + dir + call .\bootstrap-vcpkg.bat + set "VCPKG_EXE=%CD%\vcpkg.exe" + popd + echo Installed vcpkg at: %VCPKG_EXE% + ) else ( + echo ERROR: vcpkg not found. Please install it or re-run script with -force. + exit /b 1 + ) ) ) else ( - for /f "tokens=*" %%i in ('where vcpkg.exe') do set "VCPKG_EXE=%%i" + for /f "tokens=*" %%i in ('where vcpkg.exe') do ( + set "VCPKG_EXE=%%i" + goto :found_vcpkg + ) + :found_vcpkg echo Found vcpkg at: %VCPKG_EXE% ) -REM ── Ensure LOCALAPPDATA has a drive letter (GitHub Actions quirk) ─────────── +REM Check if LOCALAPPDATA starts with a drive letter, if not set it to APPDATA if /I not "%LOCALAPPDATA:~1,1%"==":" ( - echo LOCALAPPDATA lacks drive letter; pointing it at APPDATA. + echo LOCALAPPDATA does not start with a drive letter. Setting it to APPDATA. set "LOCALAPPDATA=%APPDATA%" ) -REM ── Install Hyperscan statically ──────────────────────────────────────────── -set "VCPKG_TRIPLET=x64-windows-static" -echo Installing Hyperscan (%VCPKG_TRIPLET%) via vcpkg… -call "%VCPKG_EXE%" install hyperscan:%VCPKG_TRIPLET% --clean-after-build || ( - echo ERROR: vcpkg install failed. - exit /b 1 -) +echo Installing hyperscan via vcpkg... +set +"%HOMEDRIVE%\vcpkg\vcpkg.exe" install hyperscan:x64-windows set "LIBHS_NO_PKG_CONFIG=1" -REM Path hints for vectorscan-rs-sys -set "HYPERSCAN_ROOT=%HOMEDRIVE%\vcpkg\installed\%VCPKG_TRIPLET%" +REM Point vectorscan-rs-sys at the Hyperscan install from vcpkg +set "HYPERSCAN_ROOT=%HOMEDRIVE%\vcpkg\installed\x64-windows" set "LIB=%HYPERSCAN_ROOT%\lib;%LIB%" set "INCLUDE=%HYPERSCAN_ROOT%\include;%INCLUDE%" -REM Fallback: rename vectorscan.lib -> hs.lib if vcpkg changed the name -if not exist "%HYPERSCAN_ROOT%\lib\hs.lib" if exist "%HYPERSCAN_ROOT%\lib\vectorscan.lib" ( - copy "%HYPERSCAN_ROOT%\lib\vectorscan.lib" "%HYPERSCAN_ROOT%\lib\hs.lib" >nul -) - -REM ── Install Rust toolchain if absent ──────────────────────────────────────── -where rustc.exe >nul 2>&1 +REM Check for Rust, install if missing +where rustc.exe >nul 2>nul if %ERRORLEVEL% NEQ 0 ( - echo Installing Rust via Chocolatey… - choco install rust-ms -y || exit /b 1 - choco install cmake -y --installargs "ADD_CMAKE_TO_PATH=System" || exit /b 1 + echo Installing Rust... + choco install rust-ms -y + choco install cmake -y --installargs "ADD_CMAKE_TO_PATH=System" call refreshenv + ) else ( echo Rust is already installed. ) -REM ── Build ─────────────────────────────────────────────────────────────────── -echo Building for Windows x64… +echo Building for Windows x64... cargo build --release --target x86_64-pc-windows-msvc || ( - echo ERROR: Cargo build failed. + echo Cargo build failed. exit /b 1 ) -REM ── Package & checksum ────────────────────────────────────────────────────── -echo Generating CHECKSUM.txt… +echo Generating CHECKSUM.txt... powershell -Command ^ "Get-FileHash .\target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe -Algorithm SHA256 | Out-File .\target\x86_64-pc-windows-msvc\release\CHECKSUM.txt" if not exist "target\release" mkdir "target\release" -copy /Y "target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe" "target\release\" >nul -copy /Y "target\x86_64-pc-windows-msvc\release\CHECKSUM.txt" "target\release\CHECKSUM-windows-x64.txt" >nul +copy /Y "target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe" "target\release\" >nul +copy /Y "target\x86_64-pc-windows-msvc\release\CHECKSUM.txt" "target\release\CHECKSUM-windows-x64.txt" >nul -pushd target\release +cd target\release echo Creating archive: %PROJECT_NAME%-windows-x64.zip if exist "%PROJECT_NAME%-windows-x64.zip" del /f /q "%PROJECT_NAME%-windows-x64.zip" powershell -Command "Compress-Archive -Path '%PROJECT_NAME%.exe','CHECKSUM-windows-x64.txt' -DestinationPath '%PROJECT_NAME%-windows-x64.zip' -Force" if exist "%PROJECT_NAME%-windows-x64.zip" ( + REM -- append the ZIP’s SHA-256 to the existing checksum file ---- certutil -hashfile "%PROJECT_NAME%-windows-x64.zip" SHA256 >> "CHECKSUM-windows-x64.txt" echo Created: %PROJECT_NAME%-windows-x64.zip ) else ( echo ERROR: Archive not created. ) + echo Archives in target\release: dir /b *.zip 2>nul || echo None found. -popd endlocal -exit /b 0 +exit /b 0 \ No newline at end of file diff --git a/install-precommit-hook.sh b/install-precommit-hook.sh index 40cb95b..923fb94 100755 --- a/install-precommit-hook.sh +++ b/install-precommit-hook.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # -# Install a Git pre-commit hook that runs `kingfisher scan`. -# --global → install once for all repos using core.hooksPath -# --force → overwrite an existing pre-commit hook +# Install a Git pre‑commit hook that runs `kingfisher scan`. +# +# --global → install once for all repos via core.hooksPath +# --force → overwrite an existing pre‑commit hook # set -euo pipefail @@ -48,7 +49,7 @@ fi cat >"$HOOK_PATH" <<'HOOK' #!/usr/bin/env bash -# Git pre-commit hook to run Kingfisher on staged changes +# Git pre‑commit hook to run Kingfisher on staged changes set -euo pipefail if ! command -v kingfisher >/dev/null 2>&1; then @@ -59,6 +60,14 @@ fi git diff --cached --name-only -z | \ xargs -0 --no-run-if-empty kingfisher scan --only-valid --no-update-check status=$? + +# ──────────────────────────────────────────────────────────────── +# Treat Kingfisher exit‑code 200 as success (map → 0) +# ──────────────────────────────────────────────────────────────── +if [[ $status -eq 200 ]]; then + status=0 +fi + if [[ $status -ne 0 ]]; then echo "Kingfisher detected secrets in staged files. Commit aborted." >&2 exit $status @@ -66,4 +75,4 @@ fi HOOK chmod +x "$HOOK_PATH" -echo "Pre-commit hook installed to $HOOK_PATH ($MODE mode)" +echo "Pre‑commit hook installed to $HOOK_PATH ($MODE mode)" diff --git a/src/decompress.rs b/src/decompress.rs index 8bc43aa..f91c8fa 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -91,16 +91,28 @@ fn handle_tar_archive_streaming( let out_path = base_dir.join(&path_in_tar); if let Some(parent) = out_path.parent() { - fs::create_dir_all(parent)?; + if let Err(e) = fs::create_dir_all(parent) { + tracing::debug!("failed to create directory {}: {}", parent.display(), e); + continue; + } } if !is_safe_extract_path(&out_path) { tracing::warn!("unsafe tar path: {}", out_path.display()); continue; } - let mut out_file = fs::File::create(&out_path)?; - std::io::copy(&mut entry, &mut out_file)?; - - entries_on_disk.push((logical_path, out_path)); + match fs::File::create(&out_path) { + Ok(mut out_file) => { + if let Err(e) = std::io::copy(&mut entry, &mut out_file) { + tracing::debug!("failed to extract {}: {}", out_path.display(), e); + continue; + } + entries_on_disk.push((logical_path, out_path)); + } + Err(e) => { + tracing::debug!("failed to create file {}: {}", out_path.display(), e); + continue; + } + } } } Ok(CompressedContent::ArchiveFiles(entries_on_disk)) @@ -122,16 +134,28 @@ fn handle_zip_archive_streaming( let out_path = base_dir.join(&name_in_zip); if let Some(parent) = out_path.parent() { - fs::create_dir_all(parent)?; + if let Err(e) = fs::create_dir_all(parent) { + println!("****************failed to create directory {}: {}", parent.display(), e); + continue; + } } if !is_safe_extract_path(&out_path) { - tracing::warn!("unsafe zip path: {}", out_path.display()); + println!("****************unsafe zip path: {}", out_path.display()); continue; } - let mut out_file = fs::File::create(&out_path)?; - std::io::copy(&mut zipped_file, &mut out_file)?; - - entries_on_disk.push((logical_path, out_path)); + match fs::File::create(&out_path) { + Ok(mut out_file) => { + if let Err(e) = std::io::copy(&mut zipped_file, &mut out_file) { + println!("****************failed to extract {}: {}", out_path.display(), e); + continue; + } + entries_on_disk.push((logical_path, out_path)); + } + Err(e) => { + println!("****************failed to create file {}: {}", out_path.display(), e); + continue; + } + } } } Ok(CompressedContent::ArchiveFiles(entries_on_disk)) diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs index 775e687..5a6daa4 100644 --- a/src/scanner/docker.rs +++ b/src/scanner/docker.rs @@ -238,7 +238,7 @@ impl Docker { _ => "bin", }; let digest = layer.sha256_digest(); - let file_name = format!("layer_{digest}.{ext}"); + let file_name = format!("layer_{}.{}", digest.replace(':', "_"), ext); let tmp_path = out_dir.join(file_name); let mut tmp = std::fs::File::create(&tmp_path)?; tmp.write_all(&layer.data)?;