diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beeec7f..81c6bb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,47 +1,121 @@ name: CI Pull Request on: - pull_request: - branches: - - main + pull_request: + branches: + - main env: + VCPKG_ROOT: C:\vcpkg + VCPKG_DOWNLOADS: C:\vcpkg\downloads + VCPKG_FEATURE_FLAGS: binarycaching + VCPKG_BINARY_SOURCES: clear;x-gha,readwrite RUST_TOOLCHAIN: "1.90" -# This workflow runs on pull requests to the main branch -# It builds the project for 2 platforms, Linux arm64 and macOS arm64, -# and runs tests for each platform. All platforms tested on merge to main jobs: - linux-arm64: - name: Linux arm64 - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - profile: minimal - override: true - - uses: swatinem/rust-cache@v2 - - name: Build (Makefile linux-arm64) - run: make ubuntu-arm64 - - name: Run tests - run: make tests - env: - CARGO_BUILD_JOBS: 1 + linux-arm64: + name: Linux arm64 + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + profile: minimal + override: true + - uses: swatinem/rust-cache@v2 + - name: Build (Makefile linux-arm64) + run: make ubuntu-arm64 + - name: Run tests + run: make tests + env: + CARGO_BUILD_JOBS: 1 - macos-arm64: - name: macOS arm64 - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - profile: minimal - override: true - - uses: swatinem/rust-cache@v2 - - name: Build (Makefile darwin-arm64) - run: make darwin-arm64 - - name: Run tests - run: make tests + macos-arm64: + name: macOS arm64 + runs-on: macos-14 + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + profile: minimal + override: true + - uses: swatinem/rust-cache@v2 + - name: Build (Makefile darwin-arm64) + run: make darwin-arm64 + - name: Run tests + run: make tests + + windows: + name: Windows x64 + runs-on: windows-latest + + # Windows-only env to keep vcpkg consistent and enable caching + env: + VCPKG_ROOT: C:\vcpkg + VCPKG_DOWNLOADS: C:\vcpkg\downloads + VCPKG_FEATURE_FLAGS: binarycaching + VCPKG_BINARY_SOURCES: clear;x-gha,readwrite + + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + profile: minimal + override: true + + # Cache vcpkg artifacts & downloads (so we only fetch PCRE once) + - name: Cache vcpkg artifacts + uses: actions/cache@v4 + with: + path: | + C:\vcpkg\buildtrees + C:\vcpkg\packages + C:\vcpkg\installed + C:\vcpkg\downloads + C:\vcpkg\archives + C:\Users\runneradmin\AppData\Local\vcpkg\archives + key: ${{ runner.os }}-vcpkg-hyperscan-542 + restore-keys: | + ${{ runner.os }}-vcpkg- + + # Ensure downloads dir exists and seed PCRE 8.45 zip from a working mirror + - name: Pre-seed PCRE 8.45 for vcpkg (bypass SourceForge redirect) + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "$env:VCPKG_DOWNLOADS" | Out-Null + $dst = Join-Path $env:VCPKG_DOWNLOADS "pcre-8.45.zip" + if (-not (Test-Path $dst)) { + Invoke-WebRequest ` + -Uri "https://versaweb.dl.sourceforge.net/project/pcre/pcre/8.45/pcre-8.45.zip" ` + -OutFile $dst -UseBasicParsing + } + Get-ChildItem $env:VCPKG_DOWNLOADS + + - uses: swatinem/rust-cache@v2 + + - name: Build + run: .\buildwin.bat + shell: cmd + + - name: Run tests + shell: pwsh + run: | + if (-not (Get-Command cargo-nextest -ErrorAction SilentlyContinue)) { + cargo install --locked cargo-nextest + } + Write-Host "▶ cargo nextest run --release --workspace --all-targets" + cargo nextest run --release --workspace --all-targets + + - name: Move artifact to dist + shell: bash + run: | + mkdir -p dist + cp target/release/kingfisher-windows-x64.zip dist/ + + - uses: actions/upload-artifact@v4 + with: + name: kingfisher-windows-x64 + path: dist/kingfisher-*windows-x64*.* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f2940cf..6d199b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,10 @@ on: branches: - main env: + VCPKG_ROOT: C:\vcpkg + VCPKG_DOWNLOADS: C:\vcpkg\downloads + VCPKG_FEATURE_FLAGS: binarycaching + VCPKG_BINARY_SOURCES: clear;x-gha,readwrite RUST_TOOLCHAIN: "1.90" jobs: @@ -198,13 +202,16 @@ jobs: C:\vcpkg\buildtrees C:\vcpkg\packages C:\vcpkg\installed - key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.cmake') }} + C:\vcpkg\downloads + C:\vcpkg\archives + C:\Users\runneradmin\AppData\Local\vcpkg\archives + key: ${{ runner.os }}-vcpkg-hyperscan-542 restore-keys: | ${{ runner.os }}-vcpkg- - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@v2 - name: Build - run: .\buildwin.bat -force + run: .\buildwin.bat shell: cmd - name: Run tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f41796..044bede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file. - Improved OpsGenie validation. - Automatically enable `--no-dedup` when `--manage-baseline` is supplied so baseline management keeps every finding. - This release is focused on further improving detection accuracy, before even attempting to validate findings. +- Updated GitHub Actions CI for Windows and buildwin.bat script ## [v1.61.0] - Fixed local filesystem scans to keep `open_path_as_is` enabled when opening Git repositories and only disable it for diff-based scans. diff --git a/buildwin.bat b/buildwin.bat index bd32e19..7311058 100644 --- a/buildwin.bat +++ b/buildwin.bat @@ -1,132 +1,133 @@ @echo off -REM This script builds a Windows x64 release binary and creates a tarball with checksum. -REM It requires vcpkg to be installed at root of C: drive (https://github.com/microsoft/vcpkg). -REM This script will install Rust (using chocolatey) if it is not already installed. -REM -REM Call with -force to clone and bootstrap vcpkg if it is not found -REM +REM ============================================================================ +REM buildwin.bat — Windows x64 release build + archive for Kingfisher +REM - Forces a single vcpkg root (C:\vcpkg) and avoids VS-integrated vcpkg +REM - Installs Hyperscan for x64-windows-static into that root +REM - Verifies hs.lib is present before building, then builds & packages +REM ============================================================================ setlocal - -REM Set your Cargo project name manually here if desired: set "PROJECT_NAME=kingfisher" +set "FORCE_VCPKG=0" +if /I "%~1"=="-force" set "FORCE_VCPKG=1" +if "%VCPKG_TRIPLET%"=="" set "VCPKG_TRIPLET=x64-windows-static" -REM Optional check for OS: +REM --- Ensure Windows --- if NOT "%OS%"=="Windows_NT" ( - echo This script must be run on Windows. - exit /b 1 + echo This script must be run on Windows. + exit /b 1 ) + +REM --- Find MSVC / init toolchain --- if "%VCINSTALLDIR%"=="" ( - echo VCINSTALLDIR not set - attempting auto-detection… - for %%P in ( - "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC" - "C:\Program Files\Microsoft Visual Studio\2022\Community\VC" - "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC" - "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC" - "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC" - ) do ( - if exist "%%~P\Auxiliary\Build\vcvars64.bat" ( - set "VCINSTALLDIR=%%~P" - echo Found Visual C++ Build Tools at: %%~P - goto :vc_found - ) + echo VCINSTALLDIR not set - attempting auto-detection… + for %%P in ( + "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC" + "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC" + "C:\Program Files\Microsoft Visual Studio\2022\Community\VC" + "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC" + "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC" + ) do ( + if exist "%%~P\Auxiliary\Build\vcvars64.bat" ( + set "VCINSTALLDIR=%%~P" + echo Found Visual C++ Build Tools at: %%~P + goto :vc_found ) - echo ERROR: Could not find a suitable Visual Studio installation. - echo Install “Desktop development with C++” or set VCINSTALLDIR. - exit /b 1 + ) + echo ERROR: Could not find a suitable Visual Studio installation. + echo Install "Desktop development with C++" or set VCINSTALLDIR. + exit /b 1 ) :vc_found - -REM Strip trailing backslash if present if "%VCINSTALLDIR:~-1%"=="\" set "VCINSTALLDIR=%VCINSTALLDIR:~0,-1%" - echo Initialising MSVC environment… call "%VCINSTALLDIR%\Auxiliary\Build\vcvars64.bat" || ( - echo ERROR: Failed to initialise MSVC toolchain. - exit /b 1 + echo ERROR: Failed to initialise MSVC toolchain. + exit /b 1 ) -REM Locate vcpkg.exe -where vcpkg.exe >nul 2>nul -if %ERRORLEVEL% NEQ 0 ( - if exist "%HOMEDRIVE%\vcpkg\vcpkg.exe" ( - set "VCPKG_EXE=%HOMEDRIVE%\vcpkg\vcpkg.exe" - echo Found vcpkg at: %VCPKG_EXE% - ) else ( - if "%~1"=="-force" ( - echo Cloning and bootstrapping vcpkg... - if exist "%HOMEDRIVE%\vcpkg" ( - rmdir /s /q "%HOMEDRIVE%\vcpkg" - ) - git clone https://github.com/microsoft/vcpkg.git "%HOMEDRIVE%\vcpkg" - pushd "%HOMEDRIVE%\vcpkg" - dir - call .\bootstrap-vcpkg.bat - set "VCPKG_EXE=%CD%\vcpkg.exe" - popd - echo Installed vcpkg at: %VCPKG_EXE% - ) else ( - echo ERROR: vcpkg not found. Please install it or re-run script with -force. - exit /b 1 - ) - ) -) else ( - for /f "tokens=*" %%i in ('where vcpkg.exe') do ( - set "VCPKG_EXE=%%i" - goto :found_vcpkg - ) - :found_vcpkg - echo Found vcpkg at: %VCPKG_EXE% -) +REM --- CRITICAL: Force our own vcpkg root (ignore VS-integrated vcpkg) ---- +set "VCPKG_ROOT=C:\vcpkg" +set "VCPKG_DISABLE_METRICS=1" -REM Check if LOCALAPPDATA starts with a drive letter, if not set it to APPDATA +REM --- Ensure LOCALAPPDATA sane for tools that use it --- if /I not "%LOCALAPPDATA:~1,1%"==":" ( - echo LOCALAPPDATA does not start with a drive letter. Setting it to APPDATA. - set "LOCALAPPDATA=%APPDATA%" + echo LOCALAPPDATA not drive-qualified; using APPDATA instead. + set "LOCALAPPDATA=%APPDATA%" ) -REM ── Install Hyperscan ------------------------------------------------------ -set "VCPKG_TRIPLET=x64-windows-static" +REM --- Find/Install vcpkg into C:\vcpkg --- +call :ensure_vcpkg || exit /b 1 +echo Using vcpkg root: "%VCPKG_ROOT%" +echo vcpkg executable: "%VCPKG_EXE%" + +REM --- Install Hyperscan into THIS root --- echo Installing Hyperscan (%VCPKG_TRIPLET%) via vcpkg... -pushd "%HOMEDRIVE%\vcpkg" REM ► work inside the vcpkg root -"%VCPKG_EXE%" install hyperscan:%VCPKG_TRIPLET% || ( - echo ERROR: vcpkg install failed. - popd - exit /b 1 +pushd "%VCPKG_ROOT%" || ( + echo ERROR: Cannot cd into "%VCPKG_ROOT%". + exit /b 1 +) +"%VCPKG_EXE%" --vcpkg-root "%VCPKG_ROOT%" install hyperscan:%VCPKG_TRIPLET% || ( + echo ERROR: vcpkg install failed. + popd + exit /b 1 ) popd -set "LIBHS_NO_PKG_CONFIG=1" -REM Point vectorscan‑rs‑sys at the Hyperscan install -set "HYPERSCAN_ROOT=%HOMEDRIVE%\vcpkg\installed\%VCPKG_TRIPLET%" -set "LIB=%HYPERSCAN_ROOT%\lib;%LIB%" +REM --- Point build to installed Hyperscan --- +set "LIBHS_NO_PKG_CONFIG=1" +set "HYPERSCAN_ROOT=%VCPKG_ROOT%\installed\%VCPKG_TRIPLET%" +set "HS_LIB_DIR=%HYPERSCAN_ROOT%\lib" +set "LIB=%HS_LIB_DIR%;%LIB%" set "INCLUDE=%HYPERSCAN_ROOT%\include;%INCLUDE%" -REM Check for Rust, install if missing +REM Verify hs.lib (or libhs.lib) +set "HS_LIB_FILE=%HS_LIB_DIR%\hs.lib" +if not exist "%HS_LIB_FILE%" if exist "%HS_LIB_DIR%\libhs.lib" set "HS_LIB_FILE=%HS_LIB_DIR%\libhs.lib" + +echo. +echo [DIAG] HYPERSCAN_ROOT = %HYPERSCAN_ROOT% +echo [DIAG] HS_LIB_DIR = %HS_LIB_DIR% +dir /b "%HS_LIB_DIR%\hs.lib" 2>nul +dir /b "%HS_LIB_DIR%\libhs.lib" 2>nul + +if not exist "%HS_LIB_FILE%" ( + echo ERROR: Hyperscan library not found under "%HS_LIB_DIR%". + echo Check that hyperscan:%VCPKG_TRIPLET% installed correctly. + exit /b 1 +) + +REM --- Ensure Rust/CMake present (fallback for local runs) --- where rustc.exe >nul 2>nul if %ERRORLEVEL% NEQ 0 ( - echo Installing Rust... - choco install rust-ms -y - choco install cmake -y --installargs "ADD_CMAKE_TO_PATH=System" - call refreshenv - + echo Installing Rust via Chocolatey... + choco install rust-ms -y + choco install cmake -y --installargs "ADD_CMAKE_TO_PATH=System" + call refreshenv ) else ( - echo Rust is already installed. + echo Rust is already installed. ) -set "RUSTFLAGS=%RUSTFLAGS% -C target-feature=+crt-static" +REM --- Build (static CRT) --- +if "%RUSTFLAGS%"=="" ( + set "RUSTFLAGS=-C target-feature=+crt-static" +) else ( + echo Using existing RUSTFLAGS: %RUSTFLAGS% +) +echo. echo Building static Windows x64 binary... cargo build --release --target x86_64-pc-windows-msvc || ( - echo Cargo build failed. - exit /b 1 + echo Cargo build failed. + exit /b 1 ) +REM --- Package & checksums --- echo Generating CHECKSUM.txt... powershell -Command ^ - "$hash = Get-FileHash '.\target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe' -Algorithm SHA256;" ^ - "$line = '{0} {1}' -f $hash.Hash, (Split-Path -Leaf $hash.Path);" ^ - "Set-Content -Path '.\target\x86_64-pc-windows-msvc\release\CHECKSUM.txt' -Value $line" + "$h=Get-FileHash '.\target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe' -Algorithm SHA256;" ^ + "$l='{0} {1}' -f $h.Hash,(Split-Path -Leaf $h.Path);" ^ + "Set-Content '.\target\x86_64-pc-windows-msvc\release\CHECKSUM.txt' $l" if not exist "target\release" mkdir "target\release" copy /Y "target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe" "target\release\" >nul @@ -138,18 +139,60 @@ if exist "%PROJECT_NAME%-windows-x64.zip" del /f /q "%PROJECT_NAME%-windows-x64. powershell -Command "Compress-Archive -Path '%PROJECT_NAME%.exe','CHECKSUM-windows-x64.txt' -DestinationPath '%PROJECT_NAME%-windows-x64.zip' -Force" if exist "%PROJECT_NAME%-windows-x64.zip" ( - REM -- append the ZIP’s SHA-256 to the existing checksum file ---- - powershell -Command ^ - "$hash = Get-FileHash '.\%PROJECT_NAME%-windows-x64.zip' -Algorithm SHA256;" ^ - "$line = '{0} {1}' -f $hash.Hash, (Split-Path -Leaf $hash.Path);" ^ - "Add-Content -Path '.\CHECKSUM-windows-x64.txt' -Value $line" - echo Created: %PROJECT_NAME%-windows-x64.zip + powershell -Command ^ + "$h=Get-FileHash '.\%PROJECT_NAME%-windows-x64.zip' -Algorithm SHA256;" ^ + "$l='{0} {1}' -f $h.Hash,(Split-Path -Leaf $h.Path);" ^ + "Add-Content '.\CHECKSUM-windows-x64.txt' $l" + echo Created: %PROJECT_NAME%-windows-x64.zip ) else ( - echo ERROR: Archive not created. + echo ERROR: Archive not created. + exit /b 1 ) echo Archives in target\release: dir /b *.zip 2>nul || echo None found. -endlocal +goto :eof + +REM ====================== helpers ============================================ +:ensure_vcpkg +REM If vcpkg.exe already exists under our chosen root, use it +if exist "%VCPKG_ROOT%\vcpkg.exe" ( + set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" + echo Found vcpkg at: %VCPKG_EXE% + exit /b 0 +) + +REM If on PATH (and not the VS one), still force install into C:\vcpkg +where vcpkg.exe >nul 2>nul +if %ERRORLEVEL%==0 ( + for /f "tokens=*" %%i in ('where vcpkg.exe') do ( + set "VCPKG_EXE=%%i" + ) + echo Found vcpkg on PATH: %VCPKG_EXE% +) + +REM Clone/bootstrap into our root if missing +if not exist "%VCPKG_ROOT%" mkdir "%VCPKG_ROOT%" +if "%FORCE_VCPKG%"=="1" if exist "%VCPKG_ROOT%" rmdir /s /q "%VCPKG_ROOT%" & mkdir "%VCPKG_ROOT%" + +if not exist "%VCPKG_ROOT%\vcpkg.exe" ( + echo Cloning and bootstrapping vcpkg into "%VCPKG_ROOT%"... + git clone https://github.com/microsoft/vcpkg.git "%VCPKG_ROOT%" || ( + echo ERROR: Failed to clone vcpkg repository. + exit /b 1 + ) + pushd "%VCPKG_ROOT%" + call .\bootstrap-vcpkg.bat || ( + popd + echo ERROR: Failed to bootstrap vcpkg. + exit /b 1 + ) + set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" + popd + echo Installed vcpkg at: %VCPKG_EXE% +) else ( + set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" +) + exit /b 0