From 62aeb07795b2ecff1594336e201e53b96f4b08b3 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 22 Jul 2025 08:24:45 -0700 Subject: [PATCH] Fixed permission issue with cargo-deb running after docker based linux build --- Makefile | 21 ++- README.md | 3 +- evergreen.yml | 447 -------------------------------------------------- 3 files changed, 13 insertions(+), 458 deletions(-) delete mode 100644 evergreen.yml diff --git a/Makefile b/Makefile index 8975568..7a9da74 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,8 @@ endif ARCHIVE_CMD = $(TAR_CMD) $(TAR_OPTS) SUDO_CMD := $(shell command -v sudo 2>/dev/null) +HOST_UID := $(shell id -u) +HOST_GID := $(shell id -g) .PHONY: default help create-dockerignore ubuntu-x64 ubuntu-arm64 linux-x64 linux-arm64 darwin-arm64 darwin-x64 windows-x64 windows \ linux darwin all list-archives check-docker check-rust clean tests @@ -241,7 +243,8 @@ endif linux-x64: check-docker create-dockerignore @mkdir -p target/release docker run --platform linux/amd64 --rm \ - -v "$$(pwd):/src" -w /src rust:1.88-alpine sh -eu -c '\ + -e HOST_UID=$(HOST_UID) -e HOST_GID=$(HOST_GID) \ + -v "$$(pwd):/src" -w /src rust:1.88-alpine sh -eu -c '\ apk add --no-cache \ musl-dev \ gcc g++ make cmake pkgconfig \ @@ -262,8 +265,9 @@ linux-x64: check-docker create-dockerignore cargo build --release --target x86_64-unknown-linux-musl && \ cd target/x86_64-unknown-linux-musl/release && \ find "./$(PROJECT_NAME)" -type f -executable \ - -not -name "*.d" -not -name "*.rlib" \ - -exec sha256sum {} \; > CHECKSUM.txt \ + -not -name "*.d" -not -name "*.rlib" \ + -exec sha256sum {} \; > CHECKSUM.txt && \ + chown -R $$HOST_UID:$$HOST_GID /src/target \ ' @cd target/release && \ rm -rf $(PROJECT_NAME)-linux-x64.tgz && \ @@ -278,7 +282,8 @@ linux-x64: check-docker create-dockerignore linux-arm64: check-docker create-dockerignore @mkdir -p target/release docker run --platform linux/arm64 --rm \ - -v "$$(pwd):/src" -w /src rust:1.88-alpine sh -eu -c '\ + -e HOST_UID=$(HOST_UID) -e HOST_GID=$(HOST_GID) \ + -v "$$(pwd):/src" -w /src rust:1.88-alpine sh -eu -c '\ apk add --no-cache \ musl-dev \ gcc g++ make cmake pkgconfig \ @@ -300,8 +305,9 @@ linux-arm64: check-docker create-dockerignore \ cd target/aarch64-unknown-linux-musl/release && \ find "./$(PROJECT_NAME)" -type f -executable \ - -not -name "*.d" -not -name "*.rlib" \ - -exec sha256sum {} \; > CHECKSUM.txt \ + -not -name "*.d" -not -name "*.rlib" \ + -exec sha256sum {} \; > CHECKSUM.txt && \ + chown -R $$HOST_UID:$$HOST_GID /src/target \ ' @cd target/release && \ rm -rf $(PROJECT_NAME)-linux-arm64.tgz && \ @@ -430,6 +436,3 @@ notices: @echo "Generating third-party notices..." @cargo install cargo-bundle-licenses @cargo bundle-licenses --format yaml --output THIRD_PARTY_NOTICES - -evergreen-patch: - @evergreen patch --project kingfisher --variants all --tasks build \ No newline at end of file diff --git a/README.md b/README.md index fba254f..a6142ae 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,7 @@ Kingfisher extends Nosey Parker with live secret validation via cloud-provider A - **Language‑Aware Accuracy**: AST parsing in 20+ languages via Tree‑Sitter reduces contextless regex matches. see [docs/PARSING.md](/docs/PARSING.md) - **Built-In Validation**: Hundreds of built-in detection rules, many with live-credential validators that call the relevant service APIs (AWS, Azure, GCP, Stripe, etc.) to confirm a secret is active. You can extend or override the library by adding YAML-defined rules on the command line—see [docs/RULES.md](/docs/RULES.md) for details - **Git History Scanning**: Scan local repos, remote GitHub/GitLab orgs/users, or arbitrary GitHub/GitLab repos - -# Getting Started +- **Baseline Support:** Generate and manage baseline files to ignore known secrets and report only newly introduced ones. See ([docs/BASELINE.md](docs/BASELINE.md)) for details. ## Installation diff --git a/evergreen.yml b/evergreen.yml deleted file mode 100644 index a705ac6..0000000 --- a/evergreen.yml +++ /dev/null @@ -1,447 +0,0 @@ -project: kingfisher -exec_timeout_secs: 10800 - -aliases: - - name: patch - variants: [all] - tasks: [all] - -github_pr_aliases: - - variant: ".*" - task: ".*" - -expansions: - &cdn - aws_cdn_bucket: cdn-origin-kingfisher - aws_cdn_role: arn:aws:iam::119629040606:role/s3-access.cdn-origin-kingfisher - cdn_prefix: kingfisher - -######################## -# SHARED FUNCTIONS # -######################## -functions: - - # -- fetch repo -------------------------------------------------- - fetch-source: - - command: git.get_project - params: - directory: kingfisher - shallow_clone: true - - setup-base-linux: - - command: shell.exec - params: - working_dir: kingfisher - shell: bash - script: | - set -euo pipefail - sudo apt-get update -qq - # Only a tiny set of tools the Makefile assumes exist; - # the Makefile will install everything else it needs. - sudo apt-get install -y --no-install-recommends \ - build-essential curl ca-certificates xz-utils pkg-config - - extract-version: - # Parse the value - - command: shell.exec - params: - working_dir: kingfisher - shell: bash - script: | - set -euo pipefail - # Grab the first `version = "…"` line, strip everything except the value - V=$(grep -m1 '^version *= *"' Cargo.toml | cut -d'"' -f2) - # Write it as a one-line YAML file understood by Evergreen - echo "version: \"$V\"" > version.yml - - # Load it into the task’s expansions - - command: expansions.update # ← writes the new variable - params: - file: kingfisher/version.yml # one-key YAML file we just created - ignore_missing_file: false - - # -- install rustup + Rust 1.85 on macOS ------------------------ - setup-rust-macos: - - command: shell.exec - params: - working_dir: kingfisher - shell: bash - script: | - set -euo pipefail - - REQUIRED=1.85.0 - - # install rustup if missing - if ! command -v rustup >/dev/null 2>&1; then - echo "⬇️ installing rustup…" - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | sh -s -- -y --profile minimal --default-toolchain none - export PATH="$HOME/.cargo/bin:$PATH" - hash -r - fi - - # ensure the requested tool-chain is present & default - if ! rustup toolchain list | grep -q "$REQUIRED"; then - rustup toolchain install "$REQUIRED" - fi - rustup default "$REQUIRED" - - # expose rustc version - rustc --version - - - # -- run the Makefile target passed via $BUILD ------------------ - make-build: - - command: shell.exec - params: - working_dir: kingfisher - shell: bash - add_expansions_to_env: true - script: | - set -euo pipefail - [ -f "$HOME/.cargo/env" ] && source "$HOME/.cargo/env" - export PATH="$HOME/.cargo/bin:$PATH" - echo "▶ make $BUILD" - make "$BUILD" - - package-unix: - - command: shell.exec - params: - working_dir: kingfisher - shell: bash - script: | - set -euo pipefail - mkdir -p dist - cp target/release/kingfisher-* dist/ || true - - # build Evergreen artifact manifest - cd dist - { - echo '[' - first=1 - for f in kingfisher-*; do - [ -f "$f" ] || continue - [ $first -eq 0 ] && echo ',' - first=0 - printf ' { "name": "%s", "link": "%s" }' "$f" "$f" - done - echo - echo ']' - } > artifacts.json - ls -lh - - - command: attach.artifacts - params: - working_dir: kingfisher/dist - files: ["artifacts.json"] - - macos-sign-notify: - - command: shell.exec - params: - working_dir: kingfisher - shell: bash - add_expansions_to_env: true - script: | - set -euo pipefail - shopt -s nullglob - - # One-liner: “give me the first macOS tarball if it exists” - archive=$(printf '%s\n' dist/kingfisher-darwin-*.tgz | head -n 1 || true) - - if [[ -z "$archive" || ! -f "$archive" ]]; then - echo "‼️ No macOS archive to notarize – skipping" - exit 0 - fi - - echo "✍️ Notarizing $archive …" - python evergreen/macos_notary.py "$archive" - - - - command: attach.artifacts - params: - working_dir: kingfisher/dist # where artifacts.json sits - files: ["artifacts.json"] - - package-windows: - - command: shell.exec - params: - working_dir: kingfisher - shell: powershell.exe - script: | - $ErrorActionPreference = 'Stop' - New-Item -ItemType Directory -Force dist | Out-Null - Copy-Item target\release\kingfisher-windows-x64.zip dist\ -Force - Write-Host "✓ Copied ZIP into dist\" - - # rebuild artifacts.json with the single ZIP we just copied - Get-ChildItem dist\*.zip -File | - ForEach-Object { [pscustomobject]@{name=$_.Name; link=$_.Name} } | - ConvertTo-Json -Depth 2 | - Set-Content dist\artifacts.json -Encoding ascii - Write-Host "✓ Re-generated artifacts.json" - - - command: attach.artifacts - params: - working_dir: kingfisher/dist - files: ["artifacts.json"] - - - # -- run `make tests` on Bash platforms ------------------------ - run-tests-unix: - - command: shell.exec - params: - working_dir: kingfisher - shell: bash - add_expansions_to_env: true - script: | - set -euo pipefail - [ -f "$HOME/.cargo/env" ] && source "$HOME/.cargo/env" - export PATH="$HOME/.cargo/bin:$PATH" - echo "▶ make tests" - make tests - - # -- run release tests on Windows ------------------------------ - run-tests-windows: - - command: shell.exec - params: - working_dir: kingfisher - shell: powershell.exe - script: | - $ErrorActionPreference = 'Stop' - - # Always pull the latest PATH from the registry - if (Get-Command Update-SessionEnvironment -ErrorAction SilentlyContinue) { - Update-SessionEnvironment - } else { - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - Update-SessionEnvironment - } - - # tell Git to use Windows’ native certificate store - git config --global http.sslBackend schannel - - # Ensure tools are present - if (-not (Get-Command cmake.exe -ErrorAction SilentlyContinue)) { - choco install cmake -y --installargs 'ADD_CMAKE_TO_PATH=System' - Update-SessionEnvironment - } - - if (-not (Get-Command git.exe -ErrorAction SilentlyContinue)) { - choco install git -y --params "/GitOnlyOnPath" - Update-SessionEnvironment - } - - if (-not (Get-Command cargo-nextest -ErrorAction SilentlyContinue)) { - cargo install --locked cargo-nextest - } - - # Run the release test-suite - Write-Host "▶ cargo nextest run --release" - cargo nextest run --release --workspace --all-targets - - # Install GitHub CLI ------------------------------------------------- - install-gh-cli: - # Linux (RPM/DEB not shown – easiest is the official static tarball) - - command: shell.exec - params: - shell: bash - script: | - set -euo pipefail - if [[ "$OS_FAMILY" == "windows" ]]; then exit 0; fi - if command -v gh >/dev/null 2>&1; then exit 0; fi - curl -sSL https://github.com/cli/cli/releases/latest/download/gh_$(uname -m | sed 's/aarch64/arm64/;s/x86_64/amd64/')_linux.tar.gz \ - | tar -xz --strip-components=1 -C /usr/local/bin gh_*/bin/gh - gh --version - - # macOS via brew - - command: shell.exec - params: - run_on_distro: macos-14 # only runs on macOS tasks - shell: bash - script: | - set -euo pipefail - if command -v gh >/dev/null 2>&1; then exit 0; fi - brew install gh - gh --version - - # Windows via choco - - command: shell.exec - params: - run_on_distro: windows-2022-large - shell: powershell.exe - script: | - if (Get-Command gh.exe -ErrorAction SilentlyContinue) { exit 0 } - choco install gh -y - gh --version - - - # Upload (or create) GitHub draft release --------------------------- - upload-github-release: - - command: shell.exec - params: - working_dir: kingfisher - add_expansions_to_env: true # so $version is available - shell: bash - script: | - set -euo pipefail - export GH_TOKEN="${GITHUB_TOKEN:?GITHUB_TOKEN not set}" - - TAG="v${version}" - TITLE="Kingfisher ${version}" - ASSET_DIR="dist" - - # Does the draft already exist? - if gh release view "$TAG" --json isDraft >/dev/null 2>&1; then - echo "📝 Draft release $TAG exists." - else - echo "➕ Creating draft release $TAG" - gh release create "$TAG" --draft --title "$TITLE" --notes "Automated draft for $TAG" - fi - - # Get list of already-uploaded asset names - EXISTING=$(gh release view "$TAG" --json assets -q '.assets[].name' | sort) - - for FILE in $ASSET_DIR/*.{tgz,tar.gz,zip} ; do - [[ -f "$FILE" ]] || continue - NAME=$(basename "$FILE") - if grep -qxF "$NAME" <<< "$EXISTING"; then - echo "✔ $NAME already attached – skipping" - else - echo "⬆ Uploading $NAME" - gh release upload "$TAG" "$FILE" --clobber - fi - done - - publish-cdn: - # ZIP archives - - command: s3.put - params: - working_dir: kingfisher - role_arn: ${aws_cdn_role} - bucket: ${aws_cdn_bucket} - region: us-east-1 - local_files_include_filter_prefix: kingfisher - local_files_include_filter: - - dist/*.zip - - release/*.zip - remote_file: ${cdn_prefix}/${version}/ - content_type: application/zip # ← non-blank value - permissions: public-read - visibility: public - skip_existing: true - - # .tgz archives - - command: s3.put - params: - working_dir: kingfisher - role_arn: ${aws_cdn_role} - bucket: ${aws_cdn_bucket} - region: us-east-1 - local_files_include_filter_prefix: kingfisher - local_files_include_filter: - - dist/*.tgz - - dist/*.tar.gz - - release/*.tgz - - release/*.tar.gz - remote_file: ${cdn_prefix}/${version}/ - content_type: application/x-gzip - permissions: public-read - visibility: public - skip_existing: true - - - -######################## -# TASKS # -######################## -tasks: - - - name: build-linux-docker - commands: - - func: fetch-source - - func: extract-version - - func: make-build - - func: package-unix - # - func: install-gh-cli - # - func: upload-github-release - - # ---------- macOS (x64 / arm64) ------------------------------- - - name: build-macos - commands: - - func: fetch-source - - func: setup-rust-macos - - func: extract-version - - func: make-build - - func: run-tests-unix - - func: package-unix - - func: macos-sign-notify # notarize the macOS app - # - func: install-gh-cli - # - func: upload-github-release - - # ---------- Windows ------------------------------------------- - - name: build-windows - commands: - - func: fetch-source - - command: shell.exec - params: - working_dir: kingfisher - shell: cmd.exe - add_expansions_to_env: true - script: | - .\buildwin.bat -force - - func: extract-version - - func: run-tests-windows - - func: package-windows - # - func: install-gh-cli - # - func: upload-github-release - -######################## -# BUILD VARIANTS # -######################## -buildvariants: - - # ---- Linux ---------------------------------------------------- - - name: linux-x64-docker - display_name: "🐧 Linux x64" - run_on: rhel80-docker-medium - expansions: - <<: *cdn - BUILD: linux-x64 - tasks: [build-linux-docker] - - - name: linux-arm64-docker - display_name: "🐧 Linux arm64" - run_on: ubuntu2404-arm64-latest-small - expansions: - <<: *cdn - BUILD: linux-arm64 - tasks: [build-linux-docker] - - # ---- macOS ---------------------------------------------------- - - name: darwin-x64 - display_name: "🍎 macOS x64" - run_on: macos-14 - expansions: - <<: *cdn - BUILD: darwin-x64 - tasks: [build-macos] - - - name: darwin-arm64 - display_name: "🍎 macOS arm64" - run_on: macos-14-arm64-gui - expansions: - <<: *cdn - BUILD: darwin-arm64 - tasks: [build-macos] - - # ---- Windows -------------------------------------------------- - - name: windows-x64 - display_name: "🪟 Windows x64" - run_on: windows-2022-large - expansions: - <<: *cdn - BUILD: "windows-x64" - garasign_jsign_image: 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-jsign:latest - garasign_jsign_username: ${GARASIGN_USER1_USERNAME|} - garasign_jsign_password: ${GARASIGN_USER1_PASSWORD|} - tasks: [build-windows]