diff --git a/.forgejo/workflows/release-iso.yml b/.forgejo/workflows/release-iso.yml index a30e291..162fa16 100644 --- a/.forgejo/workflows/release-iso.yml +++ b/.forgejo/workflows/release-iso.yml @@ -12,8 +12,8 @@ name: Build and release ISO # MIRROR_TOKEN — GitHub personal access token with repo scope # GPG_PRIVATE_KEY — armoured secret key for the dedicated "BOS Release Signing" # identity (releases@breadway.dev); public half is committed -# at KEYS.asc for verifying ISO SHA256SUMS only. That key -# does not sign the [breadway] pacman repo. No passphrase +# at KEYS.asc. Signs ISO SHA256SUMS here; the same secret +# signs the [breadway] repo in signed-repo.yml. No passphrase # (CI-only key, access controlled via the Forgejo secret # store). diff --git a/.forgejo/workflows/signed-repo.yml b/.forgejo/workflows/signed-repo.yml new file mode 100644 index 0000000..3ef8a65 --- /dev/null +++ b/.forgejo/workflows/signed-repo.yml @@ -0,0 +1,52 @@ +name: Publish signed [breadway] repo + +# Host job on hestia (no container:) so it can write /srv/breadway-dl, same +# as bakery releases. breadlock package.yml uses archlinux:latest and cannot +# see host /srv — do not add container: here. +# +# Collects breadlock + the ISO AUR republishes from the Forgejo Arch +# registry, detach-signs each .pkg.tar.zst, repo-add -s, publishes +# https://dl.breadway.dev/arch/x86_64/. Does not PUT to the registry +# (existing packaging workflows keep doing that). Does not flip ISO SigLevel. +# +# Required secret: GPG_PRIVATE_KEY (same BOS release key as release-iso.yml). + +on: + workflow_dispatch: + repository_dispatch: + types: [publish-signed-repo] + workflow_run: + workflows: + - Build and publish calamares + - Build and publish bibata-cursor-theme + - Build and publish powerlevel10k + - Build and publish yay-bin + types: [completed] + +concurrency: + group: signed-repo + cancel-in-progress: false + +jobs: + publish: + if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} + runs-on: [self-hosted, hestia] + steps: + - name: Clone repository + run: | + set -euo pipefail + REF="${GITHUB_REF_NAME:-main}" + rm -rf src + git clone --depth 1 --branch "$REF" \ + "https://git.breadway.dev/${GITHUB_REPOSITORY}.git" src + + - name: Sign packages and publish repo + env: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + run: | + set -euo pipefail + if [ -z "${GPG_PRIVATE_KEY:-}" ]; then + echo "GPG_PRIVATE_KEY secret is missing; refusing to publish an unsigned [breadway] repo." >&2 + exit 1 + fi + bash src/scripts/ci-publish-signed-repo.sh diff --git a/README.md b/README.md index f0987b8..1a833b6 100644 --- a/README.md +++ b/README.md @@ -96,11 +96,12 @@ bos/ ├── scripts/ │ ├── ci-stage-bakery.py # CI: minisign-verified index → $LAPTOP_HOME │ ├── ci-verify-bake.sh # CI: read-only checks before mkarchiso +│ ├── ci-publish-signed-repo.sh # CI: signed [breadway] repo → /srv/breadway-dl/arch │ └── smoke-test.sh ├── docs/ │ ├── hardware.md # Mesa only, NVIDIA, grub-btrfs recovery -│ └── signed-repo.md # future dl.breadway.dev/arch signing -├── .forgejo/workflows/ # CI: AUR republish + tagged ISO release +│ └── signed-repo.md # dl.breadway.dev/arch signing +├── .forgejo/workflows/ # CI: AUR republish + signed repo + tagged ISO ├── build-local.sh # native ISO build for this machine ├── README.md └── DESIGN.md # historical plan diff --git a/docs/signed-repo.md b/docs/signed-repo.md index 112e254..02b5340 100644 --- a/docs/signed-repo.md +++ b/docs/signed-repo.md @@ -3,16 +3,24 @@ Today the ISO's `[Breadway.os.git.breadway.dev]` section is `SigLevel = Never`. That is TLS-only integrity: packages come from Forgejo's Arch registry, which does **not** serve pacman-compatible database -signatures. `KEYS.asc` signs **ISO `SHA256SUMS` only** — it is not imported -as a pacman repo key. Do not flip `SigLevel` to `Required` on that section -until a signed repo exists and has been verified; Required without -signatures breaks the ISO and every installed system. +signatures. `KEYS.asc` signs **ISO `SHA256SUMS`** and, once published, the +`dl.breadway.dev/arch` database. It is **not** imported as a pacman repo key +on the ISO yet. Do not flip `SigLevel` to `Required` on that section until +a signed repo exists and has been verified; Required without signatures +breaks the ISO and every installed system. The signed repo belongs at `https://dl.breadway.dev/arch`, not on Forgejo's -registry. +registry. Forgejo publishing stays as it is (`package.yml` / packaging +workflows PUT unsigned `.pkg.tar.zst` so existing Never installs keep +working). ## Stand up `dl.breadway.dev/arch` +CI job: **Publish signed `[breadway]` repo** +(`.forgejo/workflows/signed-repo.yml`), host runner on hestia — **no +container**, so it can write `/srv/breadway-dl` like bakery releases. +breadlock `package.yml` uses `archlinux:latest` and cannot see host `/srv`. + Use the same release-signing key already in CI: - Public half: [`KEYS.asc`](../KEYS.asc) @@ -20,6 +28,7 @@ Use the same release-signing key already in CI: `releases@breadway.dev`) - Private half: the `GPG_PRIVATE_KEY` Forgejo secret (armoured secret key, no passphrase). Same secret `release-iso.yml` uses to sign `SHA256SUMS`. + The workflow **fails** if this secret is missing. Layout (example for `x86_64`): @@ -33,53 +42,114 @@ https://dl.breadway.dev/arch/x86_64/ breadway.files.sig ``` -Build the database **and sign it** with `repo-add -s`: +On disk: `/srv/breadway-dl/arch/x86_64/` (nginx already serves +`/srv/breadway-dl` as `https://dl.breadway.dev/`). + +The job collects the current ISO `[breadway]` set from the Forgejo Arch +registry (breadlock + calamares, zen-browser-bin, bibata-cursor-theme-bin, +zsh-theme-powerlevel10k, yay-bin). Leftover bakery-channel pacman packages +still sitting in that registry are **not** copied. Optional +`BREADWAY_PKG_DIR` on the runner overrides individual files. + +Then it detach-signs each `.pkg.tar.zst` as a **binary** sidecar (pacman +wants `.sig`, not armoured `.asc`) and builds the database with +`repo-add -s`: ```sh export GNUPGHOME=/tmp/gnupg-breadway-repo mkdir -m 700 -p "$GNUPGHOME" printf '%s\n' "$GPG_PRIVATE_KEY" | gpg --batch --import -cd /srv/dl.breadway.dev/arch/x86_64 +gpg --batch --yes --local-user releases@breadway.dev \ + --detach-sign breadlock--1-x86_64.pkg.tar.zst +# → breadlock--1-x86_64.pkg.tar.zst.sig + +cd /srv/breadway-dl/arch/x86_64 repo-add -s -k releases@breadway.dev breadway.db.tar.gz *.pkg.tar.zst ``` `repo-add -s` writes `breadway.db.tar.gz.sig` (and the `.files` pair). Pacman fetches `
.db` + `
.db.sig` from `Server`. -Package signatures are separate from the database signature. Detach-sign -each `.pkg.tar.zst` as a **binary** sidecar (pacman wants `.sig`, not -armoured `.asc`): +## Dispatch the workflow + +Forgejo UI: **Actions → "Publish signed [breadway] repo" → Run workflow**. +Select this branch (`feature/signed-repo`) until it is on `main`. + +API (`workflow_dispatch`): ```sh -gpg --batch --yes --local-user releases@breadway.dev \ - --detach-sign breadlock--1-x86_64.pkg.tar.zst -# → breadlock--1-x86_64.pkg.tar.zst.sig +curl -fsS -X POST \ + -H "Authorization: token ${RELEASE_TOKEN}" \ + -H "Content-Type: application/json" \ + "https://git.breadway.dev/api/v1/repos/Breadway/bos/actions/workflows/signed-repo.yml/dispatches" \ + -d '{"ref":"feature/signed-repo"}' ``` +After merge, use `"ref":"main"`. + +It also runs after the in-repo AUR republish workflows complete +(`calamares` / `bibata` / `powerlevel10k` / `yay-bin`). breadlock lives in +another repo; that job can fire this one with `repository_dispatch` event +`publish-signed-repo` (or dispatch from the UI after a breadlock tag). + +## Verify + +Confirm the signed db is actually served **before** touching ISO +`SigLevel` or `Server`: + +```sh +curl -fsSIL https://dl.breadway.dev/arch/x86_64/breadway.db +curl -fsSIL https://dl.breadway.dev/arch/x86_64/breadway.db.sig +``` + +Both must be HTTP 200. A 404 on `breadway.db.sig` means do **not** flip +`SigLevel` to `Required`. + +Import `KEYS.asc` and check the detached signatures: + +```sh +gpg --import KEYS.asc +curl -fsSL -o /tmp/breadway.db https://dl.breadway.dev/arch/x86_64/breadway.db +curl -fsSL -o /tmp/breadway.db.sig https://dl.breadway.dev/arch/x86_64/breadway.db.sig +gpg --verify /tmp/breadway.db.sig /tmp/breadway.db +``` + +On a throwaway Arch box (not the ISO tree): + +```sh +sudo pacman-key --add KEYS.asc +sudo pacman-key --lsign-key 56203B86A110695AE7F310934AF3323D678EB5E2 + +# Temporary /etc/pacman.conf snippet — do not commit this to the ISO: +# [breadway] +# SigLevel = Required +# Server = https://dl.breadway.dev/arch/$arch + +sudo pacman -Sy +``` + +`pacman -Sy` must fetch `breadway.db` + `breadway.db.sig` without +"missing or invalid signature". Then `pacman -Si breadlock` (and the AUR +republishes) should list the `[breadway]` section. + ## breadlock `package.yml` sidecar [`breadlock` `package.yml`](https://git.breadway.dev/Breadway/breadlock/src/branch/main/.forgejo/workflows/package.yml) -already `makepkg`s and PUTs the archive at Forgejo's registry. When the -signed repo exists, that job can also emit the sidecar and publish both -files to `dl.breadway.dev/arch`: - -```sh -PKG=$(find packaging/arch -name '*.pkg.tar.zst' | head -1) -printf '%s\n' "$GPG_PRIVATE_KEY" | gpg --batch --import -gpg --batch --yes --local-user releases@breadway.dev --detach-sign "$PKG" -# upload "$PKG" and "${PKG}.sig" to dl.breadway.dev/arch/x86_64/ -# then repo-add -s as above -``` - -Keep publishing to Forgejo until installs have been switched. The ISO -section stays `SigLevel = Never` until the signed tree is live. +still `makepkg`s and PUTs the archive at Forgejo's registry. That path +stays; Never installs keep working. The signed tree is rebuilt by the bos +workflow above (registry fetch + sign + `repo-add -s`), not by writing +`/srv` from breadlock's container. ## After the signed repo exists +Only after `https://dl.breadway.dev/arch/x86_64/breadway.db.sig` HEADs 200 +and the verify commands above succeed: + 1. Import `KEYS.asc` into the ISO keyring (`pacman-key --add` + `--lsign-key`). 2. Point `[breadway]` `Server` at `https://dl.breadway.dev/arch/$arch`. 3. Only then flip that section to `SigLevel = Required`. Do not do those three steps against Forgejo's registry. See -`iso/pacman.conf` and `iso/airootfs/etc/pacman.conf`. +`iso/pacman.conf` and `iso/airootfs/etc/pacman.conf`. This branch does +**not** change either file. diff --git a/scripts/ci-publish-signed-repo.sh b/scripts/ci-publish-signed-repo.sh new file mode 100755 index 0000000..08fbb53 --- /dev/null +++ b/scripts/ci-publish-signed-repo.sh @@ -0,0 +1,298 @@ +#!/usr/bin/env bash +# Collect the current [breadway] ISO packages, detach-sign them with the +# BOS release key (releases@breadway.dev), and publish a signed pacman db +# under /srv/breadway-dl/arch/x86_64/ (https://dl.breadway.dev/arch/x86_64/). +# +# Does not change ISO SigLevel and does not write to the Forgejo Arch +# registry — existing package.yml / packaging/*.yml PUTs stay as they are. +# +# Required env: +# GPG_PRIVATE_KEY armoured secret key (same secret as release-iso.yml) +# Optional env: +# BREADWAY_DEST publish dir (default /srv/breadway-dl/arch/x86_64) +# BREADWAY_PKG_DIR extra directory of .pkg.tar.zst to prefer over the registry +# BREADWAY_REGISTRY Forgejo Arch registry base +# BREADWAY_SIGN_ONLY=1 skip collect; sign+index BREADWAY_REPO_DIR only +set -euo pipefail + +PACKAGES=( + breadlock + calamares + zen-browser-bin + bibata-cursor-theme-bin + zsh-theme-powerlevel10k + yay-bin +) + +ARCH="${BREADWAY_ARCH:-x86_64}" +REGISTRY="${BREADWAY_REGISTRY:-https://git.breadway.dev/api/packages/Breadway/arch/os}" +DEST="${BREADWAY_DEST:-/srv/breadway-dl/arch/${ARCH}}" +KEY_ID="${BREADWAY_KEY_ID:-releases@breadway.dev}" +DB_NAME="${BREADWAY_REGISTRY_DB:-Breadway.os.git.breadway.dev.db}" +REPO_DIR="${BREADWAY_REPO_DIR:-}" + +SCRIPT_PATH="$(readlink -f "${BASH_SOURCE[0]}")" + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +need_key() { + if [[ -z "${GPG_PRIVATE_KEY:-}" ]]; then + die "GPG_PRIVATE_KEY is missing; refusing to publish an unsigned [breadway] repo." + fi +} + +urlencode() { + python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe="-._~"))' "$1" +} + +pkginfo_name() { + local pkg="$1" info + info="$(tar -xOf "$pkg" .PKGINFO 2>/dev/null || zstd -dc "$pkg" | tar -xO .PKGINFO)" + awk -F ' = ' '$1=="pkgname" {print $2; exit}' <<<"$info" +} + +import_key() { + export GNUPGHOME="${GNUPGHOME:-$(mktemp -d "${TMPDIR:-/tmp}/gnupg-breadway-repo.XXXXXX")}" + mkdir -m 700 -p "$GNUPGHOME" + printf '%s\n' "$GPG_PRIVATE_KEY" | gpg --batch --import +} + +detach_sign_pkgs() { + local pkg + shopt -s nullglob + for pkg in *.pkg.tar.zst; do + gpg --batch --yes --local-user "$KEY_ID" --detach-sign "$pkg" + done + shopt -u nullglob +} + +repo_add_signed() { + local pkgs=() + shopt -s nullglob + pkgs=(*.pkg.tar.zst) + shopt -u nullglob + (( ${#pkgs[@]} > 0 )) || die "no .pkg.tar.zst files to index" + rm -f breadway.db breadway.db.tar.gz breadway.db.sig breadway.db.tar.gz.sig \ + breadway.files breadway.files.tar.gz breadway.files.sig breadway.files.tar.gz.sig + if repo-add --help 2>&1 | grep -q -- '--include-sigs'; then + repo-add -s -k "$KEY_ID" --include-sigs breadway.db.tar.gz "${pkgs[@]}" + else + repo-add -s -k "$KEY_ID" breadway.db.tar.gz "${pkgs[@]}" + fi + [[ -e breadway.db.tar.gz.sig || -e breadway.db.sig ]] \ + || die "repo-add -s did not write breadway.db*.sig" +} + +ensure_arch_tools() { + if ! command -v gpg >/dev/null 2>&1; then + command -v pacman >/dev/null 2>&1 || die "gpg not on PATH" + pacman -Sy --noconfirm --needed gnupg + fi + command -v repo-add >/dev/null 2>&1 || die "repo-add not on PATH" + command -v gpg >/dev/null 2>&1 || die "gpg not on PATH" +} + +sign_and_index() { + local dir="$1" + [[ -d "$dir" ]] || die "repo dir missing: $dir" + need_key + ensure_arch_tools + import_key + ( + cd "$dir" + detach_sign_pkgs + repo_add_signed + ) +} + +container_runtime() { + if command -v docker >/dev/null 2>&1; then + printf '%s\n' docker + elif command -v podman >/dev/null 2>&1; then + printf '%s\n' podman + else + return 1 + fi +} + +sign_and_index_anywhere() { + local dir="$1" + if command -v repo-add >/dev/null 2>&1 && command -v gpg >/dev/null 2>&1; then + sign_and_index "$dir" + return + fi + local rt + rt="$(container_runtime)" || die \ + "need host gpg+repo-add, or docker/podman to run archlinux:latest (no Forgejo container: — host must see /srv/breadway-dl)" + # Host job + bind-mount, same reason bakery writes /srv without container:. + "$rt" run --rm --network=host \ + -e GPG_PRIVATE_KEY \ + -e BREADWAY_SIGN_ONLY=1 \ + -e BREADWAY_REPO_DIR=/repo \ + -e BREADWAY_KEY_ID="$KEY_ID" \ + -v "$dir:/repo" \ + -v "$SCRIPT_PATH:/ci-publish-signed-repo.sh:ro" \ + archlinux:latest \ + bash /ci-publish-signed-repo.sh +} + +parse_registry_db() { + local db="$1" + python3 - "$db" "${PACKAGES[@]}" <<'PY' +import sys, tarfile + +db = sys.argv[1] +want = set(sys.argv[2:]) +found = {} +with tarfile.open(db, "r:*") as tf: + for member in tf.getmembers(): + if not member.name.endswith("/desc") or not member.isfile(): + continue + fh = tf.extractfile(member) + if fh is None: + continue + text = fh.read().decode() + fields = {} + key = None + buf = [] + def flush(): + if key is not None: + fields[key] = "\n".join(buf).strip() + for line in text.splitlines(): + if line.startswith("%") and line.endswith("%") and len(line) > 2: + flush() + key = line.strip("%") + buf = [] + else: + buf.append(line) + flush() + name = fields.get("NAME", "") + filename = fields.get("FILENAME", "") + if name in want and filename: + found[name] = filename + +missing = sorted(want - set(found)) +if missing: + sys.stderr.write("registry db missing packages: " + " ".join(missing) + "\n") + raise SystemExit(1) +for name in sys.argv[2:]: + print(f"{name}\t{found[name]}") +PY +} + +copy_local_overrides() { + local dir="$1" + [[ -n "$dir" && -d "$dir" ]] || return 0 + local pkg name + shopt -s nullglob + for pkg in "$dir"/*.pkg.tar.zst "$dir"/*/*.pkg.tar.zst; do + [[ -f "$pkg" ]] || continue + name="$(pkginfo_name "$pkg")" + [[ -n "$name" ]] || continue + local wanted=0 p + for p in "${PACKAGES[@]}"; do + if [[ "$p" == "$name" ]]; then + wanted=1 + break + fi + done + if (( wanted )); then + printf 'local override: %s -> %s\n' "$name" "$(basename "$pkg")" + cp -a "$pkg" "$STAGE/$(basename "$pkg")" + fi + done + shopt -u nullglob +} + +has_pkg_named() { + local name="$1" pkg got + shopt -s nullglob + for pkg in "$STAGE"/*.pkg.tar.zst; do + got="$(pkginfo_name "$pkg")" + if [[ "$got" == "$name" ]]; then + shopt -u nullglob + return 0 + fi + done + shopt -u nullglob + return 1 +} + +collect_from_registry() { + local work db name filename enc url + work="$(mktemp -d "${TMPDIR:-/tmp}/breadway-db.XXXXXX")" + db="$work/$DB_NAME" + curl -fL --retry 3 --retry-delay 2 -o "$db" "$REGISTRY/$ARCH/$DB_NAME" \ + || die "failed to fetch $REGISTRY/$ARCH/$DB_NAME" + while IFS=$'\t' read -r name filename; do + if has_pkg_named "$name"; then + printf 'using local %s, skip registry\n' "$name" + continue + fi + enc="$(urlencode "$filename")" + url="$REGISTRY/$ARCH/$enc" + printf 'fetch %s\n' "$filename" + curl -fL --retry 3 --retry-delay 2 -o "$STAGE/$filename" "$url" \ + || die "failed to fetch $url" + done < <(parse_registry_db "$db") + rm -rf "$work" +} + +publish_tree() { + local parent dest_name prev + parent="$(dirname "$DEST")" + dest_name="$(basename "$DEST")" + mkdir -p "$parent" + chmod a+rX "$STAGE" + find "$STAGE" -type f -exec chmod a+r {} + + prev="$parent/${dest_name}.prev" + rm -rf "$prev" + if [[ -e "$DEST" ]]; then + mv "$DEST" "$prev" + fi + mv "$STAGE" "$DEST" + rm -rf "$prev" + STAGE="" +} + +if [[ "${BREADWAY_SIGN_ONLY:-0}" == 1 ]]; then + [[ -n "$REPO_DIR" ]] || die "BREADWAY_SIGN_ONLY requires BREADWAY_REPO_DIR" + sign_and_index "$REPO_DIR" + exit 0 +fi + +need_key + +DEST_PARENT="$(dirname "$DEST")" +mkdir -p "$DEST_PARENT" || die "cannot create $DEST_PARENT (runner must write /srv/breadway-dl)" +STAGE="$(mktemp -d "$DEST_PARENT/.stage-XXXXXX")" +cleanup() { + if [[ -n "${STAGE:-}" && -d "${STAGE:-}" ]]; then + rm -rf "$STAGE" + fi + if [[ -n "${GNUPGHOME:-}" && "$GNUPGHOME" == *gnupg-breadway-repo* ]]; then + rm -rf "$GNUPGHOME" + fi +} +trap cleanup EXIT + +copy_local_overrides "${BREADWAY_PKG_DIR:-}" +collect_from_registry + +missing=() +for name in "${PACKAGES[@]}"; do + has_pkg_named "$name" || missing+=("$name") +done +if (( ${#missing[@]} > 0 )); then + die "missing packages after collect: ${missing[*]}" +fi + +sign_and_index_anywhere "$STAGE" + +# Do not publish helper junk if a container left any. +rm -f "$STAGE/.sign.sh" + +publish_tree + +printf 'published signed [breadway] repo -> %s\n' "$DEST" +ls -lh "$DEST"