Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ba8722b0b7 | ||
|
|
fa8fec0b4f | ||
|
|
68976142f5 | ||
|
|
99279d5347 | ||
|
|
ea2d4994f1 | ||
|
|
c2e070d172 | ||
|
|
5e27870608 | ||
|
|
a753d300db | ||
|
|
46ea498db3 | ||
|
|
d59d7e0ad4 | ||
|
|
3616fa409f | ||
|
|
48edc09e8d | ||
|
|
cf8f09612f | ||
|
|
74fc9ef183 | ||
|
|
a476bb5352 | ||
|
|
905b2894b6 | ||
|
|
50c58c4e9e | ||
|
|
1ee5760c1b | ||
|
|
dc1cbeb12a | ||
|
|
4345732dad |
No files matched your search
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/bin/bash
|
||||
## mirror-index-copy.sh SRC DST — CI only (not copied into images).
|
||||
## SRC registry/path@sha256:<index digest> (digest-pinned upstream reference)
|
||||
## DST registry/path:<tag> (the mirror tag consumers FROM)
|
||||
##
|
||||
## Registry-to-registry copy of the WHOLE upstream image index (every
|
||||
## platform), then assertions. No local `docker pull`, so nothing is flattened
|
||||
## to the runner's platform (the old pull/tag/push step re-published only the
|
||||
## amd64 manifest, leaving the mirror without an arm64 variant).
|
||||
##
|
||||
## Ordering (protects a previously multi-arch DST tag):
|
||||
## 1. copy SRC -> DST_REPO:<tag>-staging (real tag untouched)
|
||||
## 2. assert staging: index lists every REQUIRED platform and its digest
|
||||
## equals the digest in SRC
|
||||
## 3. only then point DST at that digest (registry-side, no data transfer)
|
||||
## 4. re-assert on DST
|
||||
## A failure at 1/2 exits non-zero before the real tag is touched. A failure
|
||||
## at 4 cannot be rolled back by this script (the tag already moved) but it is
|
||||
## loud and the same digest was already proven at step 2.
|
||||
##
|
||||
## Gitea compatibility (measured against repo.anhonesthost.net, 2026-11-11):
|
||||
## * Gitea's registry accepts whole indexes (OCI index, Docker manifest list,
|
||||
## attestation children, 14-16 children) and preserves the index digest
|
||||
## byte-for-byte, so the digest-equality assertion below holds there too.
|
||||
## * It rejects an index PUT whose child manifests are not already stored as
|
||||
## by-DIGEST versions (404 MANIFEST_BLOB_UNKNOWN). buildx <= 0.20.x pushes the
|
||||
## children of a copied index to the TAG path (works on registry:2, not on
|
||||
## Gitea): the copy dies with `PUT .../manifests/<tag>: 404` and leaves a
|
||||
## stray single-manifest version named <tag>. buildx >= 0.24 pushes children by
|
||||
## digest and works. The distro buildx on the runner is 0.15.1, so the
|
||||
## workflow installs a pinned newer one and this script refuses to run on an
|
||||
## older one (MIN_BUILDX) instead of leaving a stray version behind.
|
||||
##
|
||||
## Env: REQUIRED_PLATFORMS (default "linux/amd64 linux/arm64").
|
||||
## BUILDX (default "docker buildx"): the buildx command; may be the path of
|
||||
## a pinned docker-buildx binary.
|
||||
## IMAGETOOLS (default "$BUILDX imagetools").
|
||||
## MIN_BUILDX (default 0.24.0; empty disables the version guard).
|
||||
## (buildx talks plain HTTP to a localhost registry, so the test needs no flags.)
|
||||
set -euo pipefail
|
||||
|
||||
SRC="${1:?usage: mirror-index-copy.sh SRC@sha256:DIGEST DST_REPO:TAG}"
|
||||
DST="${2:?usage: mirror-index-copy.sh SRC@sha256:DIGEST DST_REPO:TAG}"
|
||||
REQUIRED_PLATFORMS="${REQUIRED_PLATFORMS:-linux/amd64 linux/arm64}"
|
||||
BUILDX="${BUILDX:-docker buildx}"
|
||||
MIN_BUILDX="${MIN_BUILDX-0.24.0}"
|
||||
read -r -a BX <<<"$BUILDX"
|
||||
read -r -a IMGTOOLS <<<"${IMAGETOOLS:-${BUILDX} imagetools}"
|
||||
|
||||
fail() { echo "::error::mirror-index-copy: $*" >&2; exit 1; }
|
||||
|
||||
# Version guard: old buildx corrupts the copy on Gitea (see header).
|
||||
if [ -n "$MIN_BUILDX" ]; then
|
||||
have_bx=$("${BX[@]}" version 2>/dev/null | awk 'NR==1{print $2}'); have_bx="${have_bx#v}"
|
||||
[[ "$have_bx" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]] || fail "cannot determine buildx version from '${BUILDX} version'"
|
||||
[ "$(printf '%s\n%s\n' "$MIN_BUILDX" "${have_bx%%[-+]*}" | sort -V | head -1)" = "$MIN_BUILDX" ] \
|
||||
|| fail "buildx ${have_bx} is too old (need >= ${MIN_BUILDX}): older buildx pushes index children by tag, which Gitea rejects"
|
||||
fi
|
||||
|
||||
# Validate shapes (registry host may carry :port, e.g. localhost:5055).
|
||||
[[ "$SRC" =~ ^[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || fail "bad source reference '${SRC}'"
|
||||
[[ "$DST" =~ ^[a-z0-9][a-z0-9._:/-]*:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || fail "bad destination '${DST}'"
|
||||
src_digest="${SRC#*@}"
|
||||
dst_repo="${DST%:*}"; dst_tag="${DST##*:}"
|
||||
STAGING="${dst_repo}:${dst_tag}-staging"
|
||||
|
||||
# assert_ref REF: REF must be an index listing every required platform, and
|
||||
# its manifest digest must equal the upstream index digest.
|
||||
assert_ref() {
|
||||
local ref="$1" raw have want got
|
||||
raw=$("${IMGTOOLS[@]}" inspect "$ref" --raw) || fail "cannot inspect ${ref}"
|
||||
have=$(jq -r '[.manifests[]? | select(.platform.os != "unknown") | "\(.platform.os)/\(.platform.architecture)"] | unique | join(" ")' <<<"$raw")
|
||||
[ -n "$have" ] || fail "${ref} is not a multi-platform index (single-platform manifest?); platforms: none"
|
||||
for want in $REQUIRED_PLATFORMS; do
|
||||
case " $have " in *" $want "*) ;; *) fail "${ref} is missing ${want} (has: ${have})" ;; esac
|
||||
done
|
||||
# Digest = sha256 of the exact index bytes the registry serves (the same
|
||||
# thing a client pins by). Not `--format {{.Manifest.Digest}}`: buildx 0.28
|
||||
# ignores --format here and prints the human summary instead.
|
||||
got="sha256:$(printf '%s' "$raw" | sha256sum | cut -d' ' -f1)"
|
||||
[ "$got" = "$src_digest" ] || fail "${ref} digest ${got} != upstream ${src_digest}"
|
||||
echo "ok: ${ref} platforms=[${have}] digest=${got}"
|
||||
}
|
||||
|
||||
echo "::group::copy index ${SRC} -> ${STAGING}"
|
||||
"${IMGTOOLS[@]}" create -t "$STAGING" "$SRC"
|
||||
echo "::endgroup::"
|
||||
assert_ref "$STAGING"
|
||||
|
||||
echo "::group::promote staging -> ${DST}"
|
||||
"${IMGTOOLS[@]}" create -t "$DST" "${dst_repo}@${src_digest}"
|
||||
echo "::endgroup::"
|
||||
assert_ref "$DST"
|
||||
echo "mirror ok: ${DST} == ${src_digest}"
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
## resolve-base-image.sh IMAGE:TAG — CI only (not copied into images).
|
||||
## Prints IMAGE@sha256:<digest> for a Docker Hub tag, looked up DIRECTLY at
|
||||
## registry-1.docker.io over HTTPS (never via the pkg-cache mirror), so the
|
||||
## build can say `FROM IMAGE@sha256:...` and the mirror can only serve
|
||||
## content-addressed blobs that BuildKit verifies. Exits non-zero (prints
|
||||
## nothing on stdout) if the lookup fails; the workflow then skips the mirror
|
||||
## and pulls the tag directly from Docker Hub. HEAD requests do not count
|
||||
## against Docker Hub pull rate limits.
|
||||
## Canonical copy: cloud-apache-container/.gitea/scripts/resolve-base-image.sh
|
||||
## (docs/pkg-cache.md there); other repos carry identical copies.
|
||||
set -euo pipefail
|
||||
ref="${1:?usage: resolve-base-image.sh IMAGE:TAG}"
|
||||
name_re='^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)?$'
|
||||
tag_re='^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$'
|
||||
name="${ref%:*}"; tag="${ref##*:}"
|
||||
[ "$name" != "$ref" ] || { echo "resolve-base-image: '$ref' has no :tag" >&2; exit 2; }
|
||||
[[ "$name" =~ $name_re ]] || { echo "resolve-base-image: bad image name '$name'" >&2; exit 2; }
|
||||
[[ "$tag" =~ $tag_re ]] || { echo "resolve-base-image: bad tag '$tag'" >&2; exit 2; }
|
||||
repo="$name"; [[ "$repo" == */* ]] || repo="library/$repo" # official images
|
||||
|
||||
curl_opts=(-fsS --proto '=https' --retry 3 --retry-all-errors --max-time 20)
|
||||
token=$(curl "${curl_opts[@]}" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" |
|
||||
python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')
|
||||
[ -n "$token" ] || { echo "resolve-base-image: no Docker Hub token for $repo" >&2; exit 1; }
|
||||
hdrs=$(curl "${curl_opts[@]}" -I \
|
||||
-H "Authorization: Bearer ${token}" \
|
||||
-H 'Accept: application/vnd.oci.image.index.v1+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
-H 'Accept: application/vnd.oci.image.manifest.v1+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||||
"https://registry-1.docker.io/v2/${repo}/manifests/${tag}")
|
||||
digest=$(printf '%s\n' "$hdrs" | tr -d '\r' | awk 'tolower($1)=="docker-content-digest:" {print $2}' | tail -n1)
|
||||
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "resolve-base-image: no valid digest for $ref (got '${digest}')" >&2; exit 1; }
|
||||
printf '%s@%s\n' "$name" "$digest"
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
## Test for mirror-index-copy.sh. CI/dev only; needs docker + buildx + jq and a
|
||||
## THROWAWAY registry on REG (default localhost:5055):
|
||||
## docker run -d --name mirror-test-reg -p 127.0.0.1:5055:5000 registry:2
|
||||
## (remove it afterwards). Never point REG at a real registry. Builds only
|
||||
## FROM scratch images, so it needs no network beyond the registry.
|
||||
## Set MIRROR_TEST_UPSTREAM=docker.io/library/python@sha256:... to also run the
|
||||
## copy against a real multi-arch upstream digest (anonymous read-only pull).
|
||||
set -uo pipefail
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
REG="${REG:-localhost:5055}"
|
||||
case "$REG" in localhost:*|127.0.0.1:*) ;; *) echo "REG must be a localhost throwaway registry"; exit 2;; esac
|
||||
curl -fsS -m5 "http://${REG}/v2/" >/dev/null || { echo "no registry on ${REG}"; exit 2; }
|
||||
run="t$$"; work=$(mktemp -d); trap 'rm -rf "$work"' EXIT
|
||||
pass=0; failn=0
|
||||
ok() { echo "PASS: $1"; pass=$((pass+1)); }
|
||||
bad() { echo "FAIL: $1"; failn=$((failn+1)); }
|
||||
digest() { docker buildx imagetools inspect "$1" --format '{{.Manifest.Digest}}'; }
|
||||
|
||||
mkimg() { # mkimg REPO:TAG PLATFORM -> tiny FROM-scratch image
|
||||
printf 'FROM scratch\nLABEL t=%s\n' "$2-$run" >"$work/Dockerfile"
|
||||
docker buildx build -q --platform "$2" --provenance=false --sbom=false \
|
||||
--output "type=image,name=${1},push=true" "$work" >/dev/null || { echo "build $1 failed"; exit 2; }
|
||||
}
|
||||
mkimg "${REG}/${run}/amd64:v" linux/amd64
|
||||
mkimg "${REG}/${run}/arm64:v" linux/arm64
|
||||
A=$(digest "${REG}/${run}/amd64:v"); B=$(digest "${REG}/${run}/arm64:v")
|
||||
# synthetic two-platform upstream index, and a single-platform one
|
||||
docker buildx imagetools create -t "${REG}/${run}/multi:v" "${REG}/${run}/amd64@${A}" "${REG}/${run}/arm64@${B}" >/dev/null 2>&1 || exit 2
|
||||
MULTI=$(digest "${REG}/${run}/multi:v"); SINGLE="$A"
|
||||
|
||||
# 1. two-platform source: pass, digest preserved, tag lands on source digest
|
||||
if "$here/mirror-index-copy.sh" "${REG}/${run}/multi@${MULTI}" "${REG}/${run}/dst:1" >"$work/o1" 2>&1 \
|
||||
&& [ "$(digest "${REG}/${run}/dst:1")" = "$MULTI" ]; then ok "multi-arch copy passes and digest round-trips ($MULTI)"; else bad "multi-arch copy"; cat "$work/o1"; fi
|
||||
|
||||
# 2. single-platform source: must FAIL loudly and must not create the real tag
|
||||
if "$here/mirror-index-copy.sh" "${REG}/${run}/amd64@${SINGLE}" "${REG}/${run}/dst2:1" >"$work/o2" 2>&1; then bad "single-platform source was accepted"; else
|
||||
grep -q 'is not a multi-platform index\|missing linux/arm64' "$work/o2" && ok "single-platform source rejected: $(grep -m1 '::error::' "$work/o2")" || { bad "failed for the wrong reason"; cat "$work/o2"; }
|
||||
if docker buildx imagetools inspect "${REG}/${run}/dst2:1" >/dev/null 2>&1; then bad "real tag created despite failure"; else ok "real tag not created on failure"; fi
|
||||
fi
|
||||
|
||||
# 3. existing multi-arch tag must survive a failing (single-platform) refresh
|
||||
before=$(digest "${REG}/${run}/dst:1")
|
||||
if "$here/mirror-index-copy.sh" "${REG}/${run}/amd64@${SINGLE}" "${REG}/${run}/dst:1" >"$work/o3" 2>&1; then bad "downgrade accepted"; else
|
||||
[ "$(digest "${REG}/${run}/dst:1")" = "$before" ] && ok "previously multi-arch :tag untouched after failed refresh" || bad ":tag moved to single-platform"; fi
|
||||
|
||||
# 4. malformed references are refused before any registry traffic
|
||||
"$here/mirror-index-copy.sh" "${REG}/x:tag" "${REG}/${run}/d:1" >/dev/null 2>&1 && bad "tag-only source accepted" || ok "non-digest source refused"
|
||||
"$here/mirror-index-copy.sh" "${REG}/x@sha256:$(printf 'a%.0s' {1..64})" 'bad dst' >/dev/null 2>&1 && bad "bad dst accepted" || ok "bad destination refused"
|
||||
|
||||
# 6. index WITH buildx attestation children (unknown/unknown), like Docker Hub
|
||||
# official images: copy passes, digest preserved, attestations not counted as platforms
|
||||
printf 'FROM scratch\nLABEL t=att-%s\n' "$run" >"$work/Dockerfile"
|
||||
if docker buildx build -q --platform linux/amd64,linux/arm64 --provenance=mode=min --sbom=false \
|
||||
--output "type=image,name=${REG}/${run}/att:v,push=true" "$work" >/dev/null 2>&1; then
|
||||
ATT=$(digest "${REG}/${run}/att:v")
|
||||
if "$here/mirror-index-copy.sh" "${REG}/${run}/att@${ATT}" "${REG}/${run}/attdst:1" >"$work/o6" 2>&1 \
|
||||
&& [ "$(digest "${REG}/${run}/attdst:1")" = "$ATT" ]; then ok "index with attestation children copied, digest preserved ($ATT)"; else bad "attestation-index copy"; cat "$work/o6"; fi
|
||||
else bad "could not build attestation index"; fi
|
||||
|
||||
# 7. buildx version guard: a too-old buildx is refused BEFORE any registry traffic
|
||||
# (old buildx leaves a stray single-manifest tag on Gitea)
|
||||
printf '#!/bin/sh\necho "github.com/docker/buildx v0.15.1 deadbeef"\n' >"$work/old-buildx"; chmod +x "$work/old-buildx"
|
||||
if BUILDX="$work/old-buildx" "$here/mirror-index-copy.sh" "${REG}/${run}/multi@${MULTI}" "${REG}/${run}/olddst:1" >"$work/o7" 2>&1; then bad "old buildx accepted"; else
|
||||
grep -q 'too old' "$work/o7" && ok "old buildx refused: $(grep -m1 '::error::' "$work/o7")" || { bad "old-buildx failed for the wrong reason"; cat "$work/o7"; }
|
||||
docker buildx imagetools inspect "${REG}/${run}/olddst:1-staging" >/dev/null 2>&1 && bad "old buildx touched the registry" || ok "no staging tag created by refused run"
|
||||
fi
|
||||
MIN_BUILDX= BUILDX="$work/old-buildx" IMAGETOOLS="docker buildx imagetools" "$here/mirror-index-copy.sh" "${REG}/${run}/multi@${MULTI}" "${REG}/${run}/olddst2:1" >/dev/null 2>&1 \
|
||||
&& ok "MIN_BUILDX= disables the guard" || bad "guard not disabled by empty MIN_BUILDX"
|
||||
|
||||
# 5. optional: real upstream digest
|
||||
if [ -n "${MIRROR_TEST_UPSTREAM:-}" ]; then
|
||||
u="${MIRROR_TEST_UPSTREAM}"
|
||||
if "$here/mirror-index-copy.sh" "$u" "${REG}/${run}/up:1" >"$work/o5" 2>&1 && [ "$(digest "${REG}/${run}/up:1")" = "${u#*@}" ]; then ok "upstream index copied with identical digest ${u#*@}"; grep '^ok:' "$work/o5"; else bad "upstream copy"; tail -5 "$work/o5"; fi
|
||||
fi
|
||||
echo "passed=$pass failed=$failn"; [ "$failn" -eq 0 ]
|
||||
@@ -11,18 +11,45 @@ on:
|
||||
- main
|
||||
paths:
|
||||
- 'coraza-spoa/**'
|
||||
- 'scripts/assemble-multiarch-index.sh'
|
||||
- '.gitea/workflows/build-push-coraza.yaml'
|
||||
workflow_dispatch:
|
||||
|
||||
# Multi-arch (linux/amd64 + linux/arm64) in ONE job on the trusted x86 runner,
|
||||
# no QEMU: coraza-spoa/Dockerfile cross-compiles the Go binary
|
||||
# (--platform=$BUILDPLATFORM + GOOS/GOARCH). Each platform is pushed BY DIGEST
|
||||
# (untagged) to BOTH registries, then scripts/assemble-multiarch-index.sh
|
||||
# creates the :latest index from those digests. The arm64 step is
|
||||
# continue-on-error: if it fails, :latest is published amd64-only exactly as it
|
||||
# was before multi-arch, and a previous run's arm64 is never re-attached (the
|
||||
# arm64 digest is read only from THIS run's step output). Registry write
|
||||
# credentials exist only on this x86 job. Template for the other WHP image
|
||||
# repos: docs/arm64/IMAGE-PIPELINE-PLAN.md sections 4.1 / 4.4.
|
||||
jobs:
|
||||
Build-and-Push:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
# Internal package cache (CT130 pkg-cache; canonical docs: docs/pkg-cache.md
|
||||
# in cloud-apache-container). Used ONLY if it answers over TLS with our
|
||||
# pinned CA; otherwise (down, unreachable, bad/expired cert) this job builds
|
||||
# from the upstream repositories / default GOPROXY exactly as before. The
|
||||
# build stages' base (golang:1.25) comes from the in-house registry
|
||||
# (repo.anhonesthost.net, weekly digest-pinned mirror of Docker Hub -
|
||||
# mirror-base-image.yaml) and the runtime base from gcr.io, not docker.io,
|
||||
# so the pkg-cache registry mirror is never used here: repos + Go modules.
|
||||
# The CA lives under coraza-spoa/ because that is this build's context.
|
||||
- name: Probe package cache
|
||||
id: pkgcache
|
||||
run: |
|
||||
url=""
|
||||
if curl -fsS --max-time 5 --cacert coraza-spoa/keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi
|
||||
echo "pkg-cache: repos=${url:-<unreachable: upstream>} registry-mirror=<not used: no docker.io base (base from repo.anhonesthost.net)>"
|
||||
[ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories"
|
||||
echo "url=${url}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: https://github.com/docker/setup-buildx-action@v3
|
||||
@@ -34,7 +61,7 @@ jobs:
|
||||
username: ${{ secrets.CI_USER }}
|
||||
password: ${{ secrets.CI_TOKEN }}
|
||||
|
||||
# Mirror to GitHub Container Registry — see build-push.yaml for the
|
||||
# Mirror to GitHub Container Registry - see build-push.yaml for the
|
||||
# secret/username convention.
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
@@ -43,12 +70,42 @@ jobs:
|
||||
username: shadowdao
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build Image
|
||||
# amd64: required. Failure here fails the job and publishes nothing, as
|
||||
# before. Pushed untagged by digest to both registries (same digest in
|
||||
# each); the tag is only created by the assemble step below.
|
||||
- name: Build amd64 (push by digest)
|
||||
id: amd64
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./coraza-spoa
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: |
|
||||
repo.anhonesthost.net/cloud-hosting-platform/coraza-spoa:latest
|
||||
ghcr.io/shadowdao/coraza-spoa:latest
|
||||
build-args: |
|
||||
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
|
||||
outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/coraza-spoa,ghcr.io/shadowdao/coraza-spoa",push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
# arm64: best effort (cross-compiled, no emulation). Must never block or
|
||||
# alter the amd64 tag.
|
||||
- name: Build arm64 (push by digest)
|
||||
id: arm64
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./coraza-spoa
|
||||
platforms: linux/arm64
|
||||
build-args: |
|
||||
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
|
||||
outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/coraza-spoa,ghcr.io/shadowdao/coraza-spoa",push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Assemble :latest index
|
||||
env:
|
||||
AMD64_DIGEST: ${{ steps.amd64.outputs.digest }}
|
||||
ARM64_DIGEST: ${{ steps.arm64.outputs.digest }}
|
||||
ARM64_OUTCOME: ${{ steps.arm64.outcome }}
|
||||
run: |
|
||||
scripts/assemble-multiarch-index.sh \
|
||||
--amd64 "$AMD64_DIGEST" \
|
||||
--arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \
|
||||
--repo repo.anhonesthost.net/cloud-hosting-platform/coraza-spoa \
|
||||
--repo ghcr.io/shadowdao/coraza-spoa \
|
||||
--tag latest
|
||||
@@ -1,19 +1,59 @@
|
||||
name: HAProxy Manager Build and Push
|
||||
run-name: ${{ gitea.actor }} pushed a change to main
|
||||
run-name: ${{ gitea.actor }} pushed ${{ gitea.ref_name }}
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
# Release candidates: a push to release/* builds the SAME image but publishes
|
||||
# ONLY :<VERSION>-rc and :<sha>, never :latest and never the plain :<VERSION>
|
||||
# (see the "Resolve tags" step). Main keeps its behaviour.
|
||||
- 'release/**'
|
||||
tags:
|
||||
- 'rc-*'
|
||||
# Manual publish (e.g. after a [skip ci] merge, or to re-publish after a
|
||||
# failed arm64 leg).
|
||||
workflow_dispatch:
|
||||
|
||||
# Multi-arch (linux/amd64 + linux/arm64) in ONE job on the trusted x86 runner.
|
||||
# Each platform is pushed BY DIGEST (untagged) to BOTH registries, then
|
||||
# scripts/assemble-multiarch-index.sh creates :latest, :<version> and :<sha>
|
||||
# from those digests (same template as build-push-coraza.yaml; see
|
||||
# docs/arm64/IMAGE-PIPELINE-PLAN.md sections 4.1 / 4.4 in the WHP repo).
|
||||
# * amd64 is required and built exactly as before; the Dockerfile build gate
|
||||
# (unit suites + `haproxy -c`) runs in it. Failure = nothing published.
|
||||
# * arm64 is continue-on-error and runs the SAME Dockerfile gate under QEMU
|
||||
# (the gate is RUN steps, so it executes on the arm64 binary). If arm64
|
||||
# fails or times out the tags are published amd64-only, exactly as before
|
||||
# multi-arch, and a previous run's arm64 is never re-attached (the digest is
|
||||
# read only from THIS run's step output).
|
||||
jobs:
|
||||
Build-and-Push:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Internal package cache (CT130 pkg-cache; canonical docs: docs/pkg-cache.md
|
||||
# in cloud-apache-container). Used ONLY if it answers over TLS with our
|
||||
# pinned CA; otherwise (down, unreachable, bad/expired cert) this job builds
|
||||
# from the upstream repositories exactly as before. The base image comes
|
||||
# from the in-house registry (repo.anhonesthost.net, weekly digest-pinned
|
||||
# mirror of Docker Hub - mirror-base-image.yaml), not docker.io, so the
|
||||
# pkg-cache registry mirror is never used here: repos only.
|
||||
- name: Probe package cache
|
||||
id: pkgcache
|
||||
run: |
|
||||
url=""
|
||||
if curl -fsS --max-time 5 --cacert keys/pkg-cache/ca.crt -o /dev/null https://192.168.1.40/; then url="https://192.168.1.40"; fi
|
||||
echo "pkg-cache: repos=${url:-<unreachable: upstream>} registry-mirror=<not used: no docker.io base (base from repo.anhonesthost.net)>"
|
||||
[ -n "$url" ] || echo "::warning::pkg-cache unreachable - building from upstream repositories"
|
||||
echo "url=${url}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
with:
|
||||
platforms: arm64
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: https://github.com/docker/setup-buildx-action@v3
|
||||
@@ -43,19 +83,142 @@ jobs:
|
||||
id: ver
|
||||
run: echo "version=$(cat VERSION)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build Image
|
||||
# Publishing strategy. main: :latest + :<version> + :<sha> (unchanged).
|
||||
# release/* branch or rc-* tag: :<version>-rc + :<sha> ONLY. A release candidate
|
||||
# is deployed per host by explicit tag (durable per-host pin in WHP); :latest is
|
||||
# moved to it by a later main publish only after the staged rollout succeeded.
|
||||
# Any other ref (e.g. a manual workflow_dispatch on a feature branch) is refused
|
||||
# rather than guessed at.
|
||||
- name: Resolve tags
|
||||
id: tags
|
||||
env:
|
||||
REF: ${{ gitea.ref }}
|
||||
VERSION: ${{ steps.ver.outputs.version }}
|
||||
SHA: ${{ gitea.sha }}
|
||||
run: |
|
||||
case "$REF" in
|
||||
refs/heads/main)
|
||||
tags="latest ${VERSION} ${SHA}" ;;
|
||||
refs/heads/release/*|refs/tags/rc-*)
|
||||
tags="${VERSION}-rc ${SHA}" ;;
|
||||
*)
|
||||
echo "::error::refusing to publish from ref ${REF} (only main, release/* and rc-* tags)"; exit 1 ;;
|
||||
esac
|
||||
echo "tags=${tags}" >> "$GITHUB_OUTPUT"
|
||||
echo "publishing tags: ${tags}"
|
||||
|
||||
# amd64: required. Pushed untagged by digest to both registries (same
|
||||
# digest in each); tags are only created by the assemble step below.
|
||||
- name: Build amd64 (push by digest)
|
||||
id: amd64
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
VERSION=${{ steps.ver.outputs.version }}
|
||||
# Three tags per registry: :latest (moving), :<version> (human-readable
|
||||
# release), :<sha> (immutable, guaranteed-unique rollback target).
|
||||
tags: |
|
||||
repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base:latest
|
||||
repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base:${{ steps.ver.outputs.version }}
|
||||
repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base:${{ gitea.sha }}
|
||||
ghcr.io/shadowdao/haproxy-manager-base:latest
|
||||
ghcr.io/shadowdao/haproxy-manager-base:${{ steps.ver.outputs.version }}
|
||||
ghcr.io/shadowdao/haproxy-manager-base:${{ gitea.sha }}
|
||||
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
|
||||
outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base,ghcr.io/shadowdao/haproxy-manager-base",push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
# arm64: best effort (QEMU, apt-based image). Must never block or alter
|
||||
# the amd64 tags.
|
||||
- name: Build arm64 (push by digest)
|
||||
id: arm64
|
||||
continue-on-error: true
|
||||
timeout-minutes: 30
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
platforms: linux/arm64
|
||||
build-args: |
|
||||
VERSION=${{ steps.ver.outputs.version }}
|
||||
PKG_CACHE=${{ steps.pkgcache.outputs.url }}
|
||||
outputs: type=image,"name=repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base,ghcr.io/shadowdao/haproxy-manager-base",push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
# imagetools needs a NEW buildx. Same pinned, sha256-verified binary as
|
||||
# mirror-base-image.yaml (kept identical on purpose). The assemble step
|
||||
# previously ran on whatever buildx the (possibly ephemeral) runner had
|
||||
# (0.13.1 on the runner that hung in run 5375, 0.15.1 on the trusted one).
|
||||
- name: Install pinned buildx
|
||||
env:
|
||||
BUILDX_VERSION: v0.28.0
|
||||
BUILDX_SHA256: 696bc104bac3bb708eff1af3f8bbc09fda0fd88f5757c1f9b404a35117889224
|
||||
run: |
|
||||
set -euo pipefail
|
||||
dir="${RUNNER_TEMP:-/tmp}/pinned-buildx"; mkdir -p "$dir"
|
||||
curl -fsSL --retry 3 --max-time 120 -o "$dir/docker-buildx" \
|
||||
"https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
|
||||
echo "${BUILDX_SHA256} $dir/docker-buildx" | sha256sum -c -
|
||||
chmod +x "$dir/docker-buildx"
|
||||
"$dir/docker-buildx" version
|
||||
echo "BUILDX=$dir/docker-buildx" >> "$GITHUB_ENV"
|
||||
|
||||
# main: three tags per registry: :latest (moving), :<version> (human-readable
|
||||
# release), :<sha> (immutable, guaranteed-unique rollback target).
|
||||
# release/* or rc-*: :<version>-rc and :<sha> only (see "Resolve tags").
|
||||
# Hardened after run 5375 (assemble hung after pushing, TTY progress, old
|
||||
# buildx): plain progress, pinned buildx, hard 10 minute step timeout.
|
||||
- name: Assemble tag indexes
|
||||
id: assemble
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
BUILDKIT_PROGRESS: plain
|
||||
AMD64_DIGEST: ${{ steps.amd64.outputs.digest }}
|
||||
ARM64_DIGEST: ${{ steps.arm64.outputs.digest }}
|
||||
ARM64_OUTCOME: ${{ steps.arm64.outcome }}
|
||||
VERSION: ${{ steps.ver.outputs.version }}
|
||||
TAGS: ${{ steps.tags.outputs.tags }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag_args=()
|
||||
for t in $TAGS; do tag_args+=(--tag "$t"); done
|
||||
# Use the pinned buildx for create AND the script's platform inspects.
|
||||
export IMAGETOOLS="${BUILDX} imagetools"
|
||||
out=$(scripts/assemble-multiarch-index.sh \
|
||||
--amd64 "$AMD64_DIGEST" \
|
||||
--arm64 "$ARM64_DIGEST" --arm64-outcome "$ARM64_OUTCOME" \
|
||||
--repo repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base \
|
||||
--repo ghcr.io/shadowdao/haproxy-manager-base \
|
||||
"${tag_args[@]}" | tee /dev/stderr)
|
||||
platforms=$(printf '%s\n' "$out" | sed -n 's/^platforms=//p' | tail -n1)
|
||||
[ -n "$platforms" ] || { echo "::error::assemble script reported no platforms line"; exit 1; }
|
||||
if [ "$platforms" = "linux/amd64" ]; then
|
||||
echo "::warning::SINGLE-ARCH publish (amd64 only) on purpose: arm64 outcome was '${ARM64_OUTCOME}'. Tags carry no arm64 manifest."
|
||||
else
|
||||
echo "multi-arch publish: ${platforms}"
|
||||
fi
|
||||
echo "platforms=${platforms}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Final gate: every tag in every registry must resolve, carry the platforms
|
||||
# the assemble step reported, and have ONE digest across all tags and both
|
||||
# registries (they are built from the same sources). Any miss fails the job
|
||||
# instead of trusting that the create calls "looked" fine.
|
||||
- name: Verify published tags in every registry
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
BUILDKIT_PROGRESS: plain
|
||||
TAGS: ${{ steps.tags.outputs.tags }}
|
||||
PLATFORMS: ${{ steps.assemble.outputs.platforms }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
repos="repo.anhonesthost.net/cloud-hosting-platform/haproxy-manager-base ghcr.io/shadowdao/haproxy-manager-base"
|
||||
want=$(printf '%s' "$PLATFORMS" | tr ',' '\n' | sort | tr '\n' ' ')
|
||||
ref_digest=""; ref_name=""; fail=0
|
||||
for r in $repos; do
|
||||
for t in $TAGS; do
|
||||
ref="${r}:${t}"
|
||||
if ! d=$(timeout 90 $BUILDX imagetools inspect "$ref" --format '{{.Manifest.Digest}}' 2>&1); then
|
||||
echo "::error::MISSING or unreadable: ${ref}: ${d}"; fail=1; continue
|
||||
fi
|
||||
[[ "$d" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "::error::bad digest for ${ref}: ${d}"; fail=1; continue; }
|
||||
got=$(timeout 90 $BUILDX imagetools inspect "$ref" --format '{{range .Manifest.Manifests}}{{if ne .Platform.OS "unknown"}}{{.Platform.OS}}/{{.Platform.Architecture}}{{"\n"}}{{end}}{{end}}' | sort | tr '\n' ' ')
|
||||
if [ "$got" != "$want" ]; then
|
||||
echo "::error::platform mismatch for ${ref}: have [${got}] want [${want}]"; fail=1
|
||||
fi
|
||||
echo "ok ${ref} ${d} [${got}]"
|
||||
if [ -z "$ref_digest" ]; then ref_digest="$d"; ref_name="$ref"
|
||||
elif [ "$d" != "$ref_digest" ]; then
|
||||
echo "::error::digest mismatch: ${ref}=${d} vs ${ref_name}=${ref_digest}"; fail=1
|
||||
fi
|
||||
done
|
||||
done
|
||||
[ "$fail" = 0 ] || { echo "::error::tag verification FAILED"; exit 1; }
|
||||
echo "verified: ${TAGS} in both registries, digest ${ref_digest}, platforms ${PLATFORMS}"
|
||||
@@ -1,8 +1,8 @@
|
||||
name: Mirror base images
|
||||
run-name: weekly base-image mirror
|
||||
|
||||
# Pulls each declared base image from upstream and re-pushes to the in-house
|
||||
# registry, so any of our images that FROM these don't depend on docker.io's
|
||||
# Copies each declared base image (the whole multi-arch INDEX, not one
|
||||
# platform) from upstream to the in-house registry, so any of our images that FROM these don't depend on docker.io's
|
||||
# Cloudflare R2 blob storage being reachable. The 2026-05-12 Cloudflare
|
||||
# incident motivated this for python:3.12-slim and again for golang:1.25
|
||||
# when the coraza-spoa build hit the same blob-fetch failure.
|
||||
@@ -26,11 +26,58 @@ jobs:
|
||||
# others from refreshing.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# hub: the Docker Hub IMAGE:TAG whose digest is resolved DIRECTLY at
|
||||
# registry-1.docker.io (resolve-base-image.sh expects no docker.io/
|
||||
# prefix; official images resolve as library/<image>).
|
||||
image:
|
||||
- { src: 'docker.io/library/python:3.12-slim', dst_path: 'cloud-hosting-platform/python', tag: '3.12-slim' }
|
||||
- { src: 'docker.io/library/golang:1.25', dst_path: 'cloud-hosting-platform/golang', tag: '1.25' }
|
||||
- { hub: 'python:3.12-slim', dst_path: 'cloud-hosting-platform/python', tag: '3.12-slim' }
|
||||
- { hub: 'golang:1.25', dst_path: 'cloud-hosting-platform/golang', tag: '1.25' }
|
||||
|
||||
steps:
|
||||
# Only for .gitea/scripts/resolve-base-image.sh.
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Supply chain: ask Docker Hub itself (over HTTPS, never via a mirror) which
|
||||
# digest the tag points at right now, then pull exactly that digest. What
|
||||
# lands in the in-house registry is therefore content-addressed to the
|
||||
# digest Docker Hub reported - not whatever a `docker pull TAG` on the
|
||||
# runner happened to resolve through any configured registry mirror.
|
||||
# If the lookup fails the job FAILS and last week's mirror stays in place.
|
||||
- name: Resolve ${{ matrix.image.hub }} digest (Docker Hub, direct)
|
||||
id: base
|
||||
env:
|
||||
HUB_TAG: ${{ matrix.image.hub }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ref=$(.gitea/scripts/resolve-base-image.sh "$HUB_TAG")
|
||||
[[ "$ref" =~ ^[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}$ ]] || { echo "::error::bad digest reference for ${HUB_TAG}"; exit 1; }
|
||||
echo "base-image: ${HUB_TAG} -> ${ref} (digest from registry-1.docker.io)"
|
||||
echo "ref=${ref}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# imagetools needs a NEW buildx. The runner's distro buildx is 0.15.1, which
|
||||
# pushes the children of a copied index to the TAG path: Gitea answers the
|
||||
# index PUT with 404 and leaves a stray single-manifest <tag>-staging
|
||||
# version (run 5156). buildx >= 0.24 pushes children by digest and works
|
||||
# (measured on repo.anhonesthost.net). Pin one, verified by sha256
|
||||
# (linux-amd64 binary of the v0.28.0 release); mirror-index-copy.sh also
|
||||
# refuses to run on a buildx older than 0.24.
|
||||
- name: Install pinned buildx
|
||||
env:
|
||||
BUILDX_VERSION: v0.28.0
|
||||
BUILDX_SHA256: 696bc104bac3bb708eff1af3f8bbc09fda0fd88f5757c1f9b404a35117889224
|
||||
run: |
|
||||
set -euo pipefail
|
||||
dir="${RUNNER_TEMP:-/tmp}/pinned-buildx"; mkdir -p "$dir"
|
||||
curl -fsSL --retry 3 --max-time 120 -o "$dir/docker-buildx" \
|
||||
"https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
|
||||
echo "${BUILDX_SHA256} $dir/docker-buildx" | sha256sum -c -
|
||||
chmod +x "$dir/docker-buildx"
|
||||
"$dir/docker-buildx" version
|
||||
echo "BUILDX=$dir/docker-buildx" >> "$GITHUB_ENV"
|
||||
|
||||
# The registry token is written ONLY by this trusted x86 runner step
|
||||
# (unchanged from before; no new secret exposure).
|
||||
- name: Login to in-house registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -38,28 +85,25 @@ jobs:
|
||||
username: ${{ secrets.CI_USER }}
|
||||
password: ${{ secrets.CI_TOKEN }}
|
||||
|
||||
- name: Pull, retag, push ${{ matrix.image.src }}
|
||||
# Registry-to-registry copy of the WHOLE upstream index (all platforms,
|
||||
# incl. linux/arm64; $BUILDX = the pinned buildx from the step above). The old pull/tag/push re-published only the runner's
|
||||
# amd64 manifest. mirror-index-copy.sh stages to <tag>-staging, asserts
|
||||
# amd64+arm64 present and index digest == the Docker-Hub-reported digest,
|
||||
# and only then moves the real tag, so a bad copy never replaces a good
|
||||
# multi-arch mirror. Logic + tests: .gitea/scripts/mirror-index-copy.sh,
|
||||
# .gitea/scripts/test-mirror-index-copy.sh.
|
||||
- name: Copy index by digest, assert platforms + digest ${{ matrix.image.hub }}
|
||||
# Values reach the script via env, never via ${{ }} inside run:.
|
||||
env:
|
||||
REF: ${{ steps.base.outputs.ref }}
|
||||
DST_PATH: ${{ matrix.image.dst_path }}
|
||||
DST_TAG: ${{ matrix.image.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SRC="${{ matrix.image.src }}"
|
||||
DST="repo.anhonesthost.net/${{ matrix.image.dst_path }}:${{ matrix.image.tag }}"
|
||||
|
||||
echo "::group::Pulling ${SRC}"
|
||||
docker pull "${SRC}"
|
||||
echo "::endgroup::"
|
||||
|
||||
# Capture the upstream digest so the workflow log shows what we
|
||||
# actually pushed. Helps diagnose "did the mirror really update"
|
||||
# questions later.
|
||||
SRC_DIGEST=$(docker image inspect "${SRC}" -f '{{index .RepoDigests 0}}')
|
||||
echo "upstream digest: ${SRC_DIGEST}"
|
||||
|
||||
docker tag "${SRC}" "${DST}"
|
||||
|
||||
echo "::group::Pushing ${DST}"
|
||||
docker push "${DST}"
|
||||
echo "::endgroup::"
|
||||
|
||||
# Sanity: the in-house tag should now resolve to the same content.
|
||||
DST_DIGEST=$(docker image inspect "${DST}" -f '{{index .RepoDigests 0}}')
|
||||
echo "mirror digest: ${DST_DIGEST}"
|
||||
[[ "$REF" =~ ^[a-z0-9][a-z0-9._/-]*@sha256:[0-9a-f]{64}$ ]] || { echo "::error::bad digest reference '${REF}'"; exit 1; }
|
||||
[[ "$DST_PATH" =~ ^[a-z0-9][a-z0-9._/-]*$ && "$DST_TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || { echo "::error::bad destination"; exit 1; }
|
||||
name="${REF%@*}"; digest="${REF#*@}"
|
||||
[[ "$name" == */* ]] || name="library/${name}" # official images
|
||||
.gitea/scripts/mirror-index-copy.sh \
|
||||
"docker.io/${name}@${digest}" \
|
||||
"repo.anhonesthost.net/${DST_PATH}:${DST_TAG}"
|
||||
@@ -13,6 +13,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
- **API thread-pool safety**: `python3 scripts/test-gunicorn-socket-timeouts.py` - runs REAL gunicorn (~60s); asserts a silent/half-written client cannot pin a gthread pool thread (per-connection socket timeout, `HAPROXY_MGR_CLIENT_TIMEOUT`), 2050 never-sending connections are reaped (`HAPROXY_MGR_FIRST_BYTE_TIMEOUT`), slowloris header drips are cut (`HAPROXY_MGR_HEADER_DEADLINE`, counted from enqueue), 40 reconnecting drippers cannot starve legit requests, the :8000 peer filter (`HAPROXY_MGR_API_ALLOWED_PEERS`) refuses non-allowed peers, slow legit requests are not cut off, the wedge watchdog (`HAPROXY_MGR_WEDGE_SECONDS`) replaces a fully stuck worker, and an in-process call to a stalled upstream times out. Run it after touching `haproxy_gunicorn_worker.py`, start-up.sh's gunicorn lines, or upgrading gunicorn (the worker overrides gthread internals: `enqueue_req`, `accept`, `on_client_socket_readable`, `murder_keepalived`, `handle`, `handle_request`).
|
||||
- **WAF body inspection**: `python3 scripts/test-waf-body-inspection.py` - offline (runs in the image build); pins `option http-buffer-request`, the `X-WHP-WAF-Body` complete/partial stamp (strip -> stamp -> SPOE -> strip), `tune.bufsize` against the 100 ms SPOE fail-open budget and the 65535-byte SPOP frame cap, and Coraza's body limits in `coraza-spoa/config.yaml`. Run it after touching the Coraza block, `tune.bufsize`, or the SPOE engine.
|
||||
- **Superseded-cert retirement**: covered by the same file (`TestRetirementRules`, `TestSanOverlapRetirement`). A new bundle retires any OTHER pem/lineage sharing ANY name (CN or SAN) with it - the pem moved to `cert-retired/`, the lineage's live+archive+renewal moved TOGETHER to `/etc/letsencrypt/retired/<stamp>/` (leaving archive/ behind makes certbot refuse the name forever), never deleted; the bundle's own `--cert-name` lineage is `live/<primary>` exactly and is never retired - unless that cert is still the only valid one for a name this edge serves (`retirement_blockers()`). `GET /api/capabilities` advertises this to WHP, which must not use whole-site re-bundling without it. Expired pems already in the crt dir are moved to `cert-retired/` by `cert_retire_expired_pems` (never the last pem, never `default_self_signed_cert.pem`). Expired lineages are never published (`cert_is_expired` in `cert-publish-lib.sh`, `leaf_is_expired()` in Python). Fixtures: `scripts/test-fixtures/san-certs/`.
|
||||
- **Multi-arch index assembly (coraza-spoa)**: `python3 scripts/test-assemble-multiarch-index.py` - offline (NOT run in the image build; it also reads `.gitea/workflows/build-push-coraza.yaml`); `scripts/assemble-multiarch-index.sh` turns the by-digest amd64 + arm64 pushes into the tag(s): arm64 is attached only if THIS run's arm64 step outcome is `success` (failure => amd64-only tag, never a stale arm64), a bad amd64 digest publishes nothing. `scripts/e2e-coraza-multiarch.sh` rehearses the whole thing against throwaway local registries (see coraza-spoa/README.md "Multi-arch").
|
||||
- **Default splash**: `python3 scripts/test-default-splash.py` - offline (runs in the image build); unrouted hosts (server hostname, bare IP, unknown names) are answered by `http-request return ... file /haproxy/errors/default-splash.html` BEFORE the Coraza SPOE call, never for a routed host (`routed_hosts.list`/`routed_wildcards.list`, written by `generate_config()` from the routing rows and in the rollback set), never for ACME/blocked/suspended. Run it after touching the splash block, the routing ACL templates, or the domain loops in `generate_config()`. See README "Default Splash".
|
||||
- **gunicorn access logging**: `python3 scripts/test-gunicorn-access-logging.py` - real gunicorn; :8080 has no access log, :8000 access goes to `/var/log/haproxy/api-access.log` (rotated by `scripts/gunicorn-access-rotate.py`), nothing per-request on stdout (journald rate-limit/Graylog flood), gunicorn errors stay on stderr. See docs/logging.md. Run after touching start-up.sh's gunicorn lines.
|
||||
- **Access log receiver**: `python3 scripts/test-access-log-receiver.py` - offline (runs in the image build); one datagram = one line (PRI stripped, control chars neutralised), size cap + gzip generations, write errors drop without raising, real UNIX-socket path. Run it after touching `haproxy-log-receiver.py` or the `log` line in `hap_header.tpl`.
|
||||
- **Manual Testing**: Run `curl` commands against `http://localhost:8000` endpoints as shown in README.md
|
||||
|
||||
### Reading stick tables (and why it is easy to get silently wrong)
|
||||
@@ -28,8 +32,10 @@ columns; the first token is an allocation pointer (`0x...:`), not the key. Parse
|
||||
by NAME, and treat a missing field as an ERROR — never default it to `0`. The
|
||||
`web` table stores only `conn_cur`, `conn_rate`, `http_req_rate`,
|
||||
`http_err_rate`; it holds **no history and no counter of past blocks**. What was
|
||||
actually denied/tarpitted is in the edge access log on the **host** at
|
||||
`/var/log/haproxy.log` (shipped 2026.08.8), not in any stick table.
|
||||
actually denied/tarpitted is in the edge access log, since 2026.11.2 INSIDE the
|
||||
container at `/var/log/haproxy/access.log` (in-container receiver, volume
|
||||
`haproxy-logs`; README "Access Log"), not in any stick table. (2026.08.8-2026.11.1
|
||||
sent it to host rsyslog on 172.18.0.1:514, which had no listener on the fleet.)
|
||||
|
||||
This is written down because `/api/security/stats` and `show-tarpit-ips.sh`
|
||||
reported "Scan Count"/"BLOCKED" figures parsed from `gpc0`/`gpc1` — fields no
|
||||
@@ -173,4 +179,5 @@ re-reads it on reload, so a failed runtime command must degrade to
|
||||
- **The :8000 API refuses peers other than 127.0.0.1/::1 and the container's default gateway** (`HAPROXY_MGR_API_ALLOWED_PEERS`, default `127.0.0.1,::1,gateway`; ports via `HAPROXY_MGR_PEER_FILTER_PORTS`, default `8000`). haproxy-manager shares `client-net` with every tenant container; host callers arrive from the gateway via docker-proxy. Only effective with the WHP `-p 127.0.0.1:8000:8000` publish (a 0.0.0.0 publish lets a tenant hairpin through the host and arrive from the gateway). A sidecar that must call :8000 directly needs its IP added
|
||||
- gunicorn uses `haproxy_gunicorn_worker.TimeoutThreadWorker` (gthread + client socket timeout + first-byte deadline + total header deadline + wedge watchdog). Stock gthread leaves client sockets blocking with NO timeout; `--timeout` only watches the main thread's heartbeat
|
||||
- Supports deployment on servers with git directory at `/root/whp` and web file sync via rsync to `/docker/whp/web/`
|
||||
- HAProxy is version 3.0.11
|
||||
- HAProxy is the 3.2 LTS series from haproxy.debian.net (`trixie-backports-3.2`, native OpenSSL 3.5 QUIC); exact build in `/haproxy/HAPROXY_VERSION`. Debian trixie's 3.0.11 had the DNS-resolver watchdog abort (haproxy #3101)
|
||||
- HAProxy runs as `haproxy -W -db` (foreground master, started by `haproxy_manager.spawn_haproxy_master()`), NOT daemonised: `daemon` re-points the master's stderr at /dev/null and loses ALERTs and the ha_panic/watchdog thread dump. Its stderr is appended to `/var/log/haproxy/haproxy-stderr.log` (haproxy-logs volume) and `scripts/haproxy-stderr-relay.py` re-emits it on the container stderr (`haproxy-stderr: ` prefix). The in-container supervisor probes with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` (default 2 s) and only runs python when HAProxy is gone; Crash evidence (dump blocks opened only by the anchored `Thread N is about to kill` header, `[ALERT] (PID) :`, `Current worker (N) exited with code N`; ANCHORED patterns, so keywords echoed from URLs/Host/server names never match) bypasses the bucket within a budget (1000 lines / 10 min, 60 s cooldown after a bound closes a block) and goes to STDERR (journald PRIORITY 3); routine reload noise is collapsed into one `[relay] reload/start ok: N routine lines collapsed` line per reload and everything benign goes to STDOUT (PRIORITY 6; docker's journald driver does not parse `<N>` prefixes), with a per-minute `[relay] suppressed N lines` summary. `python3 scripts/test-haproxy-stderr-relay.py` (offline) covers the relay
|
||||
+80
-18
@@ -23,23 +23,80 @@ LABEL org.opencontainers.image.title="haproxy-manager-base" \
|
||||
org.opencontainers.image.version="${VERSION}" \
|
||||
org.opencontainers.image.licenses="MIT"
|
||||
|
||||
# haproxy is PINNED. It was previously unpinned, so the binary could move under
|
||||
# us at Debian's timing — an upstream release that rejected our config would have
|
||||
# broken an unrelated commit's build, or worse, shipped an edge that refuses to
|
||||
# start (see the `haproxy -c` gate below for why that matters: a config HAProxy
|
||||
# rejects leaves the container Up with 80/443 unbound and /health still 200).
|
||||
# haproxy comes from the UPSTREAM-maintained apt repo https://haproxy.debian.net
|
||||
# (Vincent Bernat's builds, the Debian maintainer's own), suite
|
||||
# `trixie-backports-<series>`, NOT from Debian trixie's archive.
|
||||
#
|
||||
# Pinning does NOT make the gate redundant, and the gate does NOT make pinning
|
||||
# unnecessary — they compose. Pinned means the version moves deliberately; the
|
||||
# gate then answers immediately whether the new binary still accepts fleet config.
|
||||
# It also makes the image reproducible, which it previously was not.
|
||||
# WHY (2026-11-16 forensics): Debian trixie ships 3.0.11 (+ security revisions).
|
||||
# That build aborts via HAProxy's own watchdog inside the DNS resolver
|
||||
# (resolv_process_responses / snr_resolution_cb, haproxy issue #3101) - whole
|
||||
# edge down 15-52 s per abort on sdbees/whp01. The fix ("dns: bind the
|
||||
# nameserver sockets to the initiating thread", "resolvers: make
|
||||
# process_resolvers() single-threaded") is in 3.0.13+, 3.1.11+ and 3.2.6+;
|
||||
# Debian trixie will not take it. See docs/go-port/haproxy-edge-upgrade-and-logging.md
|
||||
# in the WHP repo for the version evidence and the rollout plan.
|
||||
#
|
||||
# To move it: bump the version here, rebuild, and let the gate verify. If Debian
|
||||
# security-updates the package (e.g. -1+deb13u4) the build FAILS until this pin is
|
||||
# updated — that failure is the point, not a bug. Check availability with:
|
||||
# apt-cache policy haproxy
|
||||
ARG HAPROXY_VERSION=3.0.11-1+deb13u3
|
||||
RUN apt update -y && apt dist-upgrade -y && apt install socat "haproxy=${HAPROXY_VERSION}" cron certbot curl jq net-tools -y && apt-mark hold haproxy && apt clean && rm -rf /var/lib/apt/lists/*
|
||||
# Series: 3.2 (LTS, supported to 2030-Q2). Its trixie package is built with
|
||||
# USE_QUIC=1 against the system OpenSSL 3.5.x NATIVE QUIC API (depends on
|
||||
# libssl3t64 >= 3.5.1), so no USE_QUIC_OPENSSL_COMPAT layer. (3.0.x upstream
|
||||
# has no trixie packages and would need a source build.)
|
||||
#
|
||||
# The repo keeps MANY patch versions of a series (27 as of 2026-11: 3.2.10 ..
|
||||
# 3.2.26, all flagged `NotAutomatic: yes`), so "latest of the series" would let any
|
||||
# rebuild silently take an untested 3.2.27+. The pin is therefore the EXACT Debian
|
||||
# package version, HAPROXY_EXACT (default below), and the build:
|
||||
# * pins apt to it (`Pin: version <full Debian version>`; NOTE a bare
|
||||
# `3.2.26` does NOT match: apt compares the FULL version "3.2.26-1~bpo13+1",
|
||||
# and with a non-matching pin apt falls back to Debian's own 3.0.11 package);
|
||||
# * FAILS unless `dpkg-query` reports EXACTLY HAPROXY_EXACT, and `haproxy -v`
|
||||
# reports the series (so an unreachable repo, which would leave Debian's
|
||||
# 3.0.11, fails closed, and a 3.2 -> 3.3/3.4 jump can never arrive by accident);
|
||||
# * records the resolved version in /haproxy/HAPROXY_VERSION; the `haproxy -c`
|
||||
# gate below proves the binary accepts the fleet config.
|
||||
# Taking a new patch is a DELIBERATE edit of the default below (or
|
||||
# --build-arg HAPROXY_EXACT=3.2.27-1~bpo13+1), tested before it merges; CI never
|
||||
# passes it, so a rebuild is reproducible.
|
||||
# The repo signing key is COMMITTED (keys/haproxy-debian-net/, Vincent Bernat,
|
||||
# fingerprint AEF2 3487 66F3 71C6 89A7 3600 95A4 2FE8 3535 25F9) and used with
|
||||
# signed-by, never fetched at build time. The fingerprint is asserted below.
|
||||
ARG HAPROXY_UPSTREAM=3.2
|
||||
ARG HAPROXY_EXACT=3.2.26-1~bpo13+1
|
||||
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
|
||||
# (CT130 pkg-cache, https only; CI sets it only after probing the cache with the
|
||||
# pinned CA). Unset = the upstream Debian repositories, exactly as before. The
|
||||
# apt sources are restored before this RUN ends (restore FAILS the build if any
|
||||
# reference to the cache is left), so no shipped layer carries the cache address.
|
||||
# NOTE: CT130 does not proxy haproxy.debian.net (see the design doc); that one
|
||||
# repo is always fetched directly.
|
||||
# Canonical script + docs: cloud-apache-container (scripts/pkg-cache-repos.sh,
|
||||
# docs/pkg-cache.md there); this repo carries byte-identical copies.
|
||||
ARG PKG_CACHE=
|
||||
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
|
||||
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
|
||||
COPY ./keys/haproxy-debian-net/haproxy-archive-keyring.gpg /etc/apt/keyrings/haproxy-archive-keyring.gpg
|
||||
RUN /usr/local/sbin/pkg-cache-repos.sh apt && \
|
||||
apt update -y && apt dist-upgrade -y && \
|
||||
apt install -y --no-install-recommends gnupg ca-certificates && \
|
||||
keys="$(gpg --show-keys --with-colons /etc/apt/keyrings/haproxy-archive-keyring.gpg)" && \
|
||||
[ "$(printf '%s\n' "${keys}" | grep -c '^pub:')" = 1 ] && \
|
||||
[ "$(printf '%s\n' "${keys}" | awk -F: '/^fpr:/{print $10; exit}')" = 'AEF2348766F371C689A7360095A42FE8353525F9' ] && \
|
||||
apt purge -y gnupg && \
|
||||
echo "deb [signed-by=/etc/apt/keyrings/haproxy-archive-keyring.gpg] https://haproxy.debian.net trixie-backports-${HAPROXY_UPSTREAM} main" > /etc/apt/sources.list.d/haproxy-debian-net.list && \
|
||||
apt update -y && \
|
||||
pin="${HAPROXY_EXACT}" && \
|
||||
printf 'Package: haproxy\nPin: version %s\nPin-Priority: 1001\n' "${pin}" > /etc/apt/preferences.d/haproxy-upstream-pin && \
|
||||
apt install socat haproxy cron certbot curl jq net-tools -y && apt-mark hold haproxy && \
|
||||
rm -f /etc/apt/preferences.d/haproxy-upstream-pin && \
|
||||
deb_ver="$(dpkg-query -W -f='${Version}' haproxy)" && \
|
||||
[ "${deb_ver}" = "${HAPROXY_EXACT}" ] || { echo "FATAL: installed haproxy '${deb_ver}' is not the pinned '${HAPROXY_EXACT}' (repo unreachable, or the pin did not match)" >&2; exit 1; } && \
|
||||
case "${deb_ver}" in "${HAPROXY_UPSTREAM}".*) ;; *) echo "FATAL: installed haproxy ${deb_ver} is not series ${HAPROXY_UPSTREAM}" >&2; exit 1;; esac && \
|
||||
haproxy -v | head -n1 | grep -F "version ${HAPROXY_UPSTREAM}." && \
|
||||
haproxy -vv | grep -E 'USE_QUIC=1|\+QUIC ' >/dev/null && \
|
||||
if haproxy -vv | grep -q 'USE_QUIC_OPENSSL_COMPAT=1'; then echo "FATAL: QUIC compat layer build, expected native OpenSSL QUIC" >&2; exit 1; fi && \
|
||||
mkdir -p /haproxy && { haproxy -v | head -n1; echo "package ${deb_ver}"; } > /haproxy/HAPROXY_VERSION && \
|
||||
echo "haproxy OK: series ${HAPROXY_UPSTREAM}, Debian package ${deb_ver} (exact pin)" && \
|
||||
/usr/local/sbin/pkg-cache-repos.sh restore && \
|
||||
apt clean && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /haproxy
|
||||
COPY ./templates /haproxy/templates
|
||||
COPY requirements.txt /haproxy/
|
||||
@@ -82,8 +139,8 @@ RUN pip install -r requirements.txt
|
||||
# * it cannot be skipped, and it protects local `docker build` too;
|
||||
# * no workflow restructuring (build-push-action builds and pushes in one
|
||||
# step, so gating in CI would mean splitting build from push);
|
||||
# * it validates against the EXACT haproxy binary in this image. Line 26
|
||||
# installs haproxy unpinned, so that binary moves between builds - this
|
||||
# * it validates against the EXACT haproxy binary in this image. The
|
||||
# installed haproxy can move between builds (Debian revisions) - this
|
||||
# turns "the new haproxy rejects our config" from a silent production
|
||||
# risk into a build failure.
|
||||
#
|
||||
@@ -92,14 +149,19 @@ RUN pip install -r requirements.txt
|
||||
RUN python3 /haproxy/scripts/test-wpadmin-gate.py \
|
||||
&& python3 /haproxy/scripts/test-trusted-proxy-gate.py \
|
||||
&& python3 /haproxy/scripts/test-xmlrpc-rate-limit.py \
|
||||
&& python3 /haproxy/scripts/test-default-splash.py \
|
||||
&& python3 /haproxy/scripts/test-access-log-receiver.py \
|
||||
&& python3 /haproxy/scripts/test-haproxy-stderr-relay.py \
|
||||
&& python3 /haproxy/scripts/test-haproxy-supervisor.py \
|
||||
&& python3 /haproxy/scripts/test-config-rollback.py \
|
||||
&& python3 /haproxy/scripts/test-cert-write-safety.py \
|
||||
&& python3 /haproxy/scripts/test-cert-scripts.py \
|
||||
&& python3 /haproxy/scripts/test-waf-body-inspection.py \
|
||||
&& python3 /haproxy/scripts/test-gunicorn-socket-timeouts.py \
|
||||
&& python3 /haproxy/scripts/test-gunicorn-access-logging.py \
|
||||
&& python3 /haproxy/scripts/validate-rendered-config.py
|
||||
# Create log directories
|
||||
RUN mkdir -p /var/log && touch /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log
|
||||
RUN mkdir -p /var/log/haproxy && touch /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log
|
||||
RUN chmod 755 /var/log/haproxy-manager.log /var/log/haproxy-manager-errors.log
|
||||
# Set up cron for certificate renewal with proper permissions and environment
|
||||
RUN mkdir -p /var/spool/cron/crontabs && \
|
||||
|
||||
@@ -379,9 +379,141 @@ tail -f /var/log/haproxy-manager-errors.log
|
||||
| `HAPROXY_DEFAULT_PAGE_TITLE` | Title for the default page | Site Not Configured |
|
||||
| `HAPROXY_DEFAULT_MAIN_MESSAGE` | Main message on the default page | This domain has not been configured yet. Please contact your system administrator to set up this website. |
|
||||
| `HAPROXY_DEFAULT_SECONDARY_MESSAGE` | Secondary message on the default page | If you believe this is an error, please check the domain name and try again. |
|
||||
| `HAPROXY_DEFAULT_SPLASH` | `off` (or `0`/`false`/`no`) disables the default splash for unrouted hosts and restores the default-backend page below | on |
|
||||
|
||||
## Default Splash (unrouted hosts)
|
||||
|
||||
Since 2026.11.1 (404-everywhere + robots since 2026.11.2), any request whose Host is **not routed to a site** -- the
|
||||
server's own hostname (registered cert-only, no backend), a bare IP, a stale
|
||||
or unknown name, a request with no Host at all -- is answered by HAProxy
|
||||
itself with the static page `errors/default-splash.html` (`http-request
|
||||
return ... file`): `404` on EVERY path (including `/`) so search engines never
|
||||
index the server's IP or hostname as a site, with `cache-control: no-store`,
|
||||
`x-robots-tag: noindex, nofollow, noarchive` and `x-content-type-options:
|
||||
nosniff`; `/robots.txt` is `200 text/plain` `User-agent: *` / `Disallow: /`
|
||||
(`errors/robots-unrouted.txt`). Monitors pointed at the bare server hostname
|
||||
on 80/443 must accept 404 (WHP's own probes -- healthcheck.sh, the watchdog --
|
||||
accept any status). The page
|
||||
names no version, echoes nothing from the request (it is loaded with `file`,
|
||||
not `lf-file`), and links nowhere -- in particular not to the control panel.
|
||||
|
||||
**Why.** `use_backend default-backend` does not short-circuit HAProxy's
|
||||
`http-request` rules, so the old catch-all was inspected by Coraza with the
|
||||
full server-wide rule set like customer traffic. A browser on
|
||||
`https://<server-hostname>/` also sends the control panel's cookies (cookies
|
||||
are not port-scoped, so `:8443` cookies go to `:443`), and anything Coraza
|
||||
matched there showed the WAF block page for a host with no site behind it.
|
||||
The splash rules sit before the SPOE call, so unrouted requests never reach
|
||||
the WAF; there is nothing behind them to protect.
|
||||
|
||||
**What still applies / is kept out:** rate limits, the IP block (blocked IPs
|
||||
still get the blocked-ip page), the wp-* gates, and the multipart guard run
|
||||
first; `/.well-known/acme-challenge/` still reaches certbot (the server
|
||||
hostname's own certificate is issued through it); hosts in
|
||||
`suspended_domains.list` still get the suspended page. Configured sites,
|
||||
wildcards and their WAF inspection are unchanged. Plain HTTP on `:80` gets
|
||||
the splash too, not a redirect: an unknown name usually has no valid
|
||||
certificate, so a redirect would trade a clean page for a TLS warning.
|
||||
Over TLS the presented certificate is whatever SNI selects -- the
|
||||
hostname's own cert if one is issued, otherwise the default bundle.
|
||||
|
||||
**How "routed" is decided.** `generate_config()` writes
|
||||
`/etc/haproxy/routed_hosts.list` (exact names) and
|
||||
`/etc/haproxy/routed_wildcards.list` (`.base` suffixes) from the same domain
|
||||
rows, in the same call, as the routing ACLs, and both are in the rollback set
|
||||
next to `haproxy.cfg`. The splash ACLs compare the Host port-stripped and
|
||||
lowercased, a superset of the routing ACLs (which compare the raw Host), so a
|
||||
host that routes is never splashed. Never hand-edit the lists; they are
|
||||
rewritten on every regeneration (also when the splash is off, so turning it
|
||||
on never meets a missing `-f` file).
|
||||
|
||||
**Deploying to an existing host.** The page ships at
|
||||
`/haproxy/errors/default-splash.html` and the template under
|
||||
`/haproxy/templates/` -- both in the image, OUTSIDE the `/etc/haproxy` named
|
||||
volume, so the volume cannot shadow them. The lists live in the volume but are
|
||||
generated, not copied. So an image bump plus a recreate of the container is
|
||||
the whole deployment: on start `generate_config()` renders the new listener
|
||||
and writes both lists, then HAProxy loads them. Nothing to copy into the
|
||||
volume, no WHP-side change. Verify after the recreate:
|
||||
|
||||
```bash
|
||||
docker exec haproxy-manager grep -c default-splash.html /etc/haproxy/haproxy.cfg # 2
|
||||
docker exec haproxy-manager cat /etc/haproxy/routed_hosts.list # every site
|
||||
curl -skI https://<server-hostname>/ # 404 + x-robots-tag noindex
|
||||
curl -sk https://<server-hostname>/robots.txt # User-agent: * / Disallow: /
|
||||
curl -sk -o /dev/null -w '%{http_code}\n' https://<a-customer-site>/ # unchanged
|
||||
```
|
||||
|
||||
Rollback: recreate on the previous image tag, or set
|
||||
`HAPROXY_DEFAULT_SPLASH=off` and regenerate.
|
||||
|
||||
Covered by `scripts/test-default-splash.py` (template order/conditions, the
|
||||
page, list generation) and the `splash-off` scenario of
|
||||
`scripts/validate-rendered-config.py`.
|
||||
|
||||
> gunicorn access logs (:8080 none, :8000 bounded file, not stdout): see docs/logging.md.
|
||||
|
||||
## Access Log (in-container, since 2026.11.2)
|
||||
|
||||
HAProxy logs (`log /log.sock ... format rfc5424`, `hap_header.tpl`) to a UNIX
|
||||
datagram socket inside its chroot (`/var/lib/haproxy/log.sock`).
|
||||
`scripts/haproxy-log-receiver.py`, started and supervised by `start-up.sh` (a
|
||||
restart loop: back within ~2 s), writes one line per request to
|
||||
`/var/log/haproxy/access.log`, rotated at 40 MB with 4 gzip generations
|
||||
(worst case 200 MB; `HAPROXY_ACCESS_LOG_MAX_BYTES`, `HAPROXY_ACCESS_LOG_BACKUPS`).
|
||||
No network listener anywhere. Fields: client/real IP (`cip=`), Host, status,
|
||||
termination state, request id, UA, SNI, HTTP version, WAF body stamp and the
|
||||
Coraza `wrule=/wrules=/wscore=` fields (unchanged `log-format`).
|
||||
|
||||
* **Never blocks HAProxy.** Sends are non-blocking: a dead or stalled receiver,
|
||||
or a full disk, only drops log lines (counted as `DroppedLogs` in
|
||||
`echo "@1 show info" | socat stdio /tmp/haproxy-cli`). Lab: with the
|
||||
receiver SIGSTOPped, 2500/2500 requests still answered 200 in ~1 s;
|
||||
1025 lines queued, 1475 dropped and counted.
|
||||
* **Not in `docker logs`.** The receiver prints no log lines (WHP's AI log
|
||||
monitor reads `docker logs haproxy-manager`; a per-request firehose there
|
||||
would storm it). Only its start line and rate-limited errors go to stderr.
|
||||
* **Persistence.** Mount a named volume at `/var/log/haproxy` (WHP:
|
||||
`-v haproxy-logs:/var/log/haproxy`) or the log resets on every recreate.
|
||||
Give the container `--sysctl net.unix.max_dgram_qlen=1024` (a netns starts
|
||||
at 10) for burst headroom.
|
||||
* `HAPROXY_SYSLOG_TARGET=<ip>:<port>` still overrides the target for a remote
|
||||
collector. The old default `172.18.0.1:514` (host rsyslog) is gone.
|
||||
|
||||
Reading it:
|
||||
|
||||
```bash
|
||||
docker exec haproxy-manager tail -f /var/log/haproxy/access.log
|
||||
docker exec haproxy-manager grep ' 403 ' /var/log/haproxy/access.log | tail
|
||||
docker exec haproxy-manager sh -c 'zcat /var/log/haproxy/access.log.*.gz' | grep <request-id>
|
||||
docker cp haproxy-manager:/var/log/haproxy/ ./edge-logs/ # or the volume's _data dir on the host
|
||||
```
|
||||
|
||||
Rollout: image bump + recreate with the volume and sysctl (WHP's
|
||||
`lib-edge-container.sh` adds both). The receiver, its script and the template
|
||||
are in the image, outside the `/etc/haproxy` named volume, so nothing can be
|
||||
shadowed; `haproxy.cfg` is regenerated on start with the new `log` line.
|
||||
|
||||
## HAProxy stderr (panic dumps, [ALERT]s) - since 2026.11.4
|
||||
|
||||
HAProxy is started as `haproxy -W -db` so the master keeps a real stderr (with
|
||||
the `daemon` keyword HAProxy points it at /dev/null, which discarded the
|
||||
`[ALERT]` lines, `Current worker (N) exited with code 134` and the whole
|
||||
watchdog/`ha_panic` thread dump). It is appended to
|
||||
`/var/log/haproxy/haproxy-stderr.log` (the `haproxy-logs` volume, so it
|
||||
survives a recreate) and `scripts/haproxy-stderr-relay.py` re-emits every line
|
||||
on the container's stderr prefixed `haproxy-stderr: ` (rate-limited, file
|
||||
rotated at 20 MiB x 3). `docker logs haproxy-manager` and whatever docker
|
||||
logging driver the host configured therefore carry the crash text; the
|
||||
per-request access log stays out of it. The in-container supervisor probes
|
||||
with `pidof` every `HAPROXY_SUPERVISOR_INTERVAL` seconds (default 2, was 15) and
|
||||
backs off to `HAPROXY_SUPERVISOR_MAX_BACKOFF` (30) only when a restart attempt
|
||||
fails. Measured (3.2.26, empty config): crash to listeners back = 1.2-2.0 s.
|
||||
|
||||
## Default Backend Configuration
|
||||
|
||||
With the default splash on (above), unrouted hosts no longer reach this page; it still serves the blocked-IP and suspended pages, and serves unrouted hosts again when `HAPROXY_DEFAULT_SPLASH=off`.
|
||||
|
||||
When a domain is accessed that hasn't been configured in HAProxy, the system will serve a default page instead of showing an error. This default page:
|
||||
|
||||
- Informs visitors that the site is not configured
|
||||
|
||||
+40
-5
@@ -16,25 +16,60 @@ ARG CORAZA_SPOA_VERSION=v0.7.1
|
||||
# then for this image's golang:1.25), so both are mirrored at
|
||||
# repo.anhonesthost.net via the .gitea/workflows/mirror-base-image.yaml
|
||||
# weekly job.
|
||||
FROM repo.anhonesthost.net/cloud-hosting-platform/golang:1.25 AS build
|
||||
#
|
||||
# Multi-arch (linux/amd64 + linux/arm64) WITHOUT QEMU: both Go stages are pinned
|
||||
# to the BUILD host's platform (--platform=$BUILDPLATFORM, so the amd64-only
|
||||
# golang mirror is enough) and the binary is CROSS-COMPILED for the target via
|
||||
# GOOS/GOARCH. CGO is off (it already was), so cross-compiling is exact. Only
|
||||
# the final distroless stage is per target platform (gcr.io publishes arm64).
|
||||
FROM --platform=$BUILDPLATFORM repo.anhonesthost.net/cloud-hosting-platform/golang:1.25 AS build
|
||||
ARG CORAZA_SPOA_VERSION
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
# OPTIONAL internal package cache, e.g. --build-arg PKG_CACHE=https://192.168.1.40
|
||||
# (CT130 pkg-cache, https only; CI sets it only after probing the cache with the
|
||||
# pinned CA). Unset = upstream Debian repositories and the default GOPROXY,
|
||||
# exactly as before. Go modules stay verified against go.sum / sum.golang.org
|
||||
# (GOSUMDB unchanged). Every RUN that rewrites ends with `restore`, which FAILS
|
||||
# the build if any reference to the cache is left. Only build stages use it;
|
||||
# nothing from here reaches the distroless runtime image except the binaries.
|
||||
# scripts/ + keys/pkg-cache/ in this directory are byte-identical copies of the
|
||||
# repo-root ones (this image's build context is ./coraza-spoa); canonical
|
||||
# source: cloud-apache-container (scripts/pkg-cache-repos.sh, docs/pkg-cache.md).
|
||||
ARG PKG_CACHE=
|
||||
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
|
||||
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
|
||||
WORKDIR /src
|
||||
RUN apt-get update \
|
||||
RUN /usr/local/sbin/pkg-cache-repos.sh apt \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git \
|
||||
&& /usr/local/sbin/pkg-cache-repos.sh restore \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# The source itself is cloned from GitHub over HTTPS (not proxied).
|
||||
RUN git clone --depth 1 --branch "${CORAZA_SPOA_VERSION}" \
|
||||
https://github.com/corazawaf/coraza-spoa.git . \
|
||||
&& eval "$(/usr/local/sbin/pkg-cache-repos.sh go-env)" \
|
||||
&& go mod download \
|
||||
&& CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/coraza-spoa .
|
||||
&& CGO_ENABLED=0 GOOS="${TARGETOS:-linux}" GOARCH="${TARGETARCH:-amd64}" go build -trimpath -ldflags='-s -w' -o /out/coraza-spoa . \
|
||||
&& /usr/local/sbin/pkg-cache-repos.sh restore
|
||||
|
||||
# Catalog extractor: walks the bundled CRS at build time and emits
|
||||
# rules-catalog.json so WHP's UI can render rule metadata without parsing
|
||||
# .conf files at runtime. Uses the SAME coraza-coreruleset version pin as
|
||||
# the coraza-spoa binary above (drift between the two would mislabel rules).
|
||||
FROM repo.anhonesthost.net/cloud-hosting-platform/golang:1.25 AS catalog
|
||||
# Runs at build time, so it is built and RUN for the build platform (no
|
||||
# GOOS/GOARCH); its JSON output is architecture-neutral and is copied into
|
||||
# every target platform's image.
|
||||
FROM --platform=$BUILDPLATFORM repo.anhonesthost.net/cloud-hosting-platform/golang:1.25 AS catalog
|
||||
# Same optional package cache as the build stage (module downloads only).
|
||||
ARG PKG_CACHE=
|
||||
COPY ./scripts/pkg-cache-repos.sh /usr/local/sbin/pkg-cache-repos.sh
|
||||
COPY ./keys/pkg-cache/ /usr/local/share/pkg-cache/
|
||||
WORKDIR /src
|
||||
COPY catalog-extractor/ .
|
||||
RUN go build -trimpath -o /out/catalog-extractor . \
|
||||
RUN eval "$(/usr/local/sbin/pkg-cache-repos.sh go-env)" \
|
||||
&& go build -trimpath -o /out/catalog-extractor . \
|
||||
&& /usr/local/sbin/pkg-cache-repos.sh restore \
|
||||
&& /out/catalog-extractor > /out/rules-catalog.json
|
||||
|
||||
# Distroless runtime: no shell, no package manager, no /tmp by default —
|
||||
|
||||
@@ -76,3 +76,33 @@ When a legitimate request triggers a blocked rule, the audit log shows the rule
|
||||
2. **Drop from the enforce list**: remove the rule's ID range from the `ctl:ruleEngine=On` overrides; it falls back to detect-only.
|
||||
|
||||
After tuning, push the change — CI rebuilds, then `recreate coraza-spoa` on each host to apply.
|
||||
|
||||
## Multi-arch (linux/amd64 + linux/arm64)
|
||||
|
||||
The image is built for both platforms **without QEMU**: the two Go stages are
|
||||
`FROM --platform=$BUILDPLATFORM` (the in-house `golang:1.25` mirror is amd64-only,
|
||||
which is enough) and the binary is cross-compiled with
|
||||
`CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH`. The catalog-extractor runs at
|
||||
build time on the build platform; its JSON output is architecture-neutral. Only the
|
||||
final distroless stage is per target platform. The amd64 image content is
|
||||
byte-identical to the pre-multi-arch build (measured: same rootfs tree, same
|
||||
`/coraza-spoa` sha256).
|
||||
|
||||
CI (`.gitea/workflows/build-push-coraza.yaml`, one job on the trusted x86 runner):
|
||||
amd64 and arm64 are each pushed **by digest, untagged, to both registries**
|
||||
(Gitea + GHCR; the same digest lands in both), then
|
||||
`scripts/assemble-multiarch-index.sh` creates `:latest` in each registry with
|
||||
`docker buildx imagetools create` from `<repo>@<amd64>` and `<repo>@<arm64>`.
|
||||
The arm64 step is `continue-on-error`; when it fails the tag is published
|
||||
amd64-only, as before, and the arm64 digest is only ever taken from the current
|
||||
run's step output (a previous build's arm64 is never re-attached).
|
||||
|
||||
Attestations: buildx provenance is kept per platform. A by-digest push with
|
||||
attestations is a small per-platform index (image + `attestation-manifest`);
|
||||
`imagetools create` over two of them yields one index with all four entries
|
||||
(amd64, its attestation, arm64, its attestation), each attestation still pointing
|
||||
at its own platform manifest; `imagetools inspect --format '{{json .Provenance}}'`
|
||||
shows both. Nothing flattens or drops them.
|
||||
|
||||
Local rehearsal (no real registry is touched): `scripts/e2e-coraza-multiarch.sh`
|
||||
(usage in its header).
|
||||
@@ -0,0 +1,32 @@
|
||||
Trust material for the OPT-IN internal package cache (scripts/pkg-cache-repos.sh,
|
||||
docs/pkg-cache.md). Copied into images at /usr/local/share/pkg-cache/; nothing
|
||||
here is added to the system trust store.
|
||||
|
||||
ca.crt "pkg-cache internal CA (CT130)", EC P-256, valid to 2036-10-18.
|
||||
NAME-CONSTRAINED to IP 192.168.1.40 and DNS
|
||||
pkg-cache.internal.cloud-hosting.io / pkg-cache.geekhome.local.
|
||||
SHA-256 fingerprint BB:5B:9A:08:05:ED:2F:58:83:92:D0:99:9D:11:0A:30:14:88:68:F0:F0:88:65:5E:94:0B:75:EB:C2:A2:BD:F3
|
||||
(private key lives only on CT130: /etc/pkgcache-ca/ca.key)
|
||||
RPM-GPG-KEY-EPEL-10 Fedora (epel10) 7D8D 15CB FC4E 6268 8591 FB26 33D9 8517 E37E D158
|
||||
fetched 2026-10-21 over HTTPS from dl.fedoraproject.org and
|
||||
mirror.math.princeton.edu (identical); fingerprint matches
|
||||
https://fedoraproject.org/security/
|
||||
RPM-GPG-KEY-remi2024 Remi's RPM repository 2024 CF1D F005 7CE8 5DFF 5B2F 2A37 C2FD 3B2C 2A09 48E4
|
||||
fetched 2026-10-21 over HTTPS from rpms.remirepo.net; byte-identical
|
||||
to the key shipped in remi-release-10.2-1.el10.remi. Not found on
|
||||
keys.openpgp.org / keyserver.ubuntu.com - confirm out of band.
|
||||
Remi rotates keys (remi2025, ...): when remi-release starts being
|
||||
signed by a new key, `release-install remi` fails loudly - add the
|
||||
new key here (and to pkg-cache-repos.sh).
|
||||
RPM-GPG-KEY-EPEL-9 Fedora (epel9) FF8A D134 4597 106E CE81 3B91 8A38 72BF 3228 467C
|
||||
fetched 2026-10-22 over HTTPS from dl.fedoraproject.org and
|
||||
mirror.math.princeton.edu (identical); used by `release-install epel`
|
||||
on EL9 images (cloud-memcached-container, cloud-haproxy-container).
|
||||
RPM-GPG-KEY-nodesource Nodesource Operations <operations@nodesource.com>
|
||||
242B 8138 31AF 0956 2B6C 46F7 6B88 DA4E 3AF2 8A14 (rsa4096, 2023-12-22)
|
||||
fetched 2026-10-22 over HTTPS from
|
||||
https://rpm.nodesource.com/gpgkey/ns-operations-public.key (the key
|
||||
NodeSource's own setup_N.x scripts reference); verified by dnf
|
||||
gpgcheck against the nodejs RPMs. Not on keys.openpgp.org /
|
||||
keyserver.ubuntu.com - confirm out of band. Used by
|
||||
`pkg-cache-repos.sh nodesource MAJOR` (cloud-node-container).
|
||||
@@ -0,0 +1,29 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA
|
||||
Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK
|
||||
Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L
|
||||
URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf
|
||||
mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2
|
||||
XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI
|
||||
uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60
|
||||
GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix
|
||||
fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z
|
||||
BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77
|
||||
HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB
|
||||
tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB
|
||||
CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC
|
||||
BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p
|
||||
l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp
|
||||
yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg
|
||||
Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u
|
||||
scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs
|
||||
7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK
|
||||
F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg
|
||||
sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB
|
||||
0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY
|
||||
1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt
|
||||
TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k
|
||||
p9Dfd2pKuxK9Mys=
|
||||
=mhQZ
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,29 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGE3mOsBEACsU+XwJWDJVkItBaugXhXIIkb9oe+7aadELuVo0kBmc3HXt/Yp
|
||||
CJW9hHEiGZ6z2jwgPqyJjZhCvcAWvgzKcvqE+9i0NItV1rzfxrBe2BtUtZmVcuE6
|
||||
2b+SPfxQ2Hr8llaawRjt8BCFX/ZzM4/1Qk+EzlfTcEcpkMf6wdO7kD6ulBk/tbsW
|
||||
DHX2lNcxszTf+XP9HXHWJlA2xBfP+Dk4gl4DnO2Y1xR0OSywE/QtvEbN5cY94ieu
|
||||
n7CBy29AleMhmbnx9pw3NyxcFIAsEZHJoU4ZW9ulAJ/ogttSyAWeacW7eJGW31/Z
|
||||
39cS+I4KXJgeGRI20RmpqfH0tuT+X5Da59YpjYxkbhSK3HYBVnNPhoJFUc2j5iKy
|
||||
XLgkapu1xRnEJhw05kr4LCbud0NTvfecqSqa+59kuVc+zWmfTnGTYc0PXZ6Oa3rK
|
||||
44UOmE6eAT5zd/ToleDO0VesN+EO7CXfRsm7HWGpABF5wNK3vIEF2uRr2VJMvgqS
|
||||
9eNwhJyOzoca4xFSwCkc6dACGGkV+CqhufdFBhmcAsUotSxe3zmrBjqA0B/nxIvH
|
||||
DVgOAMnVCe+Lmv8T0mFgqZSJdIUdKjnOLu/GRFhjDKIak4jeMBMTYpVnU+HhMHLq
|
||||
uDiZkNEvEEGhBQmZuI8J55F/a6UURnxUwT3piyi3Pmr2IFD7ahBxPzOBCQARAQAB
|
||||
tCdGZWRvcmEgKGVwZWw5KSA8ZXBlbEBmZWRvcmFwcm9qZWN0Lm9yZz6JAk4EEwEI
|
||||
ADgWIQT/itE0RZcQbs6BO5GKOHK/MihGfAUCYTeY6wIbDwULCQgHAgYVCgkICwIE
|
||||
FgIDAQIeAQIXgAAKCRCKOHK/MihGfFX/EACBPWv20+ttYu1A5WvtHJPzwbj0U4yF
|
||||
3zTQpBglQ2UfkRpYdipTlT3Ih6j5h2VmgRPtINCc/ZE28adrWpBoeFIS2YAKOCLC
|
||||
nZYtHl2nCoLq1U7FSttUGsZ/t8uGCBgnugTfnIYcmlP1jKKA6RJAclK89evDQX5n
|
||||
R9ZD+Cq3CBMlttvSTCht0qQVlwycedH8iWyYgP/mF0W35BIn7NuuZwWhgR00n/VG
|
||||
4nbKPOzTWbsP45awcmivdrS74P6mL84WfkghipdmcoyVb1B8ZP4Y/Ke0RXOnLhNe
|
||||
CfrXXvuW+Pvg2RTfwRDtehGQPAgXbmLmz2ZkV69RGIr54HJv84NDbqZovRTMr7gL
|
||||
9k3ciCzXCiYQgM8yAyGHV0KEhFSQ1HV7gMnt9UmxbxBE2pGU7vu3CwjYga5DpwU7
|
||||
w5wu1TmM5KgZtZvuWOTDnqDLf0cKoIbW8FeeCOn24elcj32bnQDuF9DPey1mqcvT
|
||||
/yEo/Ushyz6CVYxN8DGgcy2M9JOsnmjDx02h6qgWGWDuKgb9jZrvRedpAQCeemEd
|
||||
fhEs6ihqVxRFl16HxC4EVijybhAL76SsM2nbtIqW1apBQJQpXWtQwwdvgTVpdEtE
|
||||
r4ArVJYX5LrswnWEQMOelugUG6S3ZjMfcyOa/O0364iY73vyVgaYK+2XtT2usMux
|
||||
VL469Kj5m13T6w==
|
||||
=Mjs/
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,52 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGWFsqMBEAC4YlSuNpHjOiQVTC8StolVx3J8E9SC6G4wmVWbytMD1Zivfanj
|
||||
3LKDqdy7vpbViXA9whpyGQgQYdOlPKaNHWw4GSeGTnP1FbvaK1NJ9MnAzUysh49R
|
||||
cqoktYeIQbqgOgZHw5/gVH5bYObf1KJQ2LY/je9WTlW3QMRe1TaW0F0AZnJMBIOG
|
||||
JllE33QIpxDnOQHnvdUGq+HscUOTrW4Q3AdzwsQxU4GX82d9qFFr8HjAmUjlUKyh
|
||||
PgvMpb8VDvR7NE6et+wWh6BMYILo3VJFP7/gTXGiy3WWs5GW/PNXIXw5/f2w1B+e
|
||||
Dh5tlfGG4q/4P8mbv7C6RPHoF2Lq+W69HLqSq/l42WUL41tHlFTsluCIKpeeMxAu
|
||||
olxx3Z5momHMMyh7rbZtQDqFMTAqxNjT1NinKkWyj7KP5WE9hVosyd3UMR4UYDMm
|
||||
Zb6LI4N7Hht887+CY7grxTJ9cGQhMFfvLyvlf/FWQY4S6G7sEaUV71N9StvXBNG/
|
||||
iNoIMzhwjBmrmvT/UGPkLSi0SIBOxeGw2rwngnYikIJxuTQ4V40lm8o6eMFca7rW
|
||||
BQvAEncV8pvi/3FqrUBZ/U53s7NO24hBDoaQPzFbVCw0GHHf9GD7KoQb5c9ONoCZ
|
||||
Qu2YWo7ajPLzl/h2pJ2Kn29Ywev9MpuuPTGvifiKilcBVLYP3ZJg4ZfRBwARAQAB
|
||||
tDFOb2Rlc291cmNlIE9wZXJhdGlvbnMgPG9wZXJhdGlvbnNAbm9kZXNvdXJjZS5j
|
||||
b20+iQJOBBMBCgA4FiEEJCuBODGvCVYrbEb3a4jaTjryihQFAmWFsqMCGwMFCwkI
|
||||
BwIGFQoJCAsCBBYCAwECHgECF4AACgkQa4jaTjryihTD+hAAtTU155ke7xmnT0ph
|
||||
odbGjm3mOGdkdaDH+xmWtPtufiPncgHMCYh4aIDRloog8rFE+pP3qfwJmUKxthwD
|
||||
drrLAzfUwOrGxPnYH3MWwg66RVVoiiU+Hh42gJTez+P/K0VMQZXH0XTZD96TiwT/
|
||||
S0+WvQPuJ9pda2L4cjV+YP2+m0ERUG9QPj0Shgw+/PWCGWKD6J9DRToiwfwCTOvc
|
||||
DNolF1EcRlfPMTsqI2Bdc+TQywqU1Py1MUDSAPXxdK6nW2oM7eC5l3XXefcQ6JOS
|
||||
AuNdEDIsaf+H76j6/wSGsuIcJjUo9wmKjEV1iJYwT6jLbjaVQxXgEPHBSb+kzanx
|
||||
FWyrYP6TZpeY2gT4bBMsOKd7ay1I3/nIHwe6x8Yzkqzz+7YVhNpMbybH72NTxDWQ
|
||||
UmiboJs9r7j1lkHlBjxU9RLF5laYSeCIXD62KQVoP5ao1LSPKsCXFqcdpUgLYtRv
|
||||
CXxdriiHp4m9eGzbS7RW1TLm6tMOKbI4LzOtMm4DoZzGD2FTfsUtn6ALx9FSgvl/
|
||||
VMlzqZes+rclcukACtutvtB8e0yRhuzPUN56ctLEZfI1XzQFoN8Vdj6HWtiXWRiJ
|
||||
EohuufE2cEebmX90NRsbh4KAFXNEQROlMeMf7ufuSsk7QXLTutxmWSQvWwyS+P5/
|
||||
ZqCapIDlRCJFZX4N9a1IXo0OgCC5Ag0EZYWyowEQAKz7sA6WBKFVcmzcyr6y9TI5
|
||||
RemnzotRUdqXj8tD70xvV1nBEf+BfTw3hsiV019ZboPBae+cRaD4plQbNb0du/+Q
|
||||
UFq4ja055ZEld01RqE49L562pWvA5cwSmjs9+ZMI1ja0HbTb2K4razpzX3be+Zul
|
||||
TbH20X5FHTz+Jyfmw+G3Y9okw3OIzuYo16Lv/Qs8VoBmj01PItzq8dX5wJiIlVxU
|
||||
8wKHKy1kiKGCgyPnvqcXCGen0EoUS9wsWK1QHfryscYGVF9y1TBQvJa9HIpWPrIZ
|
||||
slSiVS4tQhGHfLDVlaDsYFrH9SIUkvW7peSEiPE1TKQjvbMS3CcUZRaMcGfgyDPe
|
||||
gXwnNwZbJ2/cisXbdAJaGT49V1Y+Sc1oKSGSG4FaGUNhS3p3cB7qlX2zS/sR2ael
|
||||
90fXR6nhjvhMAS98i7663pCUv9q0Z/NXNZwdXjbSF4W8gq3PVt7hCnJrngsGSkGu
|
||||
Qz0dc227/sAjh9CxYm8kQqUuQtiPVfKwkcUUT+O7jr2/6eAXZbTYVZbvaLEGMrKA
|
||||
bdWKpZMMOKAI45f+ZAcEmJG4D68N0jzVrLCXMuuLq/yqqyEmxDjSOH0qVutl8jxg
|
||||
vfxPxKYX8nnAzJZJHGHmV/ihjfClSdYePKExHuFVVhvc4BxE1XfPUeCZG4JpB9/B
|
||||
gNCiDPsg4JIw6GO5LS2xABEBAAGJAjYEGAEKACAWIQQkK4E4Ma8JVitsRvdriNpO
|
||||
OvKKFAUCZYWyowIbDAAKCRBriNpOOvKKFOTKD/40WN6ZRfP6hVLAX26DdyN9Bqzg
|
||||
D+0hJilmHL1YFeLt9U/YPhOZJxPaDGrMfldC0FSXKKr7H5C7FQErl936108wGohz
|
||||
53KGbNid0crNeaWrQuWk3QjaMF62Yc+NVB8b5LoEk0TEt9KPO6kveYDtW6Ygkw9i
|
||||
5aaS/AC+8GChOH6lWhLHcO91T7+Hi5qRez7qyme/AlHUTV9/jE6JgFJ7thCCAqBQ
|
||||
AfX3mGFlTwSTMHDeYq81MMl9UhYgoFJtx8PuqmSV7Eb29ccJofz3J+b5OJvpmLSZ
|
||||
Hu22Csa8k2V/1ojeL0DjQHD06V75F7BerxLR/rbJ64GkIiOdm4ntXC4a335gjQfM
|
||||
5tI6LtGsIpr+Le4C068eDj7YpzS0ql3g6o4A1TbtapLccgEj4iw95H4KKh0Ro5r0
|
||||
0s6LoSLu9uva3LiLVn6xXvpw3iv8FnhTr5PEiPnorXvxcCnUrB3Rkk6Wh1SKjpxa
|
||||
u6KiFzH0CXR1AgUdWSC0rAmvEdFjXgaoQd+Z2eJFHEAsg91URd9NtdeDsyMwdbHh
|
||||
iBZPGX5ddPjJA7nwKbkibCv7xiodd9ph8NmGLqD/2SDU1rqvPuCDMqLuxc89Gvhs
|
||||
3CeLTSJK9Uzb3oNdZLxL8CECip1ZkowVfTL+jY23phS9+Z0Db/yZqot3ZZJAAG8g
|
||||
oSVkzT/Mwp9zFVQu5w==
|
||||
=drMP
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,52 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGWdEG0BEADV9c8aSIITQmoWKN0ac4c8AEhZcdzpWSRPwHIS8syVRw/gj30W
|
||||
Mn1OoxqFn2UECh2jjqYaQFECXNrM2/lzO4GhvcuoPn6p0rXFMvYvwtiOJ+rmDvTe
|
||||
UOwRxkZhYehpvsRPFoVhlYJh3U1BM0UHm2iE/TIqVusY9FHQuCMZVl0BHk81YGDa
|
||||
qE3cyu+9zOs8z1bYlsYBgLIbtb4+loQTRexFTfZkUcCWgAaNyur8Wy7Msdm5LOqN
|
||||
ZZu6bNKBOv0Cr+BOwGNoFRbi4IM+5H+0eI1n7V5if4CuXfdYHp+Pm+PQvftm/nqB
|
||||
jRLKqioRKzHjk7xNMsi5lM+SU4WyfU77pr3RXuJR6fkymLVAeu5sy2U6ySj0SXIO
|
||||
/I3oPp4/zJ8/e94jG5iDqh92b5IWyXSE+EGw4kkpUvZLrK8vyyEfk8h9Gf8qA5uL
|
||||
yGfw0vnSVjYb+RdnfZFVdPaIEmeY8tnobS87duj8JESZiSWo5sU4filkciW5e+MJ
|
||||
w9wohgl+Cqm2J5Ljqus6DRm4ptKbZD8QZRnmGcSk6O2VopBKt2XLat4eIJLr/EiN
|
||||
0CJ1EH/oJmEeZbYtUjxjsGsbPXicwDH6ZIeIMSaiDgqRxPm1LTGVLjSHAk+k0e5l
|
||||
ulEmaoA7t/p3U2peGc7ONvozXiyNHZdGiRkqFC1mA9/JpeWw4KFH6JSgmQARAQAB
|
||||
tEtSZW1pJ3MgUlBNIHJlcG9zaXRvcnkgMjAyNCAoaHR0cHM6Ly9ycG1zLnJlbWly
|
||||
ZXBvLm5ldC8pIDxyZW1pQHJlbWlyZXBvLm5ldD6JAlEEEwEIADsWIQTPHfAFfOhd
|
||||
/1svKjfC/TssKglI5AUCZZ0QbQIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIX
|
||||
gAAKCRDC/TssKglI5Na7EACDTGjTcuTyudfuu/JC41BSurclCcwZyPsGaClyc0qx
|
||||
Tk8Rs4rbyVw1u494fiFG/yEVl7LGGja84UJOlMdw82KgVUmr4fGLaBBFWm+KnZ4W
|
||||
4vvFcCIxUshENOaDrkhby9F5P379gVOK9I3n9l4Q2xt+sWUNwE74CNfCTUeYMIm/
|
||||
/AHbb1DTM9FuIT6FssgvILMTlksiyrw8U8bj+QUzPZrFCpvv+PnFA7ZDMEyCf2Tz
|
||||
/zawxoP3a3b+3/g61U9SVIkfToonnorqh9lkQYhsdKTFHW6eESqe7zcqMBZv/sSc
|
||||
3kFzl22KczSdU8eYxdujv9BmlW2SOrQsnuMYxvoD1LzhwSB/nOapNrWU+djhJCof
|
||||
ueZ+rBW4QWysmG0dEp5Wpz85TstAnQ+j5aCOMSAkiAVsrJ0Ff3JXakIiL7rEtccS
|
||||
dhgBT06LJtJy1NGIC0Sotn45n6dtfi6IuI8LnELCKeHxhdFSJcAPB8y95u0uHymy
|
||||
sqMfzjq1VcYtzC5II6Yd5Gtb2+QzyQwORslwXhUdXwZKUc5i6X5xFMSxU523fH9n
|
||||
i+kQlUTPkOUr66v4fmYDj+ZlJQSFoPylcHCke/vOo/rH5nj+YMPa6LdohsdzHKpd
|
||||
DkKrk+Udj5aDXtXGIFYwoNKO4BnCsz/ImZfB4CQ+ekJsY1pFill3+MRTM6IvSA+P
|
||||
7LkCDQRlnRBtARAAq8k4FM9euMMsXz0buxxyr0ew0ecWNhZGy24a6DXORVTEJbgI
|
||||
T3HpV/C2PWkwgEBswYzREm+en61jhWr5unCiytO2klT+K2WNcIUNW+z2ciL/JNaJ
|
||||
OcKwjkiY/A+KtDStzllCQ/XnpgAXGvFZjMjhHrNGQoAwNnPE/cMtYwcuVdW3Ipm3
|
||||
cOOtzxRmfnhP8YUEyHxdos/7AZowaiLIUkaL5RUaygGrfPGyUXPl1aXS1oiOm37W
|
||||
RiKCGUxCHSlpBqeh1WQPOaps8y+ME9jMzF9SLvqiXlKPllJIx9i8tHUOnX6Y2U2x
|
||||
5sXsCEiA+zNVMU3YQ04++Yx2Rp6sDhNEnAbTCqqequqIzaXHZWwszAeG6pj1G90m
|
||||
8PwDeHy641ySgDMwVPpnH5rHWmmT96XvF9QNg07OBtW9/phryzwAuHogrBY4DyFD
|
||||
pUxDkTv9TxfBaS/s1uJxEGknIgnKzQo79mRy206waGKRA2AV36ORSfU12sK8pl3r
|
||||
N95iIDh/OIuMl2STaXilhwb1hRpGFvwRlq3Hvvan4SY1aO219Q+VxMGXeqC5F/rm
|
||||
Anh7mRVMP2qu0rj7L8xJ/OD/bPfuWL72/PvviBr/jdRM9I+Vj8dPGjhUwhK1dpEG
|
||||
Q6yvoVXLHXucVHfIVR+ALLBmK8oHXNIFeIXhiUbgmRAilPHRz6Rf5pgbrC0AEQEA
|
||||
AYkCNgQYAQgAIBYhBM8d8AV86F3/Wy8qN8L9OywqCUjkBQJlnRBtAhsMAAoJEML9
|
||||
OywqCUjkQ2kP/iwqvt1sX85KM4OSE9ftqemwnvh19NizMKHMYGgikMAvy2CndEOI
|
||||
wzKVeE1iK9/ivZQQIbD5l0G9hzVZjIxbgu9dAcxM+jdmaPyUj7yn6AX2yeGAeHyA
|
||||
qzNKZaTLJPieRzdXk/uKPVyWUuzbR0vwR7n8N5N0pe/FEfO9O0uX5iz2kf5+b10c
|
||||
WqvzLmO+czuyCjTHnjhKfE73iIktYE3MnM8qFHPiKwm+jeTkUnmInoR3qIKR/SCV
|
||||
u2vk/q5uvpNaUSPSDKji6wh3/hbNbPocLSGIjugfWNm/4LhlSHPNnRJ2dIvSxeiI
|
||||
qtSWxhULEyHGBPMLDriVumCV7Rb/byJwiMbmCIpxB5XLtbgt7nIqWQTko+6ci71S
|
||||
4rDXolM3bytYbpqq4b/Xc1bhBuTX3omwrd0kxmGqad8eia6O6iBsP7GAl4QkySm4
|
||||
MfKQfohjXTnT60wqEzQwLGI4NKN9bHRSsGINHX5p+Kz9HLf87fTY4uglbFxg0/II
|
||||
Y3nko5KBw/gfe0WpwZY0sWjK20cXJn3xLQjoG8OjUNUCUYTRsiAvn4AiDRt++qtl
|
||||
bvJjJqOU1k4GrGoCVpEUiOc9Cw1YZREcFe8H2at3pGnxJ1DFlYA916kNAxYANp2s
|
||||
r3N3agwevgIzP78hynxjm6a5H2w0vkg189pZn2RR9zX9gmrvlqg1o0P1
|
||||
=meaU
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,15 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIICPzCCAeagAwIBAgIUJI0QJZ+PCrSQ/nKZFENwwcp8Zr0wCgYIKoZIzj0EAwIw
|
||||
TDEmMCQGA1UEAwwdcGtnLWNhY2hlIGludGVybmFsIENBIChDVDEzMCkxIjAgBgNV
|
||||
BAoMGWNsb3VkLWhvc3RpbmcuaW8gaW50ZXJuYWwwHhcNMjYxMDIxMTkzNjIyWhcN
|
||||
MzYxMDE4MTkzNjIyWjBMMSYwJAYDVQQDDB1wa2ctY2FjaGUgaW50ZXJuYWwgQ0Eg
|
||||
KENUMTMwKTEiMCAGA1UECgwZY2xvdWQtaG9zdGluZy5pbyBpbnRlcm5hbDBZMBMG
|
||||
ByqGSM49AgEGCCqGSM49AwEHA0IABBJh75sC5vSzTnkYl0CYYEA/TKUHEXhbtanV
|
||||
OVNW4NlD3za5+bu5YuKjpbFAva4TCZqWsaS9IMd/JfsaFbf5XVejgaUwgaIwEgYD
|
||||
VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFP7UCTf/
|
||||
v7KU1KBrs1vqa71/TZl0MF0GA1UdHgEB/wRTMFGgTzAKhwjAqAEo/////zAlgiNw
|
||||
a2ctY2FjaGUuaW50ZXJuYWwuY2xvdWQtaG9zdGluZy5pbzAaghhwa2ctY2FjaGUu
|
||||
Z2Vla2hvbWUubG9jYWwwCgYIKoZIzj0EAwIDRwAwRAIgOod4wPHu0kAzYAw8KeGg
|
||||
cIrFMwQ4ATKVTkDioiSFOqICIEhY+sGj/YZ4oBMst3b/AAz9zMEJAXpnCkSTI6tM
|
||||
YAIl
|
||||
-----END CERTIFICATE-----
|
||||
Executable
+296
@@ -0,0 +1,296 @@
|
||||
#!/bin/bash
|
||||
## pkg-cache-repos.sh — BUILD-TIME ONLY, OPT-IN. Points dnf/apt at the internal
|
||||
## package cache (nginx proxy_cache, CT130 "pkg-cache" on the Proxmox cluster)
|
||||
## when PKG_CACHE is set; without it the repo files are left alone and builds
|
||||
## use the upstream repositories as before.
|
||||
##
|
||||
## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi/NodeSource
|
||||
## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu, Debian, LiteSpeed
|
||||
## pkg-cache-repos.sh release-install epel [remi] # GPG-verified release RPMs (EL9/EL10)
|
||||
## pkg-cache-repos.sh nodesource MAJOR # NodeSource repo file + pinned key
|
||||
## eval "$(pkg-cache-repos.sh go-env)" # GOPROXY via the cache (go.sum/sumdb verified)
|
||||
## pkg-cache-repos.sh restore # undo everything above
|
||||
##
|
||||
## CANONICAL copy: cloud-apache-container/scripts/pkg-cache-repos.sh (docs/pkg-cache.md
|
||||
## there). Other repos carry byte-identical copies - change it there first.
|
||||
##
|
||||
## Every RUN that rewrites must `restore` before it ends: it puts rewritten files
|
||||
## back, deletes files this script created, deletes any NEW repo/apt file that
|
||||
## still references the cache, then FAILS if any reference is left — so no shipped
|
||||
## layer carries the cache's LAN address (images run on hosts that can't reach it).
|
||||
##
|
||||
## Why: on 2026-10-21 (PHP security release) builds failed on Remi mirrors lagging
|
||||
## hours behind origin, Remi's origin timing out over HTTPS, and an EPEL mirror
|
||||
## with a broken certificate. The cache fetches from the ORIGINS over HTTPS (with
|
||||
## origin/mirror fallbacks and stale-on-error), keeps metadata ~3 minutes and
|
||||
## immutable .rpm/.deb files long-term.
|
||||
##
|
||||
## Integrity (security review 2026-10-21):
|
||||
## * PKG_CACHE must be https://. The cache's certificate is issued by the
|
||||
## name-constrained "pkg-cache internal CA" (keys/pkg-cache/ca.crt, valid only
|
||||
## for 192.168.1.40 / pkg-cache.*). That CA is trusted ONLY by the rewritten
|
||||
## repos (dnf sslcacert= bundle, apt per-host CaInfo) and only during the RUN —
|
||||
## never added to the system trust store.
|
||||
## * gpgcheck=1 is forced on every rewritten repo; repo_gpgcheck=1 on Remi and
|
||||
## AlmaLinux, which sign repomd.xml (measured: repomd.xml.asc 200). EPEL
|
||||
## publishes no repomd signature (404), so its metadata relies on HTTPS end
|
||||
## to end (client->cache->dl.fedoraproject.org/mirrors, all verified).
|
||||
## * release-install downloads epel-release / remi-release (from the cache, or
|
||||
## from the HTTPS origin without PKG_CACHE), checks the package name, imports
|
||||
## the signing keys committed in keys/pkg-cache/ and requires
|
||||
## `rpm -K` "signatures OK" before installing with localpkg_gpgcheck=1.
|
||||
## * The php.net version gates (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail
|
||||
## a build that gets old PHP (replay of old-but-signed metadata).
|
||||
##
|
||||
## rpm: each rewritten baseurl keeps the upstream origin as a SECOND url, so if
|
||||
## the cache dies mid-build dnf fails over to the origin by itself.
|
||||
## apt: apt has no per-source failover; CI only sets PKG_CACHE after probing the
|
||||
## cache (with the CA), and a build without PKG_CACHE uses upstream sources.
|
||||
set -euo pipefail
|
||||
|
||||
MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-install NAME...|nodesource MAJOR|go-env|restore}"
|
||||
shift || true
|
||||
STATE=/var/lib/pkg-cache-repos # saved originals + manifest + created files
|
||||
SHARE="${PKG_CACHE_SHARE:-/usr/local/share/pkg-cache}" # ca.crt + RPM-GPG-KEY-*
|
||||
C="${PKG_CACHE:-}"
|
||||
C="${C%/}"
|
||||
|
||||
die() { echo "FATAL(pkg-cache-repos): $*" >&2; exit 1; }
|
||||
|
||||
if [ -n "$C" ]; then
|
||||
case "$C" in
|
||||
https://*) ;;
|
||||
*) die "PKG_CACHE must be an https:// URL (plain http is refused), got '$C'" ;;
|
||||
esac
|
||||
[[ "$C" =~ ^https://[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._/-]*)?$ ]] || die "PKG_CACHE has unexpected characters: '$C'"
|
||||
HOSTPORT="${C#https://}"; HOSTPORT="${HOSTPORT%%/*}"
|
||||
[ -s "$SHARE/ca.crt" ] || die "PKG_CACHE set but $SHARE/ca.crt (pkg-cache CA) is missing"
|
||||
fi
|
||||
|
||||
# System CA bundle (EL vs Debian/Ubuntu path); the pkg-cache CA is appended to a
|
||||
# copy in the state dir, never to the system store.
|
||||
sys_bundle() {
|
||||
local b
|
||||
for b in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do
|
||||
[ -s "$b" ] && { echo "$b"; return 0; }
|
||||
done
|
||||
die "no system CA bundle found"
|
||||
}
|
||||
make_bundle() { cat "$(sys_bundle)" "$SHARE/ca.crt" > "$STATE/ca-bundle.pem"; }
|
||||
|
||||
# Files that may hold repo configuration; restore re-checks all of them.
|
||||
repo_files() {
|
||||
find /etc/yum.repos.d /etc/dnf /etc/apt -type f 2>/dev/null | sort || true
|
||||
}
|
||||
|
||||
start_session() {
|
||||
[ -d "$STATE" ] && return 0
|
||||
mkdir -p "$STATE/orig"
|
||||
repo_files > "$STATE/manifest" # what existed before this RUN touched anything
|
||||
: > "$STATE/created"
|
||||
printf '%s\n' "$HOSTPORT" > "$STATE/host"
|
||||
}
|
||||
save() { # keep the first-seen (pre-rewrite) copy
|
||||
[ -e "$STATE/orig$1" ] && return 0
|
||||
mkdir -p "$STATE/orig$(dirname "$1")"
|
||||
cp -p "$1" "$STATE/orig$1"
|
||||
}
|
||||
created() { printf '%s\n' "$1" >> "$STATE/created"; }
|
||||
|
||||
if [ "$MODE" = restore ]; then
|
||||
[ -d "$STATE" ] || exit 0
|
||||
host=$(cat "$STATE/host")
|
||||
# mapfile from a process substitution, not `cmd | while read`: no pipeline,
|
||||
# so nothing for pipefail to misreport (scripts/tests/lsphp-info-probe.test.sh).
|
||||
mapfile -t saved < <(cd "$STATE/orig" && find . -type f)
|
||||
for rel in "${saved[@]}"; do
|
||||
cat "$STATE/orig/$rel" > "/${rel#./}"
|
||||
done
|
||||
while read -r f; do [ -n "$f" ] && rm -f -- "$f"; done < "$STATE/created"
|
||||
# A repo/apt file that appeared during this RUN and still names the cache
|
||||
# (e.g. written by a package from the rewritten repos) is removed too.
|
||||
mapfile -t current < <(repo_files)
|
||||
for f in "${current[@]}"; do
|
||||
if ! grep -qxF -- "$f" "$STATE/manifest" && grep -qF -- "$host" "$f"; then
|
||||
echo "pkg-cache-repos: removing new file that references the cache: $f"
|
||||
rm -f -- "$f"
|
||||
fi
|
||||
done
|
||||
left=$(repo_files | xargs -r grep -lF -- "$host" 2>/dev/null || true)
|
||||
rm -rf "$STATE"
|
||||
[ -z "$left" ] || die "cache address $host still present after restore in: $left"
|
||||
echo "pkg-cache-repos: original repository files restored (no reference to $host left)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = release-install ]; then
|
||||
[ $# -gt 0 ] || die "release-install needs: epel and/or remi"
|
||||
tmp=$(mktemp -d)
|
||||
curl_opts=(-fsS --proto '=https' --retry 5 --retry-all-errors --max-time 120)
|
||||
if [ -n "$C" ]; then
|
||||
start_session
|
||||
make_bundle
|
||||
curl_opts+=(--cacert "$STATE/ca-bundle.pem")
|
||||
fi
|
||||
files=()
|
||||
el=$(rpm -E '%{?rhel}' 2>/dev/null || true)
|
||||
case "$el" in 9|10) ;; *) die "release-install supports EL9/EL10 only (rpm -E %rhel = '$el')" ;; esac
|
||||
for name in "$@"; do
|
||||
case "$name" in
|
||||
epel) pkg=epel-release key=RPM-GPG-KEY-EPEL-$el
|
||||
path=/epel/epel-release-latest-$el.noarch.rpm
|
||||
origin=https://dl.fedoraproject.org/pub/epel/epel-release-latest-$el.noarch.rpm ;;
|
||||
remi) pkg=remi-release key=RPM-GPG-KEY-remi2024
|
||||
path=/remi/enterprise/remi-release-$el.rpm
|
||||
origin=https://rpms.remirepo.net/enterprise/remi-release-$el.rpm ;;
|
||||
*) die "unknown release package '$name'" ;;
|
||||
esac
|
||||
url="$origin"; [ -n "$C" ] && url="$C$path"
|
||||
f="$tmp/$pkg.rpm"
|
||||
curl "${curl_opts[@]}" -o "$f" "$url" || die "download failed: $url"
|
||||
rpm --import "$SHARE/$key" || die "cannot import $SHARE/$key"
|
||||
got=$(rpm -qp --nosignature --qf '%{NAME}' "$f" 2>/dev/null || true)
|
||||
[ "$got" = "$pkg" ] || die "$url is '$got', expected $pkg"
|
||||
chk=$(rpm -K "$f" 2>&1 || true)
|
||||
[[ "$chk" == *": digests signatures OK" ]] || die "signature check failed for $url: $chk"
|
||||
echo "pkg-cache-repos: $pkg verified ($chk) from $url"
|
||||
files+=("$f")
|
||||
done
|
||||
dnf install -y --setopt=localpkg_gpgcheck=1 "${files[@]}"
|
||||
rm -rf "$tmp"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -z "$C" ] && [ "$MODE" != nodesource ] && [ "$MODE" != go-env ]; then
|
||||
echo "pkg-cache-repos($MODE): PKG_CACHE not set - using upstream repositories"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rewrite_rpm() {
|
||||
local f="$1"
|
||||
grep -q -E '^#? ?baseurl=https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net|rpm\.nodesource\.com)/' "$f" || return 0
|
||||
save "$f"
|
||||
# Per [section]: a (commented or live) baseurl on a known origin becomes
|
||||
# "baseurl=<cache>/<prefix>/<path> <origin>/<path>"; that section's
|
||||
# mirrorlist/metalink is commented out (dnf prefers them over baseurl) and it
|
||||
# gets sslcacert=<system bundle + pkg-cache CA>, gpgcheck=1 and, for repos
|
||||
# that sign repomd.xml, repo_gpgcheck=1. Other sections are untouched.
|
||||
awk -v C="$C" -v BUNDLE="$STATE/ca-bundle.pem" '
|
||||
function flush( i) {
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (hit && buf[i] ~ /^(mirrorlist|metalink)=/) buf[i] = "#" buf[i]
|
||||
if (hit && buf[i] ~ /^(sslcacert|gpgcheck|repo_gpgcheck)[ \t]*=/) continue
|
||||
print buf[i]
|
||||
}
|
||||
if (hit) {
|
||||
print "sslcacert=" BUNDLE
|
||||
print "gpgcheck=1"
|
||||
if (signed) print "repo_gpgcheck=1"
|
||||
}
|
||||
n = 0; hit = 0; signed = 0
|
||||
}
|
||||
function rw(line, re, prefix, origin, sig, p) {
|
||||
if (match(line, re)) {
|
||||
p = substr(line, RSTART + RLENGTH); sub(/[ \t].*$/, "", p)
|
||||
hit = 1; if (sig) signed = 1
|
||||
return "baseurl=" C "/" prefix "/" p " " origin "/" p
|
||||
}
|
||||
return line
|
||||
}
|
||||
/^\[/ { flush() }
|
||||
{
|
||||
l = $0
|
||||
if (l ~ /^#? ?baseurl=/) {
|
||||
l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux", 1)
|
||||
l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel", 0)
|
||||
l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net", 1)
|
||||
# NodeSource signs its RPMs but not repomd.xml (404): gpgcheck only.
|
||||
l = rw(l, "^#? ?baseurl=https?://rpm\\.nodesource\\.com/", "nodesource", "https://rpm.nodesource.com", 0)
|
||||
}
|
||||
buf[++n] = l
|
||||
}
|
||||
END { flush() }
|
||||
' "$f" > "$f.pkgcache" && cat "$f.pkgcache" > "$f" && rm -f "$f.pkgcache"
|
||||
}
|
||||
|
||||
rewrite_apt() {
|
||||
local f="$1"
|
||||
grep -q -E 'https?://([a-z]{2}\.)?(archive|security)\.ubuntu\.com/|https?://rpms\.litespeedtech\.com/|https?://(deb|security)\.debian\.org/' "$f" || return 0
|
||||
save "$f"
|
||||
# Debian: debian-security first, and "debian" only as a whole path segment,
|
||||
# so .../debian-security is never rewritten as /debian/.
|
||||
sed -i -E \
|
||||
-e "s%https?://(archive|[a-z]{2}\.archive)\.ubuntu\.com/ubuntu/?%$C/ubuntu/%g" \
|
||||
-e "s%https?://security\.ubuntu\.com/ubuntu/?%$C/ubuntu-security/%g" \
|
||||
-e "s%https?://rpms\.litespeedtech\.com/%$C/litespeed/%g" \
|
||||
-e "s%https?://(deb|security)\.debian\.org/debian-security/?([[:space:]]|\$)%$C/debian-security/\\2%g" \
|
||||
-e "s%https?://security\.debian\.org/?([[:space:]]|\$)%$C/debian-security/\\1%g" \
|
||||
-e "s%https?://deb\.debian\.org/debian/?([[:space:]]|\$)%$C/debian/\\1%g" \
|
||||
"$f"
|
||||
}
|
||||
|
||||
if [ "$MODE" = go-env ]; then
|
||||
# Prints shell exports for `eval`. Without PKG_CACHE: nothing (Go defaults).
|
||||
# GOPROXY "a|b" falls through on ANY error, so a dead cache costs one
|
||||
# failed request, not the build. GOSUMDB stays the default sum.golang.org,
|
||||
# reached directly: modules are still verified against go.sum + the sumdb.
|
||||
[ -n "$C" ] || { echo "pkg-cache-repos(go-env): PKG_CACHE not set - default GOPROXY" >&2; exit 0; }
|
||||
start_session
|
||||
make_bundle
|
||||
printf "export GOPROXY='%s/goproxy|https://proxy.golang.org|direct' SSL_CERT_FILE='%s'\n" "$C" "$STATE/ca-bundle.pem"
|
||||
echo "pkg-cache-repos(go-env): GOPROXY via $C/goproxy (fallback proxy.golang.org)" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = nodesource ]; then
|
||||
# Replaces `curl https://rpm.nodesource.com/setup_N.x | bash` (unverified
|
||||
# remote script): writes the same nodesource-nodejs repo (origin URL, so the
|
||||
# shipped image keeps a normal repo file) with gpgcheck=1 and the signing key
|
||||
# committed in keys/pkg-cache/; with PKG_CACHE it is then rewritten like any
|
||||
# other repo and `restore` puts the origin version back.
|
||||
major="${1:?usage: pkg-cache-repos.sh nodesource MAJOR}"
|
||||
[[ "$major" =~ ^[0-9]{1,3}$ ]] || die "bad Node major '$major'"
|
||||
key="$SHARE/RPM-GPG-KEY-nodesource"
|
||||
[ -s "$key" ] || die "missing $key"
|
||||
rpm --import "$key" || die "cannot import $key"
|
||||
rm -f /etc/yum.repos.d/nodesource*.repo
|
||||
f=/etc/yum.repos.d/nodesource-nodejs.repo
|
||||
printf '%s\n' "[nodesource-nodejs]" \
|
||||
"name=Node.js Packages for Linux RPM based distros - \$basearch" \
|
||||
"baseurl=https://rpm.nodesource.com/pub_${major}.x/nodistro/nodejs/\$basearch" \
|
||||
"priority=9" "enabled=1" "gpgcheck=1" \
|
||||
"gpgkey=file://$key" "module_hotfixes=1" > "$f"
|
||||
echo "pkg-cache-repos: wrote $f (Node ${major}.x, key $(basename "$key"))"
|
||||
if [ -n "$C" ]; then
|
||||
start_session
|
||||
make_bundle
|
||||
rewrite_rpm "$f"
|
||||
grep -h '^baseurl=' "$f" | sed 's/^/ /'
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$MODE" in
|
||||
rpm)
|
||||
start_session
|
||||
make_bundle
|
||||
for f in /etc/yum.repos.d/*.repo; do [ -f "$f" ] && rewrite_rpm "$f"; done
|
||||
echo "pkg-cache-repos(rpm): repos now via $C:"
|
||||
grep -h '^baseurl=' /etc/yum.repos.d/*.repo | grep -F "$C/" | sort -u | sed 's/^/ /'
|
||||
;;
|
||||
apt)
|
||||
start_session
|
||||
conf=/etc/apt/apt.conf.d/99pkg-cache
|
||||
# Trust the pkg-cache CA for the cache host ONLY (apt per-host option).
|
||||
printf 'Acquire::https::%s::CaInfo "%s";\n' "${HOSTPORT%%:*}" "$SHARE/ca.crt" > "$conf"
|
||||
created "$conf"
|
||||
for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
|
||||
[ -f "$f" ] && rewrite_apt "$f"
|
||||
done
|
||||
echo "pkg-cache-repos(apt): sources now via $C:"
|
||||
# (|| true: Debian 13 images have no /etc/apt/sources.list; grep's exit 2
|
||||
# for the missing file must not fail the build under pipefail)
|
||||
{ grep -h -E '^(deb|URIs:)' /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null || true; } | grep -F "$C/" | sort -u | sed 's/^/ /' || true
|
||||
;;
|
||||
*) die "unknown mode '$MODE'" ;;
|
||||
esac
|
||||
@@ -0,0 +1,43 @@
|
||||
# Container logging policy (what reaches `docker logs`)
|
||||
|
||||
`docker logs haproxy-manager` (and the docker logging driver: json-file today,
|
||||
journald -> Graylog under WHP PR #260) must carry only low-volume, high-value
|
||||
lines. Per-request logs stay out of it.
|
||||
|
||||
| Source | Destination | Volume |
|
||||
|---|---|---|
|
||||
| HAProxy edge access log (1.5M lines/day) | `/var/log/haproxy/access.log` via `haproxy-log-receiver.py` (20 MiB cap, gzip generations) | NOT in `docker logs` |
|
||||
| HAProxy master stderr (ALERT, crash dumps) | `haproxy-stderr.log` -> `haproxy-stderr-relay.py` -> stdout (benign, rate limited) / stderr (crash evidence) | rare |
|
||||
| Supervisor / relay / init lines | stdout / stderr | rare |
|
||||
| gunicorn :8080 default-page / blocked-IP / suspended-site | NO access log | none |
|
||||
| gunicorn :8000 management API access log | `/var/log/haproxy/api-access.log` (haproxy-logs volume, mode 0640), rotated by `scripts/gunicorn-access-rotate.py` (copy-truncate, 20 MiB x 3) | NOT in `docker logs` |
|
||||
| gunicorn error log (tracebacks, worker timeouts) | stderr (`--error-logfile -`) | rare |
|
||||
|
||||
## Why
|
||||
|
||||
Before this change both gunicorns ran with `--access-logfile -`, so each :8080
|
||||
hit and each :8000 API call was one docker stdout line. With the journald
|
||||
driver that is one journal entry per request: journald's default rate limit
|
||||
(10000 messages / 30 s per sending service) can then suppress the very
|
||||
crash-dump lines the relay emits, and Graylog would be flooded.
|
||||
|
||||
## Knobs
|
||||
|
||||
`HAPROXY_MGR_API_ACCESS_LOG` (path), `HAPROXY_API_ACCESS_LOG_MAX_BYTES`
|
||||
(20971520), `HAPROXY_API_ACCESS_LOG_BACKUPS` (3), `HAPROXY_API_ACCESS_LOG_POLL`
|
||||
(30 s; the live file can exceed the cap by at most one interval of API traffic).
|
||||
|
||||
## Not done
|
||||
|
||||
Failed-auth (401/403) lines are NOT mirrored to stdout; they are in the file
|
||||
(grep the status column). Add a rate-limited mirror later if wanted.
|
||||
|
||||
## Tests
|
||||
|
||||
`python3 scripts/test-gunicorn-access-logging.py` (runs in the image build):
|
||||
real gunicorn, a burst of default-page requests produces zero stdout lines,
|
||||
API requests land in the file not stdout, handler exceptions still reach
|
||||
stderr, and rotation is bounded.
|
||||
|
||||
Deploying this needs a NEW rc image built from `feat/haproxy-upgrade`
|
||||
(VERSION stays 2026.11.5 until then).
|
||||
@@ -0,0 +1,34 @@
|
||||
<!DOCTYPE html>
|
||||
<!--
|
||||
Default splash: served by HAProxy itself (`http-request return ... file`,
|
||||
hap_listener.tpl) for any request whose Host is not routed to a site --
|
||||
the server's own hostname, a bare IP, an unconfigured name. Static on
|
||||
purpose: no version, no hostname echo, no customer data, no links (in
|
||||
particular none into the control panel). Loaded with `file`, not
|
||||
`lf-file`, so `%` is literal here and nothing from the request is
|
||||
expanded into it. Must stay well under tune.bufsize (16 KB) or HAProxy
|
||||
rejects the config.
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="robots" content="noindex, nofollow">
|
||||
<title>Web Hosting Platform</title>
|
||||
<style>
|
||||
:root{color-scheme:light dark;--bg:#f4f5f7;--card:#fff;--fg:#1d2330;--muted:#5b6475;--line:#dfe3ea}
|
||||
@media (prefers-color-scheme:dark){:root{--bg:#0f1217;--card:#181c23;--fg:#e6e9ef;--muted:#9aa3b2;--line:#2a303a}}
|
||||
html,body{margin:0;height:100%}
|
||||
body{display:flex;align-items:center;justify-content:center;background:var(--bg);color:var(--fg);font:16px/1.5 system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;padding:16px;box-sizing:border-box}
|
||||
main{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:32px 28px;max-width:480px;width:100%;text-align:center}
|
||||
h1{font-size:1.25rem;margin:0 0 .5rem}
|
||||
p{margin:0;color:var(--muted)}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Web Hosting Platform</h1>
|
||||
<p>This server is operated by a web hosting platform. There is no website configured at this address.</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,2 @@
|
||||
User-agent: *
|
||||
Disallow: /
|
||||
+154
-26
@@ -135,6 +135,16 @@ BLOCKED_IPS_MAP_BACKUP_PATH = '/etc/haproxy/blocked_ips.map.backup'
|
||||
# while leaving a broken coraza-spoe.cfg behind still fails validation.
|
||||
CORAZA_SPOE_CONFIG_PATH = '/etc/haproxy/coraza-spoe.cfg'
|
||||
CORAZA_SPOE_BACKUP_PATH = '/etc/haproxy/coraza-spoe.cfg.backup'
|
||||
# Hosts the frontend routes to a site (default splash, hap_listener.tpl). The
|
||||
# paths are hardcoded in the template's `-f` ACLs, like blocked_ips.map, and
|
||||
# written by generate_config() from the same rows as the routing ACLs.
|
||||
# HAProxy's access-log target: the receiver's socket, CHROOT-RELATIVE
|
||||
# (hap_header.tpl `chroot /var/lib/haproxy`), i.e. /var/lib/haproxy/log.sock.
|
||||
DEFAULT_LOG_TARGET = '/log.sock'
|
||||
ROUTED_HOSTS_PATH = '/etc/haproxy/routed_hosts.list'
|
||||
ROUTED_HOSTS_BACKUP_PATH = '/etc/haproxy/routed_hosts.list.backup'
|
||||
ROUTED_WILDCARDS_PATH = '/etc/haproxy/routed_wildcards.list'
|
||||
ROUTED_WILDCARDS_BACKUP_PATH = '/etc/haproxy/routed_wildcards.list.backup'
|
||||
HAPROXY_SOCKET_PATH = '/var/run/haproxy.sock'
|
||||
# HAProxy loads this path as a DIRECTORY (`bind ... ssl crt /etc/haproxy/certs`),
|
||||
# which means it tries to load EVERY file it finds in here. Nothing but final,
|
||||
@@ -756,6 +766,92 @@ def is_process_running(process_name):
|
||||
return True
|
||||
return False
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Launching the HAProxy master (2026-11 forensics change)
|
||||
#
|
||||
# haproxy.cfg carries `daemon`, and in daemon mode HAProxy re-points the
|
||||
# master's stdin/stdout/stderr at /dev/null after it has parsed the config
|
||||
# (MEASURED on 3.0.11: /proc/<master>/fd/2 -> /dev/null). That discarded every
|
||||
# [ALERT], the "exited with code 134" master line and the whole ha_panic()/
|
||||
# watchdog thread dump of the 2026-11 aborts: only the core file survived.
|
||||
#
|
||||
# `-db` overrides `daemon` (MEASURED: foreground master, still master-worker,
|
||||
# reload via the CLI keeps the same master pid), so the master keeps a real
|
||||
# stderr. We append it to HAPROXY_STDERR_LOG (haproxy-logs volume, survives a
|
||||
# container recreate) and scripts/haproxy-stderr-relay.py forwards it to the
|
||||
# container's stderr (-> docker logging driver -> host journald/Graylog).
|
||||
#
|
||||
# The master is started as `sh -c '... &'` so it is re-parented to PID 1 exactly
|
||||
# as the old daemon fork did (PID 1 reaps it; a Popen child would sit as a
|
||||
# zombie in the API worker and is_process_running('haproxy') would keep
|
||||
# reporting it alive).
|
||||
# ---------------------------------------------------------------------------
|
||||
HAPROXY_STDERR_LOG = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
|
||||
try:
|
||||
HAPROXY_START_WAIT = float(os.environ.get('HAPROXY_START_WAIT', '10'))
|
||||
except ValueError:
|
||||
HAPROXY_START_WAIT = 10.0
|
||||
|
||||
|
||||
def _stderr_log_path():
|
||||
"""HAPROXY_STDERR_LOG if its directory is writable, else a /tmp fallback
|
||||
(a missing volume must never stop HAProxy from starting)."""
|
||||
path = HAPROXY_STDERR_LOG
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
# 0640: HAProxy echoes tenant hostnames/backend names here (not world-readable)
|
||||
os.close(os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o640))
|
||||
return path
|
||||
except OSError:
|
||||
return '/tmp/haproxy-stderr.log'
|
||||
|
||||
|
||||
def spawn_haproxy_master():
|
||||
"""Start `haproxy -W -db` detached, stderr appended to the stderr log.
|
||||
|
||||
Returns a subprocess.CompletedProcess (returncode 0, .stderr = the new log
|
||||
text) once HAProxy has loaded its configuration; raises
|
||||
subprocess.CalledProcessError (.stderr = what HAProxy printed) if it exits
|
||||
or never comes up within HAPROXY_START_WAIT seconds. Same contract the old
|
||||
blocking `subprocess.run([... '-W' ...], check=True)` call sites relied on.
|
||||
"""
|
||||
cmd = ['haproxy', '-W', '-db', '-S', '/tmp/haproxy-cli,level,admin',
|
||||
'-f', HAPROXY_CONFIG_PATH]
|
||||
log_path = _stderr_log_path()
|
||||
try:
|
||||
offset = os.path.getsize(log_path)
|
||||
except OSError:
|
||||
offset = 0
|
||||
subprocess.run(
|
||||
['sh', '-c', 'log=$1; shift; (umask 027; : >>"$log"); chmod 0640 "$log" 2>/dev/null; '
|
||||
'"$@" </dev/null >>"$log" 2>&1 &', 'sh', log_path] + cmd,
|
||||
check=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
|
||||
def new_text():
|
||||
try:
|
||||
with open(log_path, 'rb') as fh:
|
||||
fh.seek(offset)
|
||||
return fh.read().decode('utf-8', 'replace')
|
||||
except OSError:
|
||||
return ''
|
||||
|
||||
deadline = time.monotonic() + HAPROXY_START_WAIT
|
||||
started = time.monotonic()
|
||||
while time.monotonic() < deadline:
|
||||
text = new_text()
|
||||
if 'Loading success.' in text:
|
||||
return subprocess.CompletedProcess(cmd, 0, '', text)
|
||||
if time.monotonic() - started > 1.0 and not is_process_running('haproxy'):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
text = new_text()
|
||||
if is_process_running('haproxy') and '[ALERT]' not in text:
|
||||
# Slow to log (big config) but alive: treat as started.
|
||||
return subprocess.CompletedProcess(cmd, 0, '', text)
|
||||
raise subprocess.CalledProcessError(1, cmd, output='', stderr=text[-4000:])
|
||||
|
||||
|
||||
# Initialize template engine
|
||||
template_loader = jinja2.FileSystemLoader(TEMPLATE_DIR)
|
||||
template_env = jinja2.Environment(loader=template_loader)
|
||||
@@ -861,12 +957,7 @@ def reload_haproxy():
|
||||
return jsonify({'status': 'success'}), 200
|
||||
else:
|
||||
# Start HAProxy if it's not running
|
||||
result = subprocess.run(
|
||||
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
result = spawn_haproxy_master()
|
||||
if result.returncode == 0:
|
||||
print("HAProxy started successfully")
|
||||
log_operation('start_haproxy', True)
|
||||
@@ -3138,15 +3229,22 @@ def generate_config():
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create {suspended_list_path}: {e}")
|
||||
|
||||
# Access-log destination for the `log` line in the global section.
|
||||
# Default 172.18.0.1:514 is the docker bridge gateway for WHP's
|
||||
# `client-net`, i.e. the host, where rsyslog's imudp listener is bound
|
||||
# by setup-haproxy-logrotate.sh. Overridable so this image stays usable
|
||||
# on standalone/home deployments with a different bridge subnet or a
|
||||
# remote log collector -- set HAPROXY_SYSLOG_TARGET to `<ip>:<port>`.
|
||||
# UDP, so an absent listener drops log lines and never affects request
|
||||
# handling.
|
||||
syslog_target = os.environ.get('HAPROXY_SYSLOG_TARGET', '172.18.0.1:514').strip()
|
||||
# Access-log destination for the `log` line in the global section:
|
||||
# the in-container receiver's UNIX socket (scripts/haproxy-log-receiver.py),
|
||||
# given relative to haproxy's chroot (/var/lib/haproxy) -- see
|
||||
# hap_header.tpl. HAPROXY_SYSLOG_TARGET overrides it (`<ip>:<port>` for
|
||||
# a remote collector). The old default, 172.18.0.1:514 (host rsyslog on
|
||||
# the docker bridge), had no listener on the fleet.
|
||||
syslog_target = (os.environ.get('HAPROXY_SYSLOG_TARGET', '').strip()
|
||||
or DEFAULT_LOG_TARGET)
|
||||
|
||||
# Default splash for unrouted hosts (server hostname, bare IP, unknown
|
||||
# names) answered by HAProxy before the WAF -- see hap_listener.tpl.
|
||||
# ON by default: it depends on no sidecar and only changes what an
|
||||
# unrouted request sees. HAPROXY_DEFAULT_SPLASH=off restores the old
|
||||
# default-backend page (WAF-inspected).
|
||||
splash_raw = os.environ.get('HAPROXY_DEFAULT_SPLASH', '').strip().lower()
|
||||
default_splash = splash_raw not in ('0', 'false', 'no', 'off')
|
||||
|
||||
# Add Haproxy Default Headers
|
||||
default_headers = template_env.get_template('hap_header.tpl').render(
|
||||
@@ -3168,6 +3266,7 @@ def generate_config():
|
||||
crt_path = SSL_CERTS_DIR,
|
||||
coraza_spoe_backend = coraza_spoe_backend,
|
||||
suspension_enabled = suspension_enabled,
|
||||
default_splash = default_splash,
|
||||
)
|
||||
config_parts.append(listener_block)
|
||||
|
||||
@@ -3176,7 +3275,10 @@ def generate_config():
|
||||
config_parts.append(letsencrypt_acl)
|
||||
config_acls = []
|
||||
config_backends = []
|
||||
|
||||
# Every name a routing ACL is rendered for (default splash lists).
|
||||
routed_hosts = []
|
||||
routed_wildcards = []
|
||||
|
||||
# Add default backend rule (will be used when no domain matches)
|
||||
default_rule = " # Default backend for unmatched domains\n default_backend default-backend\n"
|
||||
config_parts.append(default_rule)
|
||||
@@ -3231,6 +3333,7 @@ def generate_config():
|
||||
name=domain['backend_name']
|
||||
)
|
||||
config_acls.append(domain_acl)
|
||||
routed_hosts.append(domain['domain'])
|
||||
logger.info(f"Added ACL for domain: {domain['domain']}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating domain ACL for {domain['domain']}: {e}")
|
||||
@@ -3257,6 +3360,7 @@ def generate_config():
|
||||
base_domain=base_domain
|
||||
)
|
||||
config_acls.append(domain_acl)
|
||||
routed_wildcards.append('.' + base_domain)
|
||||
logger.info(f"Added wildcard ACL for domain: {domain['domain']}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating wildcard ACL for {domain['domain']}: {e}")
|
||||
@@ -3318,6 +3422,14 @@ backend default-backend
|
||||
f"{CORAZA_SPOE_CONFIG_PATH} "
|
||||
f"(SPOA target: {coraza_spoe_backend})")
|
||||
|
||||
# Routed-host lists for the default splash. Written on every render
|
||||
# (also with the splash off, so turning it on later never meets a
|
||||
# missing `-f` file, which is FATAL). Same render, same rows as the
|
||||
# routing ACLs above, and in the rollback set, so the loaded
|
||||
# haproxy.cfg and these lists always come from one generation.
|
||||
write_config_atomically(ROUTED_HOSTS_PATH, render_routed_list(routed_hosts))
|
||||
write_config_atomically(ROUTED_WILDCARDS_PATH, render_routed_list(routed_wildcards))
|
||||
|
||||
config_content = '\n'.join(config_parts)
|
||||
logger.debug("Generated HAProxy configuration")
|
||||
|
||||
@@ -3401,6 +3513,28 @@ def _config_set_matches_backup():
|
||||
return True
|
||||
|
||||
|
||||
def render_routed_list(names):
|
||||
"""Pattern-file body for the default-splash routed-host ACLs.
|
||||
|
||||
One lowercased name per line, sorted and de-duplicated so an unchanged
|
||||
fleet renders byte-identical files (create_backup() skips re-validation
|
||||
when the live set equals the backup). A name containing whitespace cannot
|
||||
be one pattern line (a newline would split it in two) and could not be a
|
||||
working `hdr(host) -i` routing ACL either; it is dropped and logged at
|
||||
ERROR, because a missing entry means the splash answers that host.
|
||||
"""
|
||||
out = set()
|
||||
for name in names:
|
||||
clean = (name or '').strip().lower()
|
||||
if not clean or any(c.isspace() for c in clean):
|
||||
logger.error(f"default splash: cannot list routed host {name!r}; "
|
||||
"it will be answered by the splash page")
|
||||
continue
|
||||
out.add(clean)
|
||||
body = ''.join(f'{n}\n' for n in sorted(out))
|
||||
return '# Generated by haproxy_manager.generate_config() - do not edit\n' + body
|
||||
|
||||
|
||||
def _config_backup_pairs():
|
||||
"""(live, backup) pairs forming one restorable config set.
|
||||
|
||||
@@ -3411,6 +3545,8 @@ def _config_backup_pairs():
|
||||
(HAPROXY_CONFIG_PATH, HAPROXY_BACKUP_PATH),
|
||||
(BLOCKED_IPS_MAP_PATH, BLOCKED_IPS_MAP_BACKUP_PATH),
|
||||
(CORAZA_SPOE_CONFIG_PATH, CORAZA_SPOE_BACKUP_PATH),
|
||||
(ROUTED_HOSTS_PATH, ROUTED_HOSTS_BACKUP_PATH),
|
||||
(ROUTED_WILDCARDS_PATH, ROUTED_WILDCARDS_BACKUP_PATH),
|
||||
)
|
||||
|
||||
|
||||
@@ -3723,10 +3859,7 @@ def reload_haproxy_safely(backup_status=None):
|
||||
else:
|
||||
# HAProxy not running, start it
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
|
||||
check=True, capture_output=True, text=True
|
||||
)
|
||||
result = spawn_haproxy_master()
|
||||
logger.info("HAProxy started successfully")
|
||||
# Now - and only now - is this config known good.
|
||||
promote_current_config_to_backup()
|
||||
@@ -4059,12 +4192,7 @@ def start_haproxy():
|
||||
return
|
||||
|
||||
# Configuration is valid, start HAProxy
|
||||
result = subprocess.run(
|
||||
['haproxy', '-W', '-S', '/tmp/haproxy-cli,level,admin', '-f', HAPROXY_CONFIG_PATH],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
result = spawn_haproxy_master()
|
||||
if result.returncode == 0:
|
||||
logger.info("HAProxy started successfully")
|
||||
log_operation('start_haproxy', True, 'HAProxy started successfully')
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,32 @@
|
||||
Trust material for the OPT-IN internal package cache (scripts/pkg-cache-repos.sh,
|
||||
docs/pkg-cache.md). Copied into images at /usr/local/share/pkg-cache/; nothing
|
||||
here is added to the system trust store.
|
||||
|
||||
ca.crt "pkg-cache internal CA (CT130)", EC P-256, valid to 2036-10-18.
|
||||
NAME-CONSTRAINED to IP 192.168.1.40 and DNS
|
||||
pkg-cache.internal.cloud-hosting.io / pkg-cache.geekhome.local.
|
||||
SHA-256 fingerprint BB:5B:9A:08:05:ED:2F:58:83:92:D0:99:9D:11:0A:30:14:88:68:F0:F0:88:65:5E:94:0B:75:EB:C2:A2:BD:F3
|
||||
(private key lives only on CT130: /etc/pkgcache-ca/ca.key)
|
||||
RPM-GPG-KEY-EPEL-10 Fedora (epel10) 7D8D 15CB FC4E 6268 8591 FB26 33D9 8517 E37E D158
|
||||
fetched 2026-10-21 over HTTPS from dl.fedoraproject.org and
|
||||
mirror.math.princeton.edu (identical); fingerprint matches
|
||||
https://fedoraproject.org/security/
|
||||
RPM-GPG-KEY-remi2024 Remi's RPM repository 2024 CF1D F005 7CE8 5DFF 5B2F 2A37 C2FD 3B2C 2A09 48E4
|
||||
fetched 2026-10-21 over HTTPS from rpms.remirepo.net; byte-identical
|
||||
to the key shipped in remi-release-10.2-1.el10.remi. Not found on
|
||||
keys.openpgp.org / keyserver.ubuntu.com - confirm out of band.
|
||||
Remi rotates keys (remi2025, ...): when remi-release starts being
|
||||
signed by a new key, `release-install remi` fails loudly - add the
|
||||
new key here (and to pkg-cache-repos.sh).
|
||||
RPM-GPG-KEY-EPEL-9 Fedora (epel9) FF8A D134 4597 106E CE81 3B91 8A38 72BF 3228 467C
|
||||
fetched 2026-10-22 over HTTPS from dl.fedoraproject.org and
|
||||
mirror.math.princeton.edu (identical); used by `release-install epel`
|
||||
on EL9 images (cloud-memcached-container, cloud-haproxy-container).
|
||||
RPM-GPG-KEY-nodesource Nodesource Operations <operations@nodesource.com>
|
||||
242B 8138 31AF 0956 2B6C 46F7 6B88 DA4E 3AF2 8A14 (rsa4096, 2023-12-22)
|
||||
fetched 2026-10-22 over HTTPS from
|
||||
https://rpm.nodesource.com/gpgkey/ns-operations-public.key (the key
|
||||
NodeSource's own setup_N.x scripts reference); verified by dnf
|
||||
gpgcheck against the nodejs RPMs. Not on keys.openpgp.org /
|
||||
keyserver.ubuntu.com - confirm out of band. Used by
|
||||
`pkg-cache-repos.sh nodesource MAJOR` (cloud-node-container).
|
||||
@@ -0,0 +1,29 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA
|
||||
Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK
|
||||
Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L
|
||||
URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf
|
||||
mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2
|
||||
XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI
|
||||
uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60
|
||||
GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix
|
||||
fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z
|
||||
BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77
|
||||
HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB
|
||||
tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB
|
||||
CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC
|
||||
BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p
|
||||
l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp
|
||||
yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg
|
||||
Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u
|
||||
scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs
|
||||
7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK
|
||||
F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg
|
||||
sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB
|
||||
0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY
|
||||
1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt
|
||||
TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k
|
||||
p9Dfd2pKuxK9Mys=
|
||||
=mhQZ
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,29 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGE3mOsBEACsU+XwJWDJVkItBaugXhXIIkb9oe+7aadELuVo0kBmc3HXt/Yp
|
||||
CJW9hHEiGZ6z2jwgPqyJjZhCvcAWvgzKcvqE+9i0NItV1rzfxrBe2BtUtZmVcuE6
|
||||
2b+SPfxQ2Hr8llaawRjt8BCFX/ZzM4/1Qk+EzlfTcEcpkMf6wdO7kD6ulBk/tbsW
|
||||
DHX2lNcxszTf+XP9HXHWJlA2xBfP+Dk4gl4DnO2Y1xR0OSywE/QtvEbN5cY94ieu
|
||||
n7CBy29AleMhmbnx9pw3NyxcFIAsEZHJoU4ZW9ulAJ/ogttSyAWeacW7eJGW31/Z
|
||||
39cS+I4KXJgeGRI20RmpqfH0tuT+X5Da59YpjYxkbhSK3HYBVnNPhoJFUc2j5iKy
|
||||
XLgkapu1xRnEJhw05kr4LCbud0NTvfecqSqa+59kuVc+zWmfTnGTYc0PXZ6Oa3rK
|
||||
44UOmE6eAT5zd/ToleDO0VesN+EO7CXfRsm7HWGpABF5wNK3vIEF2uRr2VJMvgqS
|
||||
9eNwhJyOzoca4xFSwCkc6dACGGkV+CqhufdFBhmcAsUotSxe3zmrBjqA0B/nxIvH
|
||||
DVgOAMnVCe+Lmv8T0mFgqZSJdIUdKjnOLu/GRFhjDKIak4jeMBMTYpVnU+HhMHLq
|
||||
uDiZkNEvEEGhBQmZuI8J55F/a6UURnxUwT3piyi3Pmr2IFD7ahBxPzOBCQARAQAB
|
||||
tCdGZWRvcmEgKGVwZWw5KSA8ZXBlbEBmZWRvcmFwcm9qZWN0Lm9yZz6JAk4EEwEI
|
||||
ADgWIQT/itE0RZcQbs6BO5GKOHK/MihGfAUCYTeY6wIbDwULCQgHAgYVCgkICwIE
|
||||
FgIDAQIeAQIXgAAKCRCKOHK/MihGfFX/EACBPWv20+ttYu1A5WvtHJPzwbj0U4yF
|
||||
3zTQpBglQ2UfkRpYdipTlT3Ih6j5h2VmgRPtINCc/ZE28adrWpBoeFIS2YAKOCLC
|
||||
nZYtHl2nCoLq1U7FSttUGsZ/t8uGCBgnugTfnIYcmlP1jKKA6RJAclK89evDQX5n
|
||||
R9ZD+Cq3CBMlttvSTCht0qQVlwycedH8iWyYgP/mF0W35BIn7NuuZwWhgR00n/VG
|
||||
4nbKPOzTWbsP45awcmivdrS74P6mL84WfkghipdmcoyVb1B8ZP4Y/Ke0RXOnLhNe
|
||||
CfrXXvuW+Pvg2RTfwRDtehGQPAgXbmLmz2ZkV69RGIr54HJv84NDbqZovRTMr7gL
|
||||
9k3ciCzXCiYQgM8yAyGHV0KEhFSQ1HV7gMnt9UmxbxBE2pGU7vu3CwjYga5DpwU7
|
||||
w5wu1TmM5KgZtZvuWOTDnqDLf0cKoIbW8FeeCOn24elcj32bnQDuF9DPey1mqcvT
|
||||
/yEo/Ushyz6CVYxN8DGgcy2M9JOsnmjDx02h6qgWGWDuKgb9jZrvRedpAQCeemEd
|
||||
fhEs6ihqVxRFl16HxC4EVijybhAL76SsM2nbtIqW1apBQJQpXWtQwwdvgTVpdEtE
|
||||
r4ArVJYX5LrswnWEQMOelugUG6S3ZjMfcyOa/O0364iY73vyVgaYK+2XtT2usMux
|
||||
VL469Kj5m13T6w==
|
||||
=Mjs/
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,52 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGWFsqMBEAC4YlSuNpHjOiQVTC8StolVx3J8E9SC6G4wmVWbytMD1Zivfanj
|
||||
3LKDqdy7vpbViXA9whpyGQgQYdOlPKaNHWw4GSeGTnP1FbvaK1NJ9MnAzUysh49R
|
||||
cqoktYeIQbqgOgZHw5/gVH5bYObf1KJQ2LY/je9WTlW3QMRe1TaW0F0AZnJMBIOG
|
||||
JllE33QIpxDnOQHnvdUGq+HscUOTrW4Q3AdzwsQxU4GX82d9qFFr8HjAmUjlUKyh
|
||||
PgvMpb8VDvR7NE6et+wWh6BMYILo3VJFP7/gTXGiy3WWs5GW/PNXIXw5/f2w1B+e
|
||||
Dh5tlfGG4q/4P8mbv7C6RPHoF2Lq+W69HLqSq/l42WUL41tHlFTsluCIKpeeMxAu
|
||||
olxx3Z5momHMMyh7rbZtQDqFMTAqxNjT1NinKkWyj7KP5WE9hVosyd3UMR4UYDMm
|
||||
Zb6LI4N7Hht887+CY7grxTJ9cGQhMFfvLyvlf/FWQY4S6G7sEaUV71N9StvXBNG/
|
||||
iNoIMzhwjBmrmvT/UGPkLSi0SIBOxeGw2rwngnYikIJxuTQ4V40lm8o6eMFca7rW
|
||||
BQvAEncV8pvi/3FqrUBZ/U53s7NO24hBDoaQPzFbVCw0GHHf9GD7KoQb5c9ONoCZ
|
||||
Qu2YWo7ajPLzl/h2pJ2Kn29Ywev9MpuuPTGvifiKilcBVLYP3ZJg4ZfRBwARAQAB
|
||||
tDFOb2Rlc291cmNlIE9wZXJhdGlvbnMgPG9wZXJhdGlvbnNAbm9kZXNvdXJjZS5j
|
||||
b20+iQJOBBMBCgA4FiEEJCuBODGvCVYrbEb3a4jaTjryihQFAmWFsqMCGwMFCwkI
|
||||
BwIGFQoJCAsCBBYCAwECHgECF4AACgkQa4jaTjryihTD+hAAtTU155ke7xmnT0ph
|
||||
odbGjm3mOGdkdaDH+xmWtPtufiPncgHMCYh4aIDRloog8rFE+pP3qfwJmUKxthwD
|
||||
drrLAzfUwOrGxPnYH3MWwg66RVVoiiU+Hh42gJTez+P/K0VMQZXH0XTZD96TiwT/
|
||||
S0+WvQPuJ9pda2L4cjV+YP2+m0ERUG9QPj0Shgw+/PWCGWKD6J9DRToiwfwCTOvc
|
||||
DNolF1EcRlfPMTsqI2Bdc+TQywqU1Py1MUDSAPXxdK6nW2oM7eC5l3XXefcQ6JOS
|
||||
AuNdEDIsaf+H76j6/wSGsuIcJjUo9wmKjEV1iJYwT6jLbjaVQxXgEPHBSb+kzanx
|
||||
FWyrYP6TZpeY2gT4bBMsOKd7ay1I3/nIHwe6x8Yzkqzz+7YVhNpMbybH72NTxDWQ
|
||||
UmiboJs9r7j1lkHlBjxU9RLF5laYSeCIXD62KQVoP5ao1LSPKsCXFqcdpUgLYtRv
|
||||
CXxdriiHp4m9eGzbS7RW1TLm6tMOKbI4LzOtMm4DoZzGD2FTfsUtn6ALx9FSgvl/
|
||||
VMlzqZes+rclcukACtutvtB8e0yRhuzPUN56ctLEZfI1XzQFoN8Vdj6HWtiXWRiJ
|
||||
EohuufE2cEebmX90NRsbh4KAFXNEQROlMeMf7ufuSsk7QXLTutxmWSQvWwyS+P5/
|
||||
ZqCapIDlRCJFZX4N9a1IXo0OgCC5Ag0EZYWyowEQAKz7sA6WBKFVcmzcyr6y9TI5
|
||||
RemnzotRUdqXj8tD70xvV1nBEf+BfTw3hsiV019ZboPBae+cRaD4plQbNb0du/+Q
|
||||
UFq4ja055ZEld01RqE49L562pWvA5cwSmjs9+ZMI1ja0HbTb2K4razpzX3be+Zul
|
||||
TbH20X5FHTz+Jyfmw+G3Y9okw3OIzuYo16Lv/Qs8VoBmj01PItzq8dX5wJiIlVxU
|
||||
8wKHKy1kiKGCgyPnvqcXCGen0EoUS9wsWK1QHfryscYGVF9y1TBQvJa9HIpWPrIZ
|
||||
slSiVS4tQhGHfLDVlaDsYFrH9SIUkvW7peSEiPE1TKQjvbMS3CcUZRaMcGfgyDPe
|
||||
gXwnNwZbJ2/cisXbdAJaGT49V1Y+Sc1oKSGSG4FaGUNhS3p3cB7qlX2zS/sR2ael
|
||||
90fXR6nhjvhMAS98i7663pCUv9q0Z/NXNZwdXjbSF4W8gq3PVt7hCnJrngsGSkGu
|
||||
Qz0dc227/sAjh9CxYm8kQqUuQtiPVfKwkcUUT+O7jr2/6eAXZbTYVZbvaLEGMrKA
|
||||
bdWKpZMMOKAI45f+ZAcEmJG4D68N0jzVrLCXMuuLq/yqqyEmxDjSOH0qVutl8jxg
|
||||
vfxPxKYX8nnAzJZJHGHmV/ihjfClSdYePKExHuFVVhvc4BxE1XfPUeCZG4JpB9/B
|
||||
gNCiDPsg4JIw6GO5LS2xABEBAAGJAjYEGAEKACAWIQQkK4E4Ma8JVitsRvdriNpO
|
||||
OvKKFAUCZYWyowIbDAAKCRBriNpOOvKKFOTKD/40WN6ZRfP6hVLAX26DdyN9Bqzg
|
||||
D+0hJilmHL1YFeLt9U/YPhOZJxPaDGrMfldC0FSXKKr7H5C7FQErl936108wGohz
|
||||
53KGbNid0crNeaWrQuWk3QjaMF62Yc+NVB8b5LoEk0TEt9KPO6kveYDtW6Ygkw9i
|
||||
5aaS/AC+8GChOH6lWhLHcO91T7+Hi5qRez7qyme/AlHUTV9/jE6JgFJ7thCCAqBQ
|
||||
AfX3mGFlTwSTMHDeYq81MMl9UhYgoFJtx8PuqmSV7Eb29ccJofz3J+b5OJvpmLSZ
|
||||
Hu22Csa8k2V/1ojeL0DjQHD06V75F7BerxLR/rbJ64GkIiOdm4ntXC4a335gjQfM
|
||||
5tI6LtGsIpr+Le4C068eDj7YpzS0ql3g6o4A1TbtapLccgEj4iw95H4KKh0Ro5r0
|
||||
0s6LoSLu9uva3LiLVn6xXvpw3iv8FnhTr5PEiPnorXvxcCnUrB3Rkk6Wh1SKjpxa
|
||||
u6KiFzH0CXR1AgUdWSC0rAmvEdFjXgaoQd+Z2eJFHEAsg91URd9NtdeDsyMwdbHh
|
||||
iBZPGX5ddPjJA7nwKbkibCv7xiodd9ph8NmGLqD/2SDU1rqvPuCDMqLuxc89Gvhs
|
||||
3CeLTSJK9Uzb3oNdZLxL8CECip1ZkowVfTL+jY23phS9+Z0Db/yZqot3ZZJAAG8g
|
||||
oSVkzT/Mwp9zFVQu5w==
|
||||
=drMP
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,52 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGWdEG0BEADV9c8aSIITQmoWKN0ac4c8AEhZcdzpWSRPwHIS8syVRw/gj30W
|
||||
Mn1OoxqFn2UECh2jjqYaQFECXNrM2/lzO4GhvcuoPn6p0rXFMvYvwtiOJ+rmDvTe
|
||||
UOwRxkZhYehpvsRPFoVhlYJh3U1BM0UHm2iE/TIqVusY9FHQuCMZVl0BHk81YGDa
|
||||
qE3cyu+9zOs8z1bYlsYBgLIbtb4+loQTRexFTfZkUcCWgAaNyur8Wy7Msdm5LOqN
|
||||
ZZu6bNKBOv0Cr+BOwGNoFRbi4IM+5H+0eI1n7V5if4CuXfdYHp+Pm+PQvftm/nqB
|
||||
jRLKqioRKzHjk7xNMsi5lM+SU4WyfU77pr3RXuJR6fkymLVAeu5sy2U6ySj0SXIO
|
||||
/I3oPp4/zJ8/e94jG5iDqh92b5IWyXSE+EGw4kkpUvZLrK8vyyEfk8h9Gf8qA5uL
|
||||
yGfw0vnSVjYb+RdnfZFVdPaIEmeY8tnobS87duj8JESZiSWo5sU4filkciW5e+MJ
|
||||
w9wohgl+Cqm2J5Ljqus6DRm4ptKbZD8QZRnmGcSk6O2VopBKt2XLat4eIJLr/EiN
|
||||
0CJ1EH/oJmEeZbYtUjxjsGsbPXicwDH6ZIeIMSaiDgqRxPm1LTGVLjSHAk+k0e5l
|
||||
ulEmaoA7t/p3U2peGc7ONvozXiyNHZdGiRkqFC1mA9/JpeWw4KFH6JSgmQARAQAB
|
||||
tEtSZW1pJ3MgUlBNIHJlcG9zaXRvcnkgMjAyNCAoaHR0cHM6Ly9ycG1zLnJlbWly
|
||||
ZXBvLm5ldC8pIDxyZW1pQHJlbWlyZXBvLm5ldD6JAlEEEwEIADsWIQTPHfAFfOhd
|
||||
/1svKjfC/TssKglI5AUCZZ0QbQIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIX
|
||||
gAAKCRDC/TssKglI5Na7EACDTGjTcuTyudfuu/JC41BSurclCcwZyPsGaClyc0qx
|
||||
Tk8Rs4rbyVw1u494fiFG/yEVl7LGGja84UJOlMdw82KgVUmr4fGLaBBFWm+KnZ4W
|
||||
4vvFcCIxUshENOaDrkhby9F5P379gVOK9I3n9l4Q2xt+sWUNwE74CNfCTUeYMIm/
|
||||
/AHbb1DTM9FuIT6FssgvILMTlksiyrw8U8bj+QUzPZrFCpvv+PnFA7ZDMEyCf2Tz
|
||||
/zawxoP3a3b+3/g61U9SVIkfToonnorqh9lkQYhsdKTFHW6eESqe7zcqMBZv/sSc
|
||||
3kFzl22KczSdU8eYxdujv9BmlW2SOrQsnuMYxvoD1LzhwSB/nOapNrWU+djhJCof
|
||||
ueZ+rBW4QWysmG0dEp5Wpz85TstAnQ+j5aCOMSAkiAVsrJ0Ff3JXakIiL7rEtccS
|
||||
dhgBT06LJtJy1NGIC0Sotn45n6dtfi6IuI8LnELCKeHxhdFSJcAPB8y95u0uHymy
|
||||
sqMfzjq1VcYtzC5II6Yd5Gtb2+QzyQwORslwXhUdXwZKUc5i6X5xFMSxU523fH9n
|
||||
i+kQlUTPkOUr66v4fmYDj+ZlJQSFoPylcHCke/vOo/rH5nj+YMPa6LdohsdzHKpd
|
||||
DkKrk+Udj5aDXtXGIFYwoNKO4BnCsz/ImZfB4CQ+ekJsY1pFill3+MRTM6IvSA+P
|
||||
7LkCDQRlnRBtARAAq8k4FM9euMMsXz0buxxyr0ew0ecWNhZGy24a6DXORVTEJbgI
|
||||
T3HpV/C2PWkwgEBswYzREm+en61jhWr5unCiytO2klT+K2WNcIUNW+z2ciL/JNaJ
|
||||
OcKwjkiY/A+KtDStzllCQ/XnpgAXGvFZjMjhHrNGQoAwNnPE/cMtYwcuVdW3Ipm3
|
||||
cOOtzxRmfnhP8YUEyHxdos/7AZowaiLIUkaL5RUaygGrfPGyUXPl1aXS1oiOm37W
|
||||
RiKCGUxCHSlpBqeh1WQPOaps8y+ME9jMzF9SLvqiXlKPllJIx9i8tHUOnX6Y2U2x
|
||||
5sXsCEiA+zNVMU3YQ04++Yx2Rp6sDhNEnAbTCqqequqIzaXHZWwszAeG6pj1G90m
|
||||
8PwDeHy641ySgDMwVPpnH5rHWmmT96XvF9QNg07OBtW9/phryzwAuHogrBY4DyFD
|
||||
pUxDkTv9TxfBaS/s1uJxEGknIgnKzQo79mRy206waGKRA2AV36ORSfU12sK8pl3r
|
||||
N95iIDh/OIuMl2STaXilhwb1hRpGFvwRlq3Hvvan4SY1aO219Q+VxMGXeqC5F/rm
|
||||
Anh7mRVMP2qu0rj7L8xJ/OD/bPfuWL72/PvviBr/jdRM9I+Vj8dPGjhUwhK1dpEG
|
||||
Q6yvoVXLHXucVHfIVR+ALLBmK8oHXNIFeIXhiUbgmRAilPHRz6Rf5pgbrC0AEQEA
|
||||
AYkCNgQYAQgAIBYhBM8d8AV86F3/Wy8qN8L9OywqCUjkBQJlnRBtAhsMAAoJEML9
|
||||
OywqCUjkQ2kP/iwqvt1sX85KM4OSE9ftqemwnvh19NizMKHMYGgikMAvy2CndEOI
|
||||
wzKVeE1iK9/ivZQQIbD5l0G9hzVZjIxbgu9dAcxM+jdmaPyUj7yn6AX2yeGAeHyA
|
||||
qzNKZaTLJPieRzdXk/uKPVyWUuzbR0vwR7n8N5N0pe/FEfO9O0uX5iz2kf5+b10c
|
||||
WqvzLmO+czuyCjTHnjhKfE73iIktYE3MnM8qFHPiKwm+jeTkUnmInoR3qIKR/SCV
|
||||
u2vk/q5uvpNaUSPSDKji6wh3/hbNbPocLSGIjugfWNm/4LhlSHPNnRJ2dIvSxeiI
|
||||
qtSWxhULEyHGBPMLDriVumCV7Rb/byJwiMbmCIpxB5XLtbgt7nIqWQTko+6ci71S
|
||||
4rDXolM3bytYbpqq4b/Xc1bhBuTX3omwrd0kxmGqad8eia6O6iBsP7GAl4QkySm4
|
||||
MfKQfohjXTnT60wqEzQwLGI4NKN9bHRSsGINHX5p+Kz9HLf87fTY4uglbFxg0/II
|
||||
Y3nko5KBw/gfe0WpwZY0sWjK20cXJn3xLQjoG8OjUNUCUYTRsiAvn4AiDRt++qtl
|
||||
bvJjJqOU1k4GrGoCVpEUiOc9Cw1YZREcFe8H2at3pGnxJ1DFlYA916kNAxYANp2s
|
||||
r3N3agwevgIzP78hynxjm6a5H2w0vkg189pZn2RR9zX9gmrvlqg1o0P1
|
||||
=meaU
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,15 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIICPzCCAeagAwIBAgIUJI0QJZ+PCrSQ/nKZFENwwcp8Zr0wCgYIKoZIzj0EAwIw
|
||||
TDEmMCQGA1UEAwwdcGtnLWNhY2hlIGludGVybmFsIENBIChDVDEzMCkxIjAgBgNV
|
||||
BAoMGWNsb3VkLWhvc3RpbmcuaW8gaW50ZXJuYWwwHhcNMjYxMDIxMTkzNjIyWhcN
|
||||
MzYxMDE4MTkzNjIyWjBMMSYwJAYDVQQDDB1wa2ctY2FjaGUgaW50ZXJuYWwgQ0Eg
|
||||
KENUMTMwKTEiMCAGA1UECgwZY2xvdWQtaG9zdGluZy5pbyBpbnRlcm5hbDBZMBMG
|
||||
ByqGSM49AgEGCCqGSM49AwEHA0IABBJh75sC5vSzTnkYl0CYYEA/TKUHEXhbtanV
|
||||
OVNW4NlD3za5+bu5YuKjpbFAva4TCZqWsaS9IMd/JfsaFbf5XVejgaUwgaIwEgYD
|
||||
VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFP7UCTf/
|
||||
v7KU1KBrs1vqa71/TZl0MF0GA1UdHgEB/wRTMFGgTzAKhwjAqAEo/////zAlgiNw
|
||||
a2ctY2FjaGUuaW50ZXJuYWwuY2xvdWQtaG9zdGluZy5pbzAaghhwa2ctY2FjaGUu
|
||||
Z2Vla2hvbWUubG9jYWwwCgYIKoZIzj0EAwIDRwAwRAIgOod4wPHu0kAzYAw8KeGg
|
||||
cIrFMwQ4ATKVTkDioiSFOqICIEhY+sGj/YZ4oBMst3b/AAz9zMEJAXpnCkSTI6tM
|
||||
YAIl
|
||||
-----END CERTIFICATE-----
|
||||
Executable
+90
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
# Assemble the multi-arch index tag(s) from per-platform images that were
|
||||
# already pushed BY DIGEST (no tag). Used by the trusted x86 publish job
|
||||
# (see .gitea/workflows/build-push-coraza.yaml; template for the other WHP
|
||||
# image repos, docs/arm64/IMAGE-PIPELINE-PLAN.md section 4.1/4.4).
|
||||
#
|
||||
# assemble-multiarch-index.sh \
|
||||
# --amd64 sha256:<digest> \
|
||||
# --arm64 sha256:<digest> --arm64-outcome success|failure|cancelled|skipped \
|
||||
# --repo <registry/path> [--repo <other registry/path> ...] \
|
||||
# --tag <tag> [--tag <tag> ...] [--builder <buildx builder>]
|
||||
#
|
||||
# Rules (all enforced here so the workflow stays thin and these are testable):
|
||||
# * amd64 is mandatory. A missing/malformed amd64 digest, or an amd64 digest
|
||||
# that does not inspect as linux/amd64, aborts with exit 1 and publishes
|
||||
# NOTHING (the same "build failed, no tag" outcome as before multi-arch).
|
||||
# * arm64 is attached ONLY when its step outcome is exactly "success" AND the
|
||||
# digest is well-formed AND it inspects as linux/arm64. Otherwise the tag is
|
||||
# published amd64-only (warning, exit 0). The arm64 digest must come from
|
||||
# THIS run's step output - the workflow never reads it from a registry tag,
|
||||
# so a previous build's arm64 can never be re-attached to a newer amd64.
|
||||
# * every --tag is created in every --repo from <repo>@<digest> sources in
|
||||
# that same repo (both digests were pushed to every --repo by the build).
|
||||
#
|
||||
# Env: IMAGETOOLS overrides the command (default: "docker buildx imagetools");
|
||||
# the test suite points it at a stub.
|
||||
set -euo pipefail
|
||||
|
||||
IMAGETOOLS=${IMAGETOOLS:-docker buildx imagetools}
|
||||
amd64=""; arm64=""; outcome="skipped"; builder=""
|
||||
repos=(); tags=()
|
||||
|
||||
die() { echo "assemble-multiarch-index: $*" >&2; exit 1; }
|
||||
warn() { echo "::warning::assemble-multiarch-index: $*"; }
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--amd64) amd64=${2-}; shift 2 ;;
|
||||
--arm64) arm64=${2-}; shift 2 ;;
|
||||
--arm64-outcome) outcome=${2-}; shift 2 ;;
|
||||
--repo) repos+=("${2-}"); shift 2 ;;
|
||||
--tag) tags+=("${2-}"); shift 2 ;;
|
||||
--builder) builder=${2-}; shift 2 ;;
|
||||
*) die "unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
re='^sha256:[0-9a-f]{64}$'
|
||||
[ ${#repos[@]} -gt 0 ] || die "need at least one --repo"
|
||||
[ ${#tags[@]} -gt 0 ] || die "need at least one --tag"
|
||||
[[ $amd64 =~ $re ]] || die "amd64 digest missing or malformed ('$amd64'); refusing to publish"
|
||||
|
||||
bflag=(); [ -n "$builder" ] && bflag=(--builder "$builder")
|
||||
|
||||
# arch_of <ref>: prints the image config's architecture ("" on any failure).
|
||||
arch_of() {
|
||||
# shellcheck disable=SC2086
|
||||
$IMAGETOOLS inspect "${bflag[@]}" "$1" --format '{{json .Image}}' 2>/dev/null \
|
||||
| python3 -I -c 'import json,sys; print(json.load(sys.stdin).get("architecture",""))' 2>/dev/null || true
|
||||
}
|
||||
|
||||
for r in "${repos[@]}"; do
|
||||
[ "$(arch_of "$r@$amd64")" = amd64 ] || die "$r@$amd64 does not inspect as linux/amd64; refusing to publish"
|
||||
done
|
||||
|
||||
use_arm64=0
|
||||
if [ "$outcome" != success ]; then
|
||||
warn "arm64 step outcome is '$outcome': publishing amd64-only (no arm64 digest is carried over)"
|
||||
elif ! [[ $arm64 =~ $re ]]; then
|
||||
warn "arm64 step reported success but its digest is missing/malformed: publishing amd64-only"
|
||||
elif [ "$arm64" = "$amd64" ]; then
|
||||
warn "arm64 digest equals the amd64 digest: publishing amd64-only"
|
||||
else
|
||||
use_arm64=1
|
||||
for r in "${repos[@]}"; do
|
||||
if [ "$(arch_of "$r@$arm64")" != arm64 ]; then
|
||||
warn "$r@$arm64 does not inspect as linux/arm64: publishing amd64-only"
|
||||
use_arm64=0; break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
for r in "${repos[@]}"; do
|
||||
targs=(); for t in "${tags[@]}"; do targs+=(-t "$r:$t"); done
|
||||
srcs=("$r@$amd64"); [ "$use_arm64" = 1 ] && srcs+=("$r@$arm64")
|
||||
echo "assemble-multiarch-index: $r tags=${tags[*]} sources=${#srcs[@]} ($([ "$use_arm64" = 1 ] && echo 'amd64+arm64' || echo 'amd64-only'))"
|
||||
# shellcheck disable=SC2086
|
||||
$IMAGETOOLS create "${bflag[@]}" "${targs[@]}" "${srcs[@]}"
|
||||
done
|
||||
echo "platforms=$([ "$use_arm64" = 1 ] && echo linux/amd64,linux/arm64 || echo linux/amd64)"
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# LOCAL end-to-end rehearsal of the build-push-coraza.yaml multi-arch logic
|
||||
# against THROWAWAY registries. Never touches repo.anhonesthost.net / ghcr.io.
|
||||
#
|
||||
# docker run -d --name e2ereg1 -p 127.0.0.1:5077:5000 registry:2
|
||||
# docker run -d --name e2ereg2 -p 127.0.0.1:5078:5000 registry:2
|
||||
# REG1=127.0.0.1:5077 REG2=127.0.0.1:5078 scripts/e2e-coraza-multiarch.sh
|
||||
# (then: docker rm -f e2ereg1 e2ereg2; docker buildx rm e2ebuilder)
|
||||
#
|
||||
# If the Docker daemon is not on this machine (remote/VM daemon), publish the
|
||||
# registries on an address both sides reach (e.g. -p 172.17.0.1:5077:5000) and
|
||||
# set REG1/REG2 to it; plain-HTTP is enabled for them in the builder config.
|
||||
# Scenario 1: both platforms build, pushed by digest to TWO registries,
|
||||
# index assembled; checks both platforms + attestations present.
|
||||
# Scenario 2: arm64 forced to fail (bad upstream tag) while a STALE arm64
|
||||
# digest from scenario 1 is still known: the tag must become
|
||||
# amd64-only and must not contain the stale arm64 digest.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
REG1=${REG1:?set REG1}; REG2=${REG2:?set REG2}
|
||||
B=${BUILDER:-e2ebuilder}
|
||||
R1=$REG1/cloud-hosting-platform/coraza-spoa
|
||||
R2=$REG2/shadowdao/coraza-spoa
|
||||
if ! docker buildx inspect "$B" >/dev/null 2>&1; then
|
||||
cfg=$(mktemp)
|
||||
printf '[registry."%s"]\n http=true\n insecure=true\n[registry."%s"]\n http=true\n insecure=true\n' "$REG1" "$REG2" > "$cfg"
|
||||
docker buildx create --name "$B" --driver docker-container --driver-opt network=host --buildkitd-config "$cfg" --bootstrap >/dev/null
|
||||
fi
|
||||
OUT="type=image,\"name=$R1,$R2\",push-by-digest=true,name-canonical=true,push=true"
|
||||
build() { # build <arch> [extra args]; prints digest
|
||||
local arch=$1; shift; local meta; meta=$(mktemp)
|
||||
docker buildx build --builder "$B" --platform "linux/$arch" --output "$OUT" --metadata-file "$meta" "$@" ./coraza-spoa >&2
|
||||
python3 -I -c 'import json,sys; print(json.load(open(sys.argv[1]))["containerimage.digest"])' "$meta"
|
||||
}
|
||||
inspect() { docker buildx imagetools inspect --builder "$B" "$@"; }
|
||||
|
||||
echo "=== scenario 1: both platforms ==="
|
||||
AMD=$(build amd64); ARM=$(build arm64)
|
||||
echo "amd64=$AMD arm64=$ARM"
|
||||
IMAGETOOLS="docker buildx imagetools" scripts/assemble-multiarch-index.sh --builder "$B" \
|
||||
--amd64 "$AMD" --arm64 "$ARM" --arm64-outcome success --repo "$R1" --repo "$R2" --tag latest
|
||||
for r in "$R1" "$R2"; do echo "--- $r:latest"; inspect "$r:latest" | grep -E 'Digest:|Platform:'; done
|
||||
|
||||
echo "=== scenario 2: forced arm64 failure, stale arm64 digest ($ARM) still in hand ==="
|
||||
if NEWARM=$(build arm64 --build-arg CORAZA_SPOA_VERSION=v0.0.0-does-not-exist 2>/dev/null); then echo "BUG: forced failure succeeded"; exit 1; fi
|
||||
echo "arm64 step failed as forced"
|
||||
scripts/assemble-multiarch-index.sh --builder "$B" \
|
||||
--amd64 "$AMD" --arm64 "$ARM" --arm64-outcome failure --repo "$R1" --repo "$R2" --tag latest
|
||||
for r in "$R1" "$R2"; do
|
||||
echo "--- $r:latest"; inspect "$r:latest" | grep -E 'Digest:|Platform:'
|
||||
if inspect "$r:latest" | grep -q 'linux/arm64'; then echo "FAIL: arm64 still attached"; exit 1; fi
|
||||
if inspect "$r:latest" --raw | grep -q "$ARM"; then echo "FAIL: stale arm64 digest present"; exit 1; fi
|
||||
done
|
||||
echo "E2E OK"
|
||||
@@ -34,6 +34,18 @@ def main():
|
||||
haproxy_manager.logger.warning(
|
||||
"[haproxy-supervisor] haproxy process not found — attempting in-place restart"
|
||||
)
|
||||
# Breadcrumb for the monitoring pipeline: the last thing HAProxy itself said
|
||||
# before it vanished (the relay forwards the same text; this ties it to the
|
||||
# supervisor event even if the relay is behind).
|
||||
try:
|
||||
with open(haproxy_manager._stderr_log_path(), 'rb') as fh:
|
||||
fh.seek(0, 2)
|
||||
fh.seek(max(0, fh.tell() - 1500))
|
||||
tail = fh.read().decode('utf-8', 'replace').strip().splitlines()[-6:]
|
||||
for line in tail:
|
||||
haproxy_manager.logger.warning("[haproxy-supervisor] last haproxy stderr: %s", line)
|
||||
except OSError:
|
||||
pass
|
||||
# start_haproxy() validates the config (and regenerates it if invalid)
|
||||
# before launching, and swallows its own errors, so it will not raise here.
|
||||
haproxy_manager.start_haproxy()
|
||||
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bound the management-API (:8000) gunicorn access log file.
|
||||
|
||||
Why
|
||||
---
|
||||
Gunicorn's access log used to go to stdout (`--access-logfile -`), i.e. every
|
||||
:8080 default-page / blocked-IP / suspended-site hit and every :8000 API call
|
||||
became a `docker logs` line. With the journald docker log driver that is one
|
||||
journal entry per request (journald's default rate limit, 10000 msgs / 30 s per
|
||||
sending service, can then SUPPRESS the crash-dump lines the relay emits) and it
|
||||
floods Graylog. Now:
|
||||
* :8080 (default page) has NO access log at all;
|
||||
* :8000 (API) writes its access log to API_ACCESS_LOG (default
|
||||
/var/log/haproxy/api-access.log, in the haproxy-logs volume), NOT stdout;
|
||||
* gunicorn ERROR logging (tracebacks, worker timeouts) stays on stderr.
|
||||
This process keeps that file bounded: copy-truncate rotation at
|
||||
HAPROXY_API_ACCESS_LOG_MAX_BYTES (default 20 MiB), HAPROXY_API_ACCESS_LOG_BACKUPS
|
||||
generations (default 3), rotated copies mode 0640 (same policy as
|
||||
haproxy-stderr.log). Truncating in place is safe because gunicorn opens the
|
||||
file in append mode. Several gunicorn processes share the file, so renaming it
|
||||
is not an option.
|
||||
|
||||
Bound: size is checked every HAPROXY_API_ACCESS_LOG_POLL seconds (default 30),
|
||||
so the live file can exceed the limit only by what the API writes in one
|
||||
interval (a few hundred bytes per call).
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def _env_num(name, default, cast=int):
|
||||
try:
|
||||
v = cast(os.environ.get(name, default))
|
||||
return v if v > 0 else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def rotate_if_needed(path, max_bytes, backups):
|
||||
"""Copy-truncate rotate. Returns True if it rotated."""
|
||||
try:
|
||||
if os.path.getsize(path) <= max_bytes:
|
||||
return False
|
||||
for i in range(backups, 1, -1):
|
||||
src = '%s.%d' % (path, i - 1)
|
||||
if os.path.exists(src):
|
||||
os.replace(src, '%s.%d' % (path, i))
|
||||
shutil.copyfile(path, path + '.1')
|
||||
os.chmod(path + '.1', 0o640)
|
||||
with open(path, 'r+b') as fh:
|
||||
fh.truncate(0)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except OSError as exc:
|
||||
sys.stderr.write('gunicorn-access-rotate: rotate failed: %s\n' % exc)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
path = os.environ.get('HAPROXY_MGR_API_ACCESS_LOG', '/var/log/haproxy/api-access.log')
|
||||
max_bytes = _env_num('HAPROXY_API_ACCESS_LOG_MAX_BYTES', 20 * 1024 * 1024)
|
||||
backups = _env_num('HAPROXY_API_ACCESS_LOG_BACKUPS', 3)
|
||||
poll = _env_num('HAPROXY_API_ACCESS_LOG_POLL', 30.0, float)
|
||||
while True:
|
||||
try:
|
||||
rotate_if_needed(path, max_bytes, backups)
|
||||
except Exception as exc: # never die; the supervisor loop restarts us anyway
|
||||
sys.stderr.write('gunicorn-access-rotate: %r\n' % exc)
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
"""In-container receiver for the HAProxy access log.
|
||||
|
||||
HAProxy sends its access log (`log <socket> ... format rfc5424`, hap_header.tpl)
|
||||
as datagrams to a UNIX socket inside this container. This process is the only
|
||||
reader: it writes one line per datagram to a size-capped, rotating file in the
|
||||
haproxy-logs volume (default /var/log/haproxy/access.log).
|
||||
|
||||
Why it looks the way it does
|
||||
----------------------------
|
||||
* UNIX DATAGRAM socket, no network listener. Nothing is exposed on the host
|
||||
or on client-net (the old target was the host's rsyslog on the docker
|
||||
bridge, 172.18.0.1:514, which had no listener on the fleet).
|
||||
* HAProxy never waits for us. Its log sends are non-blocking: a dead receiver
|
||||
(ECONNREFUSED/ENOENT) or a full socket buffer (EAGAIN) drops that line and
|
||||
HAProxy carries on. So a crash, a stall or a full disk here can only lose
|
||||
log lines, never slow or fail a request. That is the intended direction.
|
||||
* Bounded on disk: the live file is rotated at MAX_BYTES and BACKUPS gzip
|
||||
generations are kept, so the worst case (incompressible data) is
|
||||
(1 + BACKUPS) * MAX_BYTES -- 5 * 40 MB = 200 MB by default. Compression runs
|
||||
in a thread so a rotation does not stall reading.
|
||||
* A full disk (ENOSPC) or any write error drops lines and is reported at most
|
||||
once a minute; the file is reopened on the next attempt.
|
||||
* Lines go to the FILE ONLY, never stdout: `docker logs haproxy-manager` is a
|
||||
source of WHP's AI log monitor, and a per-request firehose there would storm
|
||||
it (see the WHP note on audit/firehose sources). Our own status messages are
|
||||
rare and go to stderr.
|
||||
* Control characters inside a datagram are replaced so one datagram is always
|
||||
exactly one line (a crafted header cannot forge extra log lines).
|
||||
|
||||
Supervised by start-up.sh (restart loop), started before HAProxy.
|
||||
|
||||
Environment
|
||||
-----------
|
||||
HAPROXY_LOG_SOCKET socket path (default /var/lib/haproxy/log.sock)
|
||||
HAPROXY_ACCESS_LOG file path (default /var/log/haproxy/access.log)
|
||||
HAPROXY_ACCESS_LOG_MAX_BYTES rotate size (default 41943040 = 40 MiB)
|
||||
HAPROXY_ACCESS_LOG_BACKUPS gzip generations kept (default 4)
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
# HAProxy sends at most `len` (2048) bytes per line; leave headroom.
|
||||
MAX_DATAGRAM = 65535
|
||||
FLUSH_INTERVAL = 1.0
|
||||
ERROR_REPORT_INTERVAL = 60.0
|
||||
# Kernel receive buffer to absorb bursts while a write is slow.
|
||||
RCVBUF = 4 * 1024 * 1024
|
||||
|
||||
_CTRL = {c: ' ' for c in range(0x20) if c != 0x09}
|
||||
_CTRL[0x7f] = ' '
|
||||
|
||||
|
||||
def _env_int(name, default, minimum):
|
||||
try:
|
||||
value = int(os.environ.get(name, default))
|
||||
except ValueError:
|
||||
value = default
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
def clean_line(datagram):
|
||||
"""One datagram -> one log line (str, no trailing newline).
|
||||
|
||||
Drops the RFC 5424 `<PRI>` prefix (constant for our single facility) and
|
||||
replaces any control character so the datagram cannot span lines.
|
||||
"""
|
||||
text = datagram.decode('utf-8', 'replace').rstrip('\r\n\x00')
|
||||
if text.startswith('<'):
|
||||
end = text.find('>', 1, 6)
|
||||
if end > 0 and text[1:end].isdigit():
|
||||
text = text[end + 1:]
|
||||
return text.translate(_CTRL)
|
||||
|
||||
|
||||
class RotatingWriter:
|
||||
"""Size-capped writer: live file + `backups` gzip generations."""
|
||||
|
||||
def __init__(self, path, max_bytes, backups, report=None):
|
||||
self.path = path
|
||||
self.max_bytes = max_bytes
|
||||
self.backups = backups
|
||||
self.report = report or (lambda msg: None)
|
||||
self.fh = None
|
||||
self.size = 0
|
||||
self.dirty = False
|
||||
self.dropped = 0
|
||||
self._compressor = None
|
||||
self._last_error = 0.0
|
||||
|
||||
# -- file handling --------------------------------------------------
|
||||
def _open(self):
|
||||
os.makedirs(os.path.dirname(self.path) or '.', exist_ok=True)
|
||||
self.fh = open(self.path, 'a', encoding='utf-8', buffering=65536)
|
||||
os.chmod(self.path, 0o640)
|
||||
self.size = self.fh.tell()
|
||||
|
||||
def _error(self, what, exc):
|
||||
self.dropped += 1
|
||||
try:
|
||||
if self.fh:
|
||||
self.fh.close()
|
||||
except OSError:
|
||||
pass
|
||||
self.fh = None
|
||||
now = time.monotonic()
|
||||
if now - self._last_error >= ERROR_REPORT_INTERVAL:
|
||||
self._last_error = now
|
||||
self.report(f'{what} failed ({exc}); dropping access-log lines '
|
||||
f'(dropped so far: {self.dropped})')
|
||||
|
||||
def _gen(self, n):
|
||||
return f'{self.path}.{n}.gz'
|
||||
|
||||
def _compress(self, src, dst):
|
||||
tmp = dst + '.tmp'
|
||||
try:
|
||||
with open(src, 'rb') as fin, gzip.open(tmp, 'wb', compresslevel=6) as fout:
|
||||
shutil.copyfileobj(fin, fout, 1024 * 1024)
|
||||
os.replace(tmp, dst)
|
||||
os.chmod(dst, 0o640)
|
||||
except OSError as exc:
|
||||
self.report(f'compressing {src} failed ({exc}); removing it to keep the size cap')
|
||||
for leftover in (tmp,):
|
||||
try:
|
||||
os.unlink(leftover)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
os.unlink(src)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def rotate(self):
|
||||
if self._compressor is not None:
|
||||
self._compressor.join()
|
||||
self._compressor = None
|
||||
if self.fh:
|
||||
try:
|
||||
self.fh.close()
|
||||
except OSError:
|
||||
pass
|
||||
self.fh = None
|
||||
if self.backups <= 0:
|
||||
try:
|
||||
os.unlink(self.path)
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
os.unlink(self._gen(self.backups))
|
||||
except OSError:
|
||||
pass
|
||||
for n in range(self.backups - 1, 0, -1):
|
||||
if os.path.exists(self._gen(n)):
|
||||
os.replace(self._gen(n), self._gen(n + 1))
|
||||
staged = f'{self.path}.1'
|
||||
if os.path.exists(self.path):
|
||||
os.replace(self.path, staged)
|
||||
self._compressor = threading.Thread(
|
||||
target=self._compress, args=(staged, self._gen(1)), daemon=True)
|
||||
self._compressor.start()
|
||||
self.size = 0
|
||||
|
||||
# -- public ---------------------------------------------------------
|
||||
def write(self, line):
|
||||
data = line + '\n'
|
||||
try:
|
||||
if self.fh is None:
|
||||
self._open()
|
||||
if self.size and self.size + len(data) > self.max_bytes:
|
||||
self.rotate()
|
||||
self._open()
|
||||
self.fh.write(data)
|
||||
self.size += len(data.encode('utf-8'))
|
||||
self.dirty = True
|
||||
except OSError as exc:
|
||||
self._error('writing ' + self.path, exc)
|
||||
|
||||
def flush(self):
|
||||
if self.fh and self.dirty:
|
||||
try:
|
||||
self.fh.flush()
|
||||
self.dirty = False
|
||||
except OSError as exc:
|
||||
self._error('flushing ' + self.path, exc)
|
||||
|
||||
def close(self):
|
||||
self.flush()
|
||||
if self._compressor is not None:
|
||||
self._compressor.join()
|
||||
if self.fh:
|
||||
try:
|
||||
self.fh.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def bind_socket(path):
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
try:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, RCVBUF)
|
||||
except OSError:
|
||||
pass
|
||||
os.makedirs(os.path.dirname(path) or '.', exist_ok=True)
|
||||
try:
|
||||
os.unlink(path) # stale socket from a previous (crashed) receiver
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
sock.bind(path)
|
||||
# HAProxy's workers run as user haproxy (hap_header.tpl `user haproxy`).
|
||||
os.chmod(path, 0o666)
|
||||
sock.settimeout(FLUSH_INTERVAL)
|
||||
return sock
|
||||
|
||||
|
||||
def serve(sock, writer, stop=None):
|
||||
last_flush = time.monotonic()
|
||||
while stop is None or not stop.is_set():
|
||||
try:
|
||||
datagram = sock.recv(MAX_DATAGRAM)
|
||||
except socket.timeout:
|
||||
datagram = None
|
||||
except OSError:
|
||||
if stop is not None and stop.is_set():
|
||||
break
|
||||
raise
|
||||
if datagram:
|
||||
writer.write(clean_line(datagram))
|
||||
now = time.monotonic()
|
||||
if now - last_flush >= FLUSH_INTERVAL:
|
||||
writer.flush()
|
||||
last_flush = now
|
||||
|
||||
|
||||
def main():
|
||||
sock_path = os.environ.get('HAPROXY_LOG_SOCKET', '/var/lib/haproxy/log.sock')
|
||||
log_path = os.environ.get('HAPROXY_ACCESS_LOG', '/var/log/haproxy/access.log')
|
||||
max_bytes = _env_int('HAPROXY_ACCESS_LOG_MAX_BYTES', 40 * 1024 * 1024, 64 * 1024)
|
||||
backups = _env_int('HAPROXY_ACCESS_LOG_BACKUPS', 4, 0)
|
||||
|
||||
def report(msg):
|
||||
sys.stderr.write(f'haproxy-log-receiver: {msg}\n')
|
||||
sys.stderr.flush()
|
||||
|
||||
writer = RotatingWriter(log_path, max_bytes, backups, report)
|
||||
sock = bind_socket(sock_path)
|
||||
report(f'listening on {sock_path} -> {log_path} '
|
||||
f'(rotate at {max_bytes} bytes, keep {backups} gz)')
|
||||
try:
|
||||
serve(sock, writer)
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+363
@@ -0,0 +1,363 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Forward HAProxy's own stderr to the container's stderr, and bound the file.
|
||||
|
||||
Why
|
||||
---
|
||||
HAProxy's master stderr carries what the access log never will: `[ALERT]`
|
||||
lines, "Current worker (N) exited with code 134", and the complete
|
||||
ha_panic()/watchdog thread dump of a crash. haproxy_manager.spawn_haproxy_master()
|
||||
launches `haproxy -W -db` with that stderr appended to HAPROXY_STDERR_LOG (in the
|
||||
haproxy-logs volume, so it survives a container recreate). This process tails the
|
||||
file and re-emits each line on ITS stderr, prefixed, which is the container's
|
||||
stderr: `docker logs`, and through the docker logging driver the host journal
|
||||
and Graylog.
|
||||
|
||||
Properties
|
||||
----------
|
||||
* Rare by nature: HAProxy only writes here on start/reload/server state
|
||||
changes/crashes. The per-request access log is NOT here (it goes to the
|
||||
receiver socket), so this does not feed the WHP AI monitor a firehose.
|
||||
* Crash evidence bypasses the rate limit: lines that match the ANCHORED CRASH_RE
|
||||
("[ALERT] (PID) :", "Current worker (N) exited with code N", "Thread N is
|
||||
about to kill the process", "Thread N : id=0x..."; a keyword inside a URL/Host/
|
||||
server name never matches) and every line of a dump block (from a panic/watchdog header to
|
||||
the master's "All workers exited" line, bounded by
|
||||
DUMP_MAX_LINES / DUMP_MAX_SECONDS, then a DUMP_COOLDOWN before another block
|
||||
can open; blocks open only on the anchored "Thread N is about to kill" header)
|
||||
are forwarded in full even when the bucket is empty, within a total budget of
|
||||
CRASH_BUDGET lines per CRASH_WINDOW (after that they take the normal limiter
|
||||
and are counted in the suppressed summary). They go to STDERR.
|
||||
* Tenant-driven state lines (TENANT_RE: "backend 'b' has no server available!",
|
||||
"Server b/s is DOWN|UP ...") are NOT crash evidence even though HAProxy prints
|
||||
the first at [ALERT]: every tenant container stop/crash/deploy makes them.
|
||||
Own rate-limited STDOUT class (own bucket TENANT_BURST/TENANT_PER_MIN, own
|
||||
"[relay] suppressed N tenant-driven ..." summary). They also reach the WHP AI
|
||||
monitor through `docker logs haproxy-manager`; rate-limiting them is intended.
|
||||
Dump blocks have their own budget (DUMP_BUDGET) that nothing else can spend.
|
||||
* Routine reload noise (ROUTINE_RE: "Proxy X stopped (cumulated conns ...)",
|
||||
config "parsing [...]" warnings, "Former worker (N) exited with code 0",
|
||||
"Initializing new worker", "Loading success", ...) never touches the bucket:
|
||||
it is counted and collapsed into ONE "[relay] reload/start ok: N routine
|
||||
lines collapsed" line per reload (plus one on the minute tick for the late
|
||||
"Proxy stopped" lines). The full text stays in the stderr file.
|
||||
* Everything else is rate-limited (token bucket, HAPROXY_STDERR_RELAY_BURST
|
||||
lines burst, HAPROXY_STDERR_RELAY_PER_MIN refill) with a per-minute
|
||||
"[relay] suppressed N lines" summary.
|
||||
* Priority: the docker journald driver maps stdout -> PRIORITY 6 (info) and
|
||||
stderr -> PRIORITY 3 (err) and does NOT parse "<N>" prefixes. So benign
|
||||
lines (routine, rate-limited remainder, relay summaries) are written to
|
||||
STDOUT and crash evidence to STDERR: Graylog/journal can alert on priority.
|
||||
* Bounded file: copy-truncate rotation at HAPROXY_STDERR_LOG_MAX_BYTES
|
||||
(default 20 MiB), HAPROXY_STDERR_LOG_BACKUPS generations (default 3).
|
||||
Truncating is safe because HAProxy's fd is O_APPEND.
|
||||
* Never blocks or affects HAProxy: it only reads a file.
|
||||
|
||||
Environment
|
||||
-----------
|
||||
HAPROXY_STDERR_LOG, HAPROXY_STDERR_RELAY_BURST (400), HAPROXY_STDERR_RELAY_PER_MIN (120),
|
||||
HAPROXY_STDERR_LOG_MAX_BYTES (20971520), HAPROXY_STDERR_LOG_BACKUPS (3),
|
||||
HAPROXY_STDERR_RELAY_POLL (0.5 s)
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
PREFIX = 'haproxy-stderr: '
|
||||
MAX_LINE = 2000 # longer lines are truncated (hostile / runaway output)
|
||||
MAX_PENDING = 65536 # an unterminated line is force-emitted past this size
|
||||
DUMP_MAX_LINES = 500 # a dump block never bypasses the limiter longer than this
|
||||
DUMP_MAX_SECONDS = 30.0
|
||||
SUMMARY_EVERY = 60.0
|
||||
|
||||
# Every pattern is ANCHORED to HAProxy's own message structure (captured from
|
||||
# real 3.2 crashes): a keyword inside a request path / Host header / server
|
||||
# name that HAProxy echoes into a warning can never match, because the line
|
||||
# prefix "[ALERT] (PID) :" is written by HAProxy itself.
|
||||
# Crash evidence: forwarded (to stderr) without the bucket, within CRASH_BUDGET.
|
||||
CRASH_RE = re.compile(
|
||||
r'^(?:'
|
||||
r'\[ALERT\]\s+\(\d+\)\s*:\s*(?:Current worker \(\d+\) exited with code|exit-on-failure:|A worker process unexpectedly died)'
|
||||
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Current|Former) worker \(\d+\) exited with code (?!0\b)\d+'
|
||||
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*A worker process unexpectedly died'
|
||||
r'|\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b'
|
||||
r'|Thread \d+ is about to kill the process'
|
||||
r'|\s*\*?\s*Thread \d+ : id=0x[0-9a-f]+ '
|
||||
r')')
|
||||
# Tenant-driven state lines. WHP gives every tenant server `check`, so every
|
||||
# tenant container stop/crash/OOM/deploy and every host-wide container restart
|
||||
# makes HAProxy print these at runtime (at ALERT/WARNING level). They are NOT crash
|
||||
# evidence: own RATE-LIMITED class on STDOUT with their own bucket and summary, so
|
||||
# they can neither reach priority 3 nor spend the general bucket / crash budgets.
|
||||
TENANT_RE = re.compile(
|
||||
r'^\[(?:ALERT|WARNING|NOTICE)\]\s+\(\d+\)\s*:\s*(?:'
|
||||
r"backend '[^']*' has no server available!"
|
||||
r'|Server \S+ (?:is (?:DOWN|UP|going DOWN|stopping|NOLB)\b|was DOWN\b)'
|
||||
r'|Health check for (?:backup )?server \S+ '
|
||||
r')')
|
||||
# A dump block opens ONLY on the watchdog/panic header, and runs until DUMP_END_RE.
|
||||
DUMP_START_RE = re.compile(r'^Thread \d+ is about to kill the process')
|
||||
DUMP_END_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*All workers exited\b')
|
||||
# A new start/reload cycle also closes a dump block.
|
||||
CYCLE_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:Reloading HAProxy|Initializing new worker|Loading success)')
|
||||
# Routine per-start/reload noise: counted and collapsed, never rate-limited.
|
||||
ROUTINE_RE = re.compile(
|
||||
r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*(?:'
|
||||
r'Proxy \S+ stopped \(cumulated conns'
|
||||
r'|config\s*:\s*parsing \[[^\]]*\]\s*:'
|
||||
r'|Former worker \(\d+\) exited with code 0\b'
|
||||
r'|Reloading HAProxy|Initializing new worker|Loading success'
|
||||
r'|path to executable is|haproxy version is'
|
||||
r"|config: Can't load stats-file|'option forwardfor' ignored)")
|
||||
LOADED_RE = re.compile(r'^\[(?:NOTICE|WARNING)\]\s+\(\d+\)\s*:\s*Loading success')
|
||||
# Total exemption budget: even real-looking evidence cannot flood without bound.
|
||||
CRASH_BUDGET = 1000 # exempt lines per window (a 4-thread dump is ~70)
|
||||
CRASH_WINDOW = 600.0 # seconds
|
||||
DUMP_BUDGET = 1000 # dump-block lines (header + body) per window: a SEPARATE budget that
|
||||
# ordinary evidence lines (worker exits, alerts) can never spend
|
||||
TENANT_BURST = 100 # tenant-driven state lines: own bucket (burst / refill per minute)
|
||||
TENANT_PER_MIN = 30
|
||||
DUMP_COOLDOWN = 60.0 # after a BOUND closes a block, no new block opens for this long
|
||||
|
||||
|
||||
def _env_num(name, default, cast=int):
|
||||
try:
|
||||
v = cast(os.environ.get(name, default))
|
||||
return v if v > 0 else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, path, out, burst, per_min, max_bytes, backups, clock=time.monotonic, err=None):
|
||||
self.path = path
|
||||
self.out = out # benign lines (docker journald: PRIORITY 6)
|
||||
self.err = err if err is not None else out # crash lines (PRIORITY 3)
|
||||
self.burst, self.per_min = burst, per_min
|
||||
self.max_bytes, self.backups = max_bytes, backups
|
||||
self.clock = clock
|
||||
self.tokens = float(burst)
|
||||
self.last = clock()
|
||||
self.suppressed = 0
|
||||
self.routine = 0 # collapsed routine lines since the last summary
|
||||
self.summary_at = clock()
|
||||
self.in_dump = False
|
||||
self.dump_lines = 0
|
||||
self.dump_since = 0.0
|
||||
self.cooldown_until = 0.0
|
||||
self.budget_left = CRASH_BUDGET
|
||||
self.window_start = clock()
|
||||
self.dump_budget_left = DUMP_BUDGET
|
||||
self.dump_window_start = clock()
|
||||
self.tenant_tokens = float(TENANT_BURST)
|
||||
self.tenant_last = clock()
|
||||
self.tenant_suppressed = 0
|
||||
self.offset = None # None = first pass: start at EOF (do not replay history)
|
||||
self.pending = ''
|
||||
self._last_stream = None
|
||||
|
||||
def _take(self):
|
||||
now = self.clock()
|
||||
self.tokens = min(self.burst, self.tokens + (now - self.last) * self.per_min / 60.0)
|
||||
self.last = now
|
||||
if self.tokens >= 1:
|
||||
self.tokens -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def _take_tenant(self):
|
||||
now = self.clock()
|
||||
self.tenant_tokens = min(TENANT_BURST, self.tenant_tokens + (now - self.tenant_last) * TENANT_PER_MIN / 60.0)
|
||||
self.tenant_last = now
|
||||
if self.tenant_tokens >= 1:
|
||||
self.tenant_tokens -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def _write(self, stream, text):
|
||||
# keep the two streams in emission order: flush the other one first
|
||||
if self._last_stream is not None and self._last_stream is not stream:
|
||||
self._last_stream.flush()
|
||||
stream.write(PREFIX + text + '\n')
|
||||
self._last_stream = stream
|
||||
|
||||
def _flush_all(self):
|
||||
for st in {id(self.out): self.out, id(self.err): self.err}.values():
|
||||
try:
|
||||
st.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _summaries(self, force=False):
|
||||
"""Per-minute summary of what the limiter / collapse withheld."""
|
||||
now = self.clock()
|
||||
if not force and now - self.summary_at < SUMMARY_EVERY:
|
||||
return
|
||||
self.summary_at = now
|
||||
if self.routine:
|
||||
self._write(self.out, '[relay] %d routine reload/start lines collapsed (full text in the stderr file)'
|
||||
% self.routine)
|
||||
self.routine = 0
|
||||
if self.tenant_suppressed:
|
||||
self._write(self.out, '[relay] suppressed %d tenant-driven server/backend state lines '
|
||||
'(rate limit; full text in the stderr file)' % self.tenant_suppressed)
|
||||
self.tenant_suppressed = 0
|
||||
if self.suppressed:
|
||||
self._write(self.out, '[relay] suppressed %d lines (rate limit; full text in the stderr file)'
|
||||
% self.suppressed)
|
||||
self.suppressed = 0
|
||||
|
||||
def _budget(self, now, kind='crash'):
|
||||
"""Take one unit of a budget; False once the window's budget is spent.
|
||||
|
||||
kind 'crash': ordinary evidence lines; kind 'dump': dump-block lines (own budget)."""
|
||||
if kind == 'dump':
|
||||
if now - self.dump_window_start >= CRASH_WINDOW:
|
||||
self.dump_window_start, self.dump_budget_left = now, DUMP_BUDGET
|
||||
if self.dump_budget_left > 0:
|
||||
self.dump_budget_left -= 1
|
||||
return True
|
||||
return False
|
||||
if now - self.window_start >= CRASH_WINDOW:
|
||||
self.window_start, self.budget_left = now, CRASH_BUDGET
|
||||
if self.budget_left > 0:
|
||||
self.budget_left -= 1
|
||||
return True
|
||||
return False
|
||||
|
||||
def _end_dump(self, now, why=None):
|
||||
self.in_dump = False
|
||||
if why: # closed by a bound, not by the master's end line: no re-arming for a while
|
||||
self.cooldown_until = now + DUMP_COOLDOWN
|
||||
self._write(self.err, '[relay] dump block closed: %s; rate limit applies again' % why)
|
||||
|
||||
def _emit(self, line):
|
||||
now = self.clock()
|
||||
if self.in_dump:
|
||||
if (self.dump_lines >= DUMP_MAX_LINES) or (now - self.dump_since > DUMP_MAX_SECONDS):
|
||||
self._end_dump(now, 'bound reached (%d lines / %.0fs)' % (self.dump_lines, now - self.dump_since))
|
||||
elif CYCLE_RE.match(line):
|
||||
self._end_dump(now)
|
||||
if self.in_dump or CRASH_RE.match(line):
|
||||
is_dump = self.in_dump or bool(DUMP_START_RE.match(line))
|
||||
kind = 'dump' if is_dump else 'crash'
|
||||
if self._budget(now, kind):
|
||||
# crash evidence: bypasses the bucket, the collapse and the summary
|
||||
if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:
|
||||
self.in_dump, self.dump_lines, self.dump_since = True, 0, now
|
||||
if self.in_dump:
|
||||
self.dump_lines += 1
|
||||
self._write(self.err, line)
|
||||
if self.in_dump and DUMP_END_RE.match(line):
|
||||
self._end_dump(now)
|
||||
return
|
||||
if self.in_dump: # budget spent mid-block: the normal limiter takes over
|
||||
self._end_dump(now, 'crash-evidence budget (%d lines/%.0fs) spent' % (CRASH_BUDGET, CRASH_WINDOW))
|
||||
if ROUTINE_RE.match(line):
|
||||
self.routine += 1
|
||||
if LOADED_RE.match(line): # one summary per start/reload
|
||||
self._write(self.out, '[relay] reload/start ok: %d routine lines collapsed (full text in the stderr file)'
|
||||
% self.routine)
|
||||
self.routine = 0
|
||||
self._summaries()
|
||||
return
|
||||
if TENANT_RE.match(line):
|
||||
if self._take_tenant():
|
||||
self._write(self.out, line)
|
||||
else:
|
||||
self.tenant_suppressed += 1
|
||||
self._summaries()
|
||||
return
|
||||
if self._take():
|
||||
self._write(self.out, line)
|
||||
else:
|
||||
self.suppressed += 1
|
||||
self._summaries()
|
||||
|
||||
@staticmethod
|
||||
def _clean(line):
|
||||
line = ''.join(c if c >= ' ' or c == '\t' else ' ' for c in line).rstrip()
|
||||
if len(line) > MAX_LINE:
|
||||
line = line[:MAX_LINE] + ' [relay: line truncated, %d chars]' % len(line)
|
||||
return line
|
||||
|
||||
def rotate_if_needed(self):
|
||||
try:
|
||||
if os.path.getsize(self.path) <= self.max_bytes:
|
||||
return
|
||||
for i in range(self.backups, 1, -1):
|
||||
src = '%s.%d' % (self.path, i - 1)
|
||||
if os.path.exists(src):
|
||||
os.replace(src, '%s.%d' % (self.path, i))
|
||||
shutil.copyfile(self.path, self.path + '.1')
|
||||
os.chmod(self.path + '.1', 0o640)
|
||||
with open(self.path, 'r+b') as fh:
|
||||
fh.truncate(0)
|
||||
self.offset = 0
|
||||
except OSError as exc:
|
||||
sys.stderr.write('haproxy-stderr-relay: rotate failed: %s\n' % exc)
|
||||
|
||||
def poll_once(self):
|
||||
try:
|
||||
size = os.path.getsize(self.path)
|
||||
except OSError:
|
||||
if self.offset is None:
|
||||
self.offset = 0 # file not created yet: whatever appears is new
|
||||
return
|
||||
if self.offset is None:
|
||||
self.offset = size
|
||||
return
|
||||
if size < self.offset: # truncated by someone else
|
||||
self.offset = 0
|
||||
if size == self.offset:
|
||||
return
|
||||
with open(self.path, 'rb') as fh:
|
||||
fh.seek(self.offset)
|
||||
chunk = fh.read(1 << 20)
|
||||
self.offset += len(chunk)
|
||||
text = self.pending + chunk.decode('utf-8', 'replace')
|
||||
lines = text.split('\n')
|
||||
self.pending = lines.pop() # incomplete trailing line stays buffered
|
||||
if len(self.pending) > MAX_PENDING: # hostile / runaway unterminated line
|
||||
lines.append(self.pending)
|
||||
self.pending = ''
|
||||
for line in lines:
|
||||
line = self._clean(line)
|
||||
if line:
|
||||
self._emit(line)
|
||||
self._summaries()
|
||||
self._flush_all()
|
||||
|
||||
def tick(self):
|
||||
"""Idle-time housekeeping: emit the per-minute summaries even when no line arrives."""
|
||||
self._summaries()
|
||||
self._flush_all()
|
||||
|
||||
|
||||
def main():
|
||||
path = os.environ.get('HAPROXY_STDERR_LOG', '/var/log/haproxy/haproxy-stderr.log')
|
||||
relay = Relay(path, sys.stdout,
|
||||
_env_num('HAPROXY_STDERR_RELAY_BURST', 400),
|
||||
_env_num('HAPROXY_STDERR_RELAY_PER_MIN', 120),
|
||||
_env_num('HAPROXY_STDERR_LOG_MAX_BYTES', 20 * 1024 * 1024),
|
||||
_env_num('HAPROXY_STDERR_LOG_BACKUPS', 3),
|
||||
err=sys.stderr)
|
||||
poll = _env_num('HAPROXY_STDERR_RELAY_POLL', 0.5, float)
|
||||
sys.stdout.write('haproxy-stderr-relay: following %s\n' % path)
|
||||
sys.stdout.flush()
|
||||
ticks = 0
|
||||
while True:
|
||||
try:
|
||||
relay.poll_once()
|
||||
relay.tick()
|
||||
ticks += 1
|
||||
if ticks % 120 == 0:
|
||||
relay.rotate_if_needed()
|
||||
except Exception as exc: # never die: the supervisor loop restarts us anyway
|
||||
sys.stderr.write('haproxy-stderr-relay: %r\n' % exc)
|
||||
time.sleep(poll)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
# In-container haproxy supervisor (started in the background by start-up.sh).
|
||||
#
|
||||
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
|
||||
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
|
||||
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
|
||||
# policy never fires, and haproxy is down until the external host watchdog
|
||||
# full-restarts the whole container minutes later (dropping every connection).
|
||||
# This loop revives haproxy in place within one interval.
|
||||
#
|
||||
# The liveness probe is `pidof` (a few ms), NOT python: ensure_haproxy.py imports
|
||||
# the 4000-line manager (Flask, Jinja, ...) which costs ~0.5 s of CPU per run and
|
||||
# was affordable at 15 s but not at 2 s. python only runs when haproxy is gone.
|
||||
# 2026-11 evidence: every watchdog abort took the whole edge down for 15-52 s,
|
||||
# most of it detection latency + start; the default is now 2 s (was 15).
|
||||
#
|
||||
# Two backoffs:
|
||||
# * a restart attempt that FAILS (config HAProxy rejects): the loop doubles its
|
||||
# delay up to HAPROXY_SUPERVISOR_MAX_BACKOFF (default 30 s) instead of
|
||||
# re-rendering/re-validating the same broken config every 2 s;
|
||||
# * haproxy that starts fine but DIES again within HAPROXY_SUPERVISOR_QUICK_DEATH
|
||||
# seconds (default 10) of that start (a crash loop): extra wait before the
|
||||
# next restart, stepping through HAPROXY_SUPERVISOR_BACKOFF_STEPS (default
|
||||
# "2 5 10 30", each capped at MAX_BACKOFF). A run that survives
|
||||
# HAPROXY_SUPERVISOR_STABLE seconds (default 60) resets it. The first death
|
||||
# after a long run is still revived within one interval.
|
||||
#
|
||||
# All knobs are validated as positive INTEGERS (seconds); anything else (e.g. 0.5,
|
||||
# "abc") falls back to the default instead of killing this subshell under set -e.
|
||||
set -eo pipefail
|
||||
|
||||
_posint() { # _posint VALUE DEFAULT
|
||||
case "$1" in
|
||||
''|*[!0-9]*) echo "$2" ;;
|
||||
*) if [ "$1" -ge 1 ]; then echo "$1"; else echo "$2"; fi ;;
|
||||
esac
|
||||
}
|
||||
|
||||
INTERVAL="$(_posint "${HAPROXY_SUPERVISOR_INTERVAL:-}" 2)"
|
||||
MAX_BACKOFF="$(_posint "${HAPROXY_SUPERVISOR_MAX_BACKOFF:-}" 30)"
|
||||
QUICK_DEATH="$(_posint "${HAPROXY_SUPERVISOR_QUICK_DEATH:-}" 10)"
|
||||
STABLE="$(_posint "${HAPROXY_SUPERVISOR_STABLE:-}" 60)"
|
||||
STEPS=""
|
||||
for s in ${HAPROXY_SUPERVISOR_BACKOFF_STEPS:-2 5 10 30}; do
|
||||
v="$(_posint "$s" 0)"
|
||||
[ "$v" -ge 1 ] && STEPS="${STEPS} ${v}"
|
||||
done
|
||||
[ -n "${STEPS}" ] || STEPS=" 2 5 10 30"
|
||||
|
||||
# step N (1-based, past the end = last), capped at MAX_BACKOFF
|
||||
_step() {
|
||||
local n="$1" i=0 v=0
|
||||
for s in ${STEPS}; do
|
||||
i=$((i + 1)); v="$s"
|
||||
[ "${i}" -ge "${n}" ] && break
|
||||
done
|
||||
[ "${v}" -gt "${MAX_BACKOFF}" ] && v="${MAX_BACKOFF}"
|
||||
echo "${v}"
|
||||
}
|
||||
|
||||
delay="${INTERVAL}"
|
||||
last_start=-1 # SECONDS at our last successful (re)start; -1 = none yet
|
||||
quick=0 # consecutive deaths within QUICK_DEATH of a start
|
||||
while true; do
|
||||
sleep "${delay}"
|
||||
if pidof -q haproxy; then
|
||||
delay="${INTERVAL}"
|
||||
if [ "${last_start}" -ge 0 ] && [ $((SECONDS - last_start)) -ge "${STABLE}" ]; then
|
||||
quick=0; last_start=-1
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "${last_start}" -ge 0 ]; then
|
||||
up=$((SECONDS - last_start))
|
||||
if [ "${up}" -lt "${QUICK_DEATH}" ]; then
|
||||
quick=$((quick + 1))
|
||||
wait_s="$(_step "${quick}")"
|
||||
echo "haproxy-supervisor: haproxy died ${up}s after start (quick death #${quick}); waiting ${wait_s}s before restart" >&2
|
||||
sleep "${wait_s}"
|
||||
pidof -q haproxy && continue
|
||||
elif [ "${up}" -ge "${STABLE}" ]; then
|
||||
quick=0
|
||||
fi
|
||||
fi
|
||||
if python /haproxy/scripts/ensure_haproxy.py 2>&1; then
|
||||
delay="${INTERVAL}"
|
||||
last_start="${SECONDS}"
|
||||
else
|
||||
delay=$(( delay * 2 ))
|
||||
[ "${delay}" -gt "${MAX_BACKOFF}" ] && delay="${MAX_BACKOFF}"
|
||||
fi
|
||||
done
|
||||
@@ -20,13 +20,13 @@
|
||||
# against.
|
||||
#
|
||||
# Usage: monitor-attacks.sh [live]
|
||||
# Env: LOG_FILE=<path> access log to read (default /var/log/haproxy.log)
|
||||
# Env: LOG_FILE=<path> access log to read (default /var/log/haproxy/access.log)
|
||||
# LOG_LINES=<n> how many trailing log lines to scan (default 500)
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
LOG_FILE="${LOG_FILE:-/var/log/haproxy.log}"
|
||||
LOG_FILE="${LOG_FILE:-/var/log/haproxy/access.log}"
|
||||
LOG_LINES="${LOG_LINES:-500}"
|
||||
|
||||
# --- Section 1: current rates (real stick-table data) -----------------------
|
||||
@@ -47,12 +47,12 @@ show_recent_blocks() {
|
||||
syslog on the DOCKER HOST, and the file lives on the host, not in here.
|
||||
Read it from the host instead:
|
||||
|
||||
grep -aE ' (PT|PR)--' /var/log/haproxy.log | tail -20 # tarpit / deny
|
||||
grep -aE ' (429|403) ' /var/log/haproxy.log | tail -20 # rate-limited / blocked
|
||||
grep -a 'cip=<IP>' /var/log/haproxy.log | tail -50 # one client IP
|
||||
grep -a 'id=<uuid>' /var/log/haproxy.log # one request reference
|
||||
grep -aE ' (PT|PR)--' /var/log/haproxy/access.log | tail -20 # tarpit / deny
|
||||
grep -aE ' (429|403) ' /var/log/haproxy/access.log | tail -20 # rate-limited / blocked
|
||||
grep -a 'cip=<IP>' /var/log/haproxy/access.log | tail -50 # one client IP
|
||||
grep -a 'id=<uuid>' /var/log/haproxy/access.log # one request reference
|
||||
# (the UUID on the block page)
|
||||
tail -f /var/log/haproxy.log | grep -aE ' (PT|PR)--' # live
|
||||
tail -f /var/log/haproxy/access.log | grep -aE ' (PT|PR)--' # live
|
||||
|
||||
Or point this script at a copy: LOG_FILE=/path/to/haproxy.log $0
|
||||
MSG
|
||||
|
||||
Executable
+296
@@ -0,0 +1,296 @@
|
||||
#!/bin/bash
|
||||
## pkg-cache-repos.sh — BUILD-TIME ONLY, OPT-IN. Points dnf/apt at the internal
|
||||
## package cache (nginx proxy_cache, CT130 "pkg-cache" on the Proxmox cluster)
|
||||
## when PKG_CACHE is set; without it the repo files are left alone and builds
|
||||
## use the upstream repositories as before.
|
||||
##
|
||||
## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh rpm # AlmaLinux/EPEL/Remi/NodeSource
|
||||
## PKG_CACHE=https://192.168.1.40 pkg-cache-repos.sh apt # Ubuntu, Debian, LiteSpeed
|
||||
## pkg-cache-repos.sh release-install epel [remi] # GPG-verified release RPMs (EL9/EL10)
|
||||
## pkg-cache-repos.sh nodesource MAJOR # NodeSource repo file + pinned key
|
||||
## eval "$(pkg-cache-repos.sh go-env)" # GOPROXY via the cache (go.sum/sumdb verified)
|
||||
## pkg-cache-repos.sh restore # undo everything above
|
||||
##
|
||||
## CANONICAL copy: cloud-apache-container/scripts/pkg-cache-repos.sh (docs/pkg-cache.md
|
||||
## there). Other repos carry byte-identical copies - change it there first.
|
||||
##
|
||||
## Every RUN that rewrites must `restore` before it ends: it puts rewritten files
|
||||
## back, deletes files this script created, deletes any NEW repo/apt file that
|
||||
## still references the cache, then FAILS if any reference is left — so no shipped
|
||||
## layer carries the cache's LAN address (images run on hosts that can't reach it).
|
||||
##
|
||||
## Why: on 2026-10-21 (PHP security release) builds failed on Remi mirrors lagging
|
||||
## hours behind origin, Remi's origin timing out over HTTPS, and an EPEL mirror
|
||||
## with a broken certificate. The cache fetches from the ORIGINS over HTTPS (with
|
||||
## origin/mirror fallbacks and stale-on-error), keeps metadata ~3 minutes and
|
||||
## immutable .rpm/.deb files long-term.
|
||||
##
|
||||
## Integrity (security review 2026-10-21):
|
||||
## * PKG_CACHE must be https://. The cache's certificate is issued by the
|
||||
## name-constrained "pkg-cache internal CA" (keys/pkg-cache/ca.crt, valid only
|
||||
## for 192.168.1.40 / pkg-cache.*). That CA is trusted ONLY by the rewritten
|
||||
## repos (dnf sslcacert= bundle, apt per-host CaInfo) and only during the RUN —
|
||||
## never added to the system trust store.
|
||||
## * gpgcheck=1 is forced on every rewritten repo; repo_gpgcheck=1 on Remi and
|
||||
## AlmaLinux, which sign repomd.xml (measured: repomd.xml.asc 200). EPEL
|
||||
## publishes no repomd signature (404), so its metadata relies on HTTPS end
|
||||
## to end (client->cache->dl.fedoraproject.org/mirrors, all verified).
|
||||
## * release-install downloads epel-release / remi-release (from the cache, or
|
||||
## from the HTTPS origin without PKG_CACHE), checks the package name, imports
|
||||
## the signing keys committed in keys/pkg-cache/ and requires
|
||||
## `rpm -K` "signatures OK" before installing with localpkg_gpgcheck=1.
|
||||
## * The php.net version gates (EXPECT_PHP_MIN / EXPECT_PHP_VERSION) still fail
|
||||
## a build that gets old PHP (replay of old-but-signed metadata).
|
||||
##
|
||||
## rpm: each rewritten baseurl keeps the upstream origin as a SECOND url, so if
|
||||
## the cache dies mid-build dnf fails over to the origin by itself.
|
||||
## apt: apt has no per-source failover; CI only sets PKG_CACHE after probing the
|
||||
## cache (with the CA), and a build without PKG_CACHE uses upstream sources.
|
||||
set -euo pipefail
|
||||
|
||||
MODE="${1:?usage: pkg-cache-repos.sh rpm|apt|release-install NAME...|nodesource MAJOR|go-env|restore}"
|
||||
shift || true
|
||||
STATE=/var/lib/pkg-cache-repos # saved originals + manifest + created files
|
||||
SHARE="${PKG_CACHE_SHARE:-/usr/local/share/pkg-cache}" # ca.crt + RPM-GPG-KEY-*
|
||||
C="${PKG_CACHE:-}"
|
||||
C="${C%/}"
|
||||
|
||||
die() { echo "FATAL(pkg-cache-repos): $*" >&2; exit 1; }
|
||||
|
||||
if [ -n "$C" ]; then
|
||||
case "$C" in
|
||||
https://*) ;;
|
||||
*) die "PKG_CACHE must be an https:// URL (plain http is refused), got '$C'" ;;
|
||||
esac
|
||||
[[ "$C" =~ ^https://[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._/-]*)?$ ]] || die "PKG_CACHE has unexpected characters: '$C'"
|
||||
HOSTPORT="${C#https://}"; HOSTPORT="${HOSTPORT%%/*}"
|
||||
[ -s "$SHARE/ca.crt" ] || die "PKG_CACHE set but $SHARE/ca.crt (pkg-cache CA) is missing"
|
||||
fi
|
||||
|
||||
# System CA bundle (EL vs Debian/Ubuntu path); the pkg-cache CA is appended to a
|
||||
# copy in the state dir, never to the system store.
|
||||
sys_bundle() {
|
||||
local b
|
||||
for b in /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; do
|
||||
[ -s "$b" ] && { echo "$b"; return 0; }
|
||||
done
|
||||
die "no system CA bundle found"
|
||||
}
|
||||
make_bundle() { cat "$(sys_bundle)" "$SHARE/ca.crt" > "$STATE/ca-bundle.pem"; }
|
||||
|
||||
# Files that may hold repo configuration; restore re-checks all of them.
|
||||
repo_files() {
|
||||
find /etc/yum.repos.d /etc/dnf /etc/apt -type f 2>/dev/null | sort || true
|
||||
}
|
||||
|
||||
start_session() {
|
||||
[ -d "$STATE" ] && return 0
|
||||
mkdir -p "$STATE/orig"
|
||||
repo_files > "$STATE/manifest" # what existed before this RUN touched anything
|
||||
: > "$STATE/created"
|
||||
printf '%s\n' "$HOSTPORT" > "$STATE/host"
|
||||
}
|
||||
save() { # keep the first-seen (pre-rewrite) copy
|
||||
[ -e "$STATE/orig$1" ] && return 0
|
||||
mkdir -p "$STATE/orig$(dirname "$1")"
|
||||
cp -p "$1" "$STATE/orig$1"
|
||||
}
|
||||
created() { printf '%s\n' "$1" >> "$STATE/created"; }
|
||||
|
||||
if [ "$MODE" = restore ]; then
|
||||
[ -d "$STATE" ] || exit 0
|
||||
host=$(cat "$STATE/host")
|
||||
# mapfile from a process substitution, not `cmd | while read`: no pipeline,
|
||||
# so nothing for pipefail to misreport (scripts/tests/lsphp-info-probe.test.sh).
|
||||
mapfile -t saved < <(cd "$STATE/orig" && find . -type f)
|
||||
for rel in "${saved[@]}"; do
|
||||
cat "$STATE/orig/$rel" > "/${rel#./}"
|
||||
done
|
||||
while read -r f; do [ -n "$f" ] && rm -f -- "$f"; done < "$STATE/created"
|
||||
# A repo/apt file that appeared during this RUN and still names the cache
|
||||
# (e.g. written by a package from the rewritten repos) is removed too.
|
||||
mapfile -t current < <(repo_files)
|
||||
for f in "${current[@]}"; do
|
||||
if ! grep -qxF -- "$f" "$STATE/manifest" && grep -qF -- "$host" "$f"; then
|
||||
echo "pkg-cache-repos: removing new file that references the cache: $f"
|
||||
rm -f -- "$f"
|
||||
fi
|
||||
done
|
||||
left=$(repo_files | xargs -r grep -lF -- "$host" 2>/dev/null || true)
|
||||
rm -rf "$STATE"
|
||||
[ -z "$left" ] || die "cache address $host still present after restore in: $left"
|
||||
echo "pkg-cache-repos: original repository files restored (no reference to $host left)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = release-install ]; then
|
||||
[ $# -gt 0 ] || die "release-install needs: epel and/or remi"
|
||||
tmp=$(mktemp -d)
|
||||
curl_opts=(-fsS --proto '=https' --retry 5 --retry-all-errors --max-time 120)
|
||||
if [ -n "$C" ]; then
|
||||
start_session
|
||||
make_bundle
|
||||
curl_opts+=(--cacert "$STATE/ca-bundle.pem")
|
||||
fi
|
||||
files=()
|
||||
el=$(rpm -E '%{?rhel}' 2>/dev/null || true)
|
||||
case "$el" in 9|10) ;; *) die "release-install supports EL9/EL10 only (rpm -E %rhel = '$el')" ;; esac
|
||||
for name in "$@"; do
|
||||
case "$name" in
|
||||
epel) pkg=epel-release key=RPM-GPG-KEY-EPEL-$el
|
||||
path=/epel/epel-release-latest-$el.noarch.rpm
|
||||
origin=https://dl.fedoraproject.org/pub/epel/epel-release-latest-$el.noarch.rpm ;;
|
||||
remi) pkg=remi-release key=RPM-GPG-KEY-remi2024
|
||||
path=/remi/enterprise/remi-release-$el.rpm
|
||||
origin=https://rpms.remirepo.net/enterprise/remi-release-$el.rpm ;;
|
||||
*) die "unknown release package '$name'" ;;
|
||||
esac
|
||||
url="$origin"; [ -n "$C" ] && url="$C$path"
|
||||
f="$tmp/$pkg.rpm"
|
||||
curl "${curl_opts[@]}" -o "$f" "$url" || die "download failed: $url"
|
||||
rpm --import "$SHARE/$key" || die "cannot import $SHARE/$key"
|
||||
got=$(rpm -qp --nosignature --qf '%{NAME}' "$f" 2>/dev/null || true)
|
||||
[ "$got" = "$pkg" ] || die "$url is '$got', expected $pkg"
|
||||
chk=$(rpm -K "$f" 2>&1 || true)
|
||||
[[ "$chk" == *": digests signatures OK" ]] || die "signature check failed for $url: $chk"
|
||||
echo "pkg-cache-repos: $pkg verified ($chk) from $url"
|
||||
files+=("$f")
|
||||
done
|
||||
dnf install -y --setopt=localpkg_gpgcheck=1 "${files[@]}"
|
||||
rm -rf "$tmp"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -z "$C" ] && [ "$MODE" != nodesource ] && [ "$MODE" != go-env ]; then
|
||||
echo "pkg-cache-repos($MODE): PKG_CACHE not set - using upstream repositories"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rewrite_rpm() {
|
||||
local f="$1"
|
||||
grep -q -E '^#? ?baseurl=https?://(repo\.almalinux\.org|download\.example|rpms\.remirepo\.net|rpm\.nodesource\.com)/' "$f" || return 0
|
||||
save "$f"
|
||||
# Per [section]: a (commented or live) baseurl on a known origin becomes
|
||||
# "baseurl=<cache>/<prefix>/<path> <origin>/<path>"; that section's
|
||||
# mirrorlist/metalink is commented out (dnf prefers them over baseurl) and it
|
||||
# gets sslcacert=<system bundle + pkg-cache CA>, gpgcheck=1 and, for repos
|
||||
# that sign repomd.xml, repo_gpgcheck=1. Other sections are untouched.
|
||||
awk -v C="$C" -v BUNDLE="$STATE/ca-bundle.pem" '
|
||||
function flush( i) {
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (hit && buf[i] ~ /^(mirrorlist|metalink)=/) buf[i] = "#" buf[i]
|
||||
if (hit && buf[i] ~ /^(sslcacert|gpgcheck|repo_gpgcheck)[ \t]*=/) continue
|
||||
print buf[i]
|
||||
}
|
||||
if (hit) {
|
||||
print "sslcacert=" BUNDLE
|
||||
print "gpgcheck=1"
|
||||
if (signed) print "repo_gpgcheck=1"
|
||||
}
|
||||
n = 0; hit = 0; signed = 0
|
||||
}
|
||||
function rw(line, re, prefix, origin, sig, p) {
|
||||
if (match(line, re)) {
|
||||
p = substr(line, RSTART + RLENGTH); sub(/[ \t].*$/, "", p)
|
||||
hit = 1; if (sig) signed = 1
|
||||
return "baseurl=" C "/" prefix "/" p " " origin "/" p
|
||||
}
|
||||
return line
|
||||
}
|
||||
/^\[/ { flush() }
|
||||
{
|
||||
l = $0
|
||||
if (l ~ /^#? ?baseurl=/) {
|
||||
l = rw(l, "^#? ?baseurl=https?://repo\\.almalinux\\.org/almalinux/", "almalinux", "https://repo.almalinux.org/almalinux", 1)
|
||||
l = rw(l, "^#? ?baseurl=https?://download\\.example/pub/epel/", "epel", "https://dl.fedoraproject.org/pub/epel", 0)
|
||||
l = rw(l, "^#? ?baseurl=https?://rpms\\.remirepo\\.net/", "remi", "https://rpms.remirepo.net", 1)
|
||||
# NodeSource signs its RPMs but not repomd.xml (404): gpgcheck only.
|
||||
l = rw(l, "^#? ?baseurl=https?://rpm\\.nodesource\\.com/", "nodesource", "https://rpm.nodesource.com", 0)
|
||||
}
|
||||
buf[++n] = l
|
||||
}
|
||||
END { flush() }
|
||||
' "$f" > "$f.pkgcache" && cat "$f.pkgcache" > "$f" && rm -f "$f.pkgcache"
|
||||
}
|
||||
|
||||
rewrite_apt() {
|
||||
local f="$1"
|
||||
grep -q -E 'https?://([a-z]{2}\.)?(archive|security)\.ubuntu\.com/|https?://rpms\.litespeedtech\.com/|https?://(deb|security)\.debian\.org/' "$f" || return 0
|
||||
save "$f"
|
||||
# Debian: debian-security first, and "debian" only as a whole path segment,
|
||||
# so .../debian-security is never rewritten as /debian/.
|
||||
sed -i -E \
|
||||
-e "s%https?://(archive|[a-z]{2}\.archive)\.ubuntu\.com/ubuntu/?%$C/ubuntu/%g" \
|
||||
-e "s%https?://security\.ubuntu\.com/ubuntu/?%$C/ubuntu-security/%g" \
|
||||
-e "s%https?://rpms\.litespeedtech\.com/%$C/litespeed/%g" \
|
||||
-e "s%https?://(deb|security)\.debian\.org/debian-security/?([[:space:]]|\$)%$C/debian-security/\\2%g" \
|
||||
-e "s%https?://security\.debian\.org/?([[:space:]]|\$)%$C/debian-security/\\1%g" \
|
||||
-e "s%https?://deb\.debian\.org/debian/?([[:space:]]|\$)%$C/debian/\\1%g" \
|
||||
"$f"
|
||||
}
|
||||
|
||||
if [ "$MODE" = go-env ]; then
|
||||
# Prints shell exports for `eval`. Without PKG_CACHE: nothing (Go defaults).
|
||||
# GOPROXY "a|b" falls through on ANY error, so a dead cache costs one
|
||||
# failed request, not the build. GOSUMDB stays the default sum.golang.org,
|
||||
# reached directly: modules are still verified against go.sum + the sumdb.
|
||||
[ -n "$C" ] || { echo "pkg-cache-repos(go-env): PKG_CACHE not set - default GOPROXY" >&2; exit 0; }
|
||||
start_session
|
||||
make_bundle
|
||||
printf "export GOPROXY='%s/goproxy|https://proxy.golang.org|direct' SSL_CERT_FILE='%s'\n" "$C" "$STATE/ca-bundle.pem"
|
||||
echo "pkg-cache-repos(go-env): GOPROXY via $C/goproxy (fallback proxy.golang.org)" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$MODE" = nodesource ]; then
|
||||
# Replaces `curl https://rpm.nodesource.com/setup_N.x | bash` (unverified
|
||||
# remote script): writes the same nodesource-nodejs repo (origin URL, so the
|
||||
# shipped image keeps a normal repo file) with gpgcheck=1 and the signing key
|
||||
# committed in keys/pkg-cache/; with PKG_CACHE it is then rewritten like any
|
||||
# other repo and `restore` puts the origin version back.
|
||||
major="${1:?usage: pkg-cache-repos.sh nodesource MAJOR}"
|
||||
[[ "$major" =~ ^[0-9]{1,3}$ ]] || die "bad Node major '$major'"
|
||||
key="$SHARE/RPM-GPG-KEY-nodesource"
|
||||
[ -s "$key" ] || die "missing $key"
|
||||
rpm --import "$key" || die "cannot import $key"
|
||||
rm -f /etc/yum.repos.d/nodesource*.repo
|
||||
f=/etc/yum.repos.d/nodesource-nodejs.repo
|
||||
printf '%s\n' "[nodesource-nodejs]" \
|
||||
"name=Node.js Packages for Linux RPM based distros - \$basearch" \
|
||||
"baseurl=https://rpm.nodesource.com/pub_${major}.x/nodistro/nodejs/\$basearch" \
|
||||
"priority=9" "enabled=1" "gpgcheck=1" \
|
||||
"gpgkey=file://$key" "module_hotfixes=1" > "$f"
|
||||
echo "pkg-cache-repos: wrote $f (Node ${major}.x, key $(basename "$key"))"
|
||||
if [ -n "$C" ]; then
|
||||
start_session
|
||||
make_bundle
|
||||
rewrite_rpm "$f"
|
||||
grep -h '^baseurl=' "$f" | sed 's/^/ /'
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$MODE" in
|
||||
rpm)
|
||||
start_session
|
||||
make_bundle
|
||||
for f in /etc/yum.repos.d/*.repo; do [ -f "$f" ] && rewrite_rpm "$f"; done
|
||||
echo "pkg-cache-repos(rpm): repos now via $C:"
|
||||
grep -h '^baseurl=' /etc/yum.repos.d/*.repo | grep -F "$C/" | sort -u | sed 's/^/ /'
|
||||
;;
|
||||
apt)
|
||||
start_session
|
||||
conf=/etc/apt/apt.conf.d/99pkg-cache
|
||||
# Trust the pkg-cache CA for the cache host ONLY (apt per-host option).
|
||||
printf 'Acquire::https::%s::CaInfo "%s";\n' "${HOSTPORT%%:*}" "$SHARE/ca.crt" > "$conf"
|
||||
created "$conf"
|
||||
for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do
|
||||
[ -f "$f" ] && rewrite_apt "$f"
|
||||
done
|
||||
echo "pkg-cache-repos(apt): sources now via $C:"
|
||||
# (|| true: Debian 13 images have no /etc/apt/sources.list; grep's exit 2
|
||||
# for the missing file must not fail the build under pipefail)
|
||||
{ grep -h -E '^(deb|URIs:)' /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null || true; } | grep -F "$C/" | sort -u | sed 's/^/ /' || true
|
||||
;;
|
||||
*) die "unknown mode '$MODE'" ;;
|
||||
esac
|
||||
@@ -12,9 +12,9 @@
|
||||
#
|
||||
# The stick table also keeps NO history, so nothing in it can identify who was
|
||||
# tarpitted. Real enforcement events live in the access log ON THE DOCKER HOST
|
||||
# at /var/log/haproxy.log (it does not exist inside this container):
|
||||
# grep -aE ' (PT|PR)--' /var/log/haproxy.log | tail -20 # tarpit / deny
|
||||
# grep -aE ' (429|403) ' /var/log/haproxy.log | tail -20 # rate-limit / block
|
||||
# at /var/log/haproxy/access.log (inside this container since 2026.11.2):
|
||||
# grep -aE ' (PT|PR)--' /var/log/haproxy/access.log | tail -20 # tarpit / deny
|
||||
# grep -aE ' (429|403) ' /var/log/haproxy/access.log | tail -20 # rate-limit / block
|
||||
#
|
||||
# What IS knowable from the stick table — the current per-IP rates — is printed
|
||||
# by show-edge-ip-rates.sh, which this shim now runs.
|
||||
@@ -25,8 +25,8 @@ cat >&2 <<'NOTE'
|
||||
NOTE: show-tarpit-ips.sh is deprecated and cannot report tarpits.
|
||||
The HAProxy stick table stores no history and no gpc0/gpc1 counters, so
|
||||
the old "Scan Count"/"BLOCKED" columns were fabricated numbers.
|
||||
Actual tarpit/deny events are in /var/log/haproxy.log ON THE HOST:
|
||||
grep -aE ' (PT|PR)--' /var/log/haproxy.log | tail -20
|
||||
Actual tarpit/deny events are in /var/log/haproxy/access.log (in this container since 2026.11.2):
|
||||
grep -aE ' (PT|PR)--' /var/log/haproxy/access.log | tail -20
|
||||
Running show-edge-ip-rates.sh instead (current rates, real values):
|
||||
|
||||
NOTE
|
||||
|
||||
+60
-17
@@ -55,26 +55,69 @@ if [ ! -f /etc/haproxy/wpadmin_gate_exempt.list ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# HAProxy access log receiver (see scripts/haproxy-log-receiver.py and the
|
||||
# `log` line in hap_header.tpl). HAProxy sends datagrams to a UNIX socket
|
||||
# inside its chroot; this process writes them to a size-capped, rotating
|
||||
# /var/log/haproxy/access.log (WHP mounts the haproxy-logs volume there).
|
||||
# Started BEFORE init.py so haproxy's first lines land. Supervised by a
|
||||
# restart loop: if it dies it is back within 2s, and while it is down HAProxy
|
||||
# just drops log lines (non-blocking sends), it never blocks on them.
|
||||
# stdout is discarded on purpose -- it never prints log lines, and
|
||||
# `docker logs` must not carry a per-request firehose (WHP's AI monitor reads
|
||||
# it); its rare status/error lines go to stderr.
|
||||
mkdir -p /var/log/haproxy /var/lib/haproxy
|
||||
(
|
||||
while true; do
|
||||
python /haproxy/scripts/haproxy-log-receiver.py >/dev/null || true
|
||||
sleep 2
|
||||
done
|
||||
) &
|
||||
|
||||
cron &
|
||||
|
||||
# HAProxy stderr relay (see scripts/haproxy-stderr-relay.py and
|
||||
# haproxy_manager.spawn_haproxy_master()). HAProxy's master stderr (ALERTs,
|
||||
# "exited with code 134", the watchdog/ha_panic thread dump) is appended to
|
||||
# /var/log/haproxy/haproxy-stderr.log; this loop re-emits it on the container's
|
||||
# stderr so `docker logs` / the docker logging driver / Graylog get it. Started
|
||||
# BEFORE init.py so the first start's lines are followed. Restarted within 2 s
|
||||
# if it ever dies; it only reads a file, so it can never affect HAProxy.
|
||||
(
|
||||
while true; do
|
||||
python /haproxy/scripts/haproxy-stderr-relay.py || true
|
||||
sleep 2
|
||||
done
|
||||
) &
|
||||
|
||||
# gunicorn access logs (incident-prevention for the journald log driver).
|
||||
# gunicorn used to log every request to stdout (`--access-logfile -`), so every
|
||||
# :8080 default-page / blocked-IP hit and every :8000 API call became a docker
|
||||
# log line -> journal entry -> Graylog line, and could trip journald's rate
|
||||
# limit and suppress the relay's crash-dump lines. Now:
|
||||
# :8080 no access log at all;
|
||||
# :8000 access log to a size-bounded FILE in the haproxy-logs volume (mode
|
||||
# 0640), rotated by scripts/gunicorn-access-rotate.py (copy-truncate,
|
||||
# 20 MiB x 3). Set HAPROXY_MGR_API_ACCESS_LOG to relocate it.
|
||||
# Gunicorn ERROR logging (--error-logfile -) stays on stderr -> `docker logs`.
|
||||
API_ACCESS_LOG="${HAPROXY_MGR_API_ACCESS_LOG:-/var/log/haproxy/api-access.log}"
|
||||
export HAPROXY_MGR_API_ACCESS_LOG="${API_ACCESS_LOG}"
|
||||
mkdir -p "$(dirname "${API_ACCESS_LOG}")"
|
||||
[ -e "${API_ACCESS_LOG}" ] || : > "${API_ACCESS_LOG}"
|
||||
chmod 0640 "${API_ACCESS_LOG}" 2>/dev/null || true
|
||||
(
|
||||
while true; do
|
||||
python /haproxy/scripts/gunicorn-access-rotate.py || true
|
||||
sleep 2
|
||||
done
|
||||
) &
|
||||
|
||||
# Phase 1: container init
|
||||
python /haproxy/scripts/init.py
|
||||
|
||||
# Phase 1.5: in-container haproxy supervisor.
|
||||
# haproxy runs as a background child of PID 1 (gunicorn) with NOTHING watching
|
||||
# it after init. If the haproxy master dies mid-life (e.g. SIGABRT -> exit 134,
|
||||
# segfault), the container stays "up" (gunicorn is PID 1), Docker's --restart
|
||||
# policy never fires, and haproxy is down until the external host watchdog
|
||||
# full-restarts the whole container minutes later (dropping every connection).
|
||||
# This loop revives haproxy in place within one interval. ensure_haproxy.py is
|
||||
# idempotent — a cheap no-op whenever haproxy is already running.
|
||||
HAPROXY_SUPERVISOR_INTERVAL="${HAPROXY_SUPERVISOR_INTERVAL:-15}"
|
||||
(
|
||||
while true; do
|
||||
sleep "${HAPROXY_SUPERVISOR_INTERVAL}"
|
||||
python /haproxy/scripts/ensure_haproxy.py 2>&1 || true
|
||||
done
|
||||
) &
|
||||
# Phase 1.5: in-container haproxy supervisor (see scripts/haproxy-supervisor.sh:
|
||||
# revives a dead haproxy master within HAPROXY_SUPERVISOR_INTERVAL seconds, with
|
||||
# backoff on failed restarts and on crash loops; knobs validated as integers).
|
||||
/haproxy/scripts/haproxy-supervisor.sh &
|
||||
|
||||
# Phase 2: WSGI servers
|
||||
# Tunable via env: HAPROXY_MGR_API_WORKERS (default 2), HAPROXY_MGR_API_TIMEOUT
|
||||
@@ -114,7 +157,7 @@ gunicorn \
|
||||
--workers 1 --threads 4 --worker-class "${WORKER_CLASS}" \
|
||||
--max-requests "${MAX_REQ}" --max-requests-jitter "${MAX_REQ_JITTER}" \
|
||||
--timeout 30 \
|
||||
--access-logfile - --error-logfile - --log-level info \
|
||||
--error-logfile - --log-level info \
|
||||
--pythonpath /haproxy \
|
||||
'haproxy_manager:default_app' &
|
||||
|
||||
@@ -126,6 +169,6 @@ exec gunicorn \
|
||||
--workers "${API_WORKERS}" --threads 4 --worker-class "${WORKER_CLASS}" \
|
||||
--max-requests "${MAX_REQ}" --max-requests-jitter "${MAX_REQ_JITTER}" \
|
||||
--timeout "${API_TIMEOUT}" \
|
||||
--access-logfile - --error-logfile - --log-level info \
|
||||
--access-logfile "${API_ACCESS_LOG}" --error-logfile - --log-level info \
|
||||
--pythonpath /haproxy \
|
||||
'haproxy_manager:app'
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for scripts/haproxy-log-receiver.py (in-container access log).
|
||||
|
||||
Pins: one datagram == one line (PRI stripped, control chars neutralised),
|
||||
the size cap (live file + N gzip generations, oldest dropped), a write error
|
||||
(full disk / unwritable dir) drops lines without raising, and an end-to-end
|
||||
datagram path through a real UNIX socket.
|
||||
|
||||
Running
|
||||
-------
|
||||
python3 scripts/test-access-log-receiver.py
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import importlib.util
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
'haproxy_log_receiver', os.path.join(HERE, 'haproxy-log-receiver.py'))
|
||||
rx = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(rx)
|
||||
|
||||
|
||||
class CleanLine(unittest.TestCase):
|
||||
|
||||
def test_strips_pri_and_trailing_newline(self):
|
||||
self.assertEqual(rx.clean_line(b'<150>1 2026-11-08T00:00:00Z h haproxy 7 - - GET /\n'),
|
||||
'1 2026-11-08T00:00:00Z h haproxy 7 - - GET /')
|
||||
|
||||
def test_embedded_newlines_cannot_forge_lines(self):
|
||||
out = rx.clean_line(b'<150>1 x ua=evil\r\nFAKE 200 line\x1b[2J')
|
||||
self.assertNotIn('\n', out)
|
||||
self.assertNotIn('\r', out)
|
||||
self.assertNotIn('\x1b', out)
|
||||
self.assertIn('FAKE 200 line', out)
|
||||
|
||||
def test_invalid_utf8_does_not_raise(self):
|
||||
self.assertIn('�', rx.clean_line(b'<150>1 \xff\xfe'))
|
||||
|
||||
def test_non_pri_angle_bracket_kept(self):
|
||||
self.assertEqual(rx.clean_line(b'<html> x'), '<html> x')
|
||||
|
||||
|
||||
class Rotation(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix='hlr-test-')
|
||||
self.path = os.path.join(self.dir, 'access.log')
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.dir, ignore_errors=True)
|
||||
|
||||
def _files(self):
|
||||
return sorted(os.listdir(self.dir))
|
||||
|
||||
def test_size_cap_and_generations(self):
|
||||
max_bytes, backups = 4096, 3
|
||||
w = rx.RotatingWriter(self.path, max_bytes, backups)
|
||||
line = 'x' * 99 # 100 bytes with newline
|
||||
for i in range(1000): # ~100 KB -> ~24 rotations
|
||||
w.write(f'{i:05d}' + line[5:])
|
||||
w.close()
|
||||
files = self._files()
|
||||
self.assertEqual(files, ['access.log', 'access.log.1.gz',
|
||||
'access.log.2.gz', 'access.log.3.gz'], files)
|
||||
self.assertLessEqual(os.path.getsize(self.path), max_bytes)
|
||||
total = sum(os.path.getsize(os.path.join(self.dir, f)) for f in files)
|
||||
self.assertLessEqual(total, (1 + backups) * max_bytes)
|
||||
# newest generation holds the lines written just before the live file
|
||||
with gzip.open(os.path.join(self.dir, 'access.log.1.gz'), 'rt') as fh:
|
||||
gen1 = fh.read().splitlines()
|
||||
with open(self.path) as fh:
|
||||
live = fh.read().splitlines()
|
||||
self.assertEqual(int(gen1[-1][:5]) + 1, int(live[0][:5]))
|
||||
self.assertEqual(live[-1][:5], '00999')
|
||||
for l in gen1:
|
||||
self.assertEqual(len(l), 99)
|
||||
|
||||
def test_zero_backups_keeps_only_live_file(self):
|
||||
w = rx.RotatingWriter(self.path, 1024, 0)
|
||||
for _ in range(100):
|
||||
w.write('y' * 63)
|
||||
w.close()
|
||||
self.assertEqual(self._files(), ['access.log'])
|
||||
self.assertLessEqual(os.path.getsize(self.path), 1024)
|
||||
|
||||
def test_reopens_existing_file_and_counts_its_size(self):
|
||||
with open(self.path, 'w') as fh:
|
||||
fh.write('z' * 1000 + '\n')
|
||||
w = rx.RotatingWriter(self.path, 1500, 2)
|
||||
w.write('a' * 600) # 1001 + 601 > 1500 -> rotate first
|
||||
w.close()
|
||||
self.assertIn('access.log.1.gz', self._files())
|
||||
with open(self.path) as fh:
|
||||
self.assertEqual(fh.read(), 'a' * 600 + '\n')
|
||||
|
||||
def test_write_error_drops_and_never_raises(self):
|
||||
blocker = os.path.join(self.dir, 'not-a-dir')
|
||||
with open(blocker, 'w') as fh:
|
||||
fh.write('')
|
||||
msgs = []
|
||||
w = rx.RotatingWriter(os.path.join(blocker, 'access.log'), 4096, 2, msgs.append)
|
||||
for _ in range(50):
|
||||
w.write('line')
|
||||
w.flush()
|
||||
w.close()
|
||||
self.assertEqual(w.dropped, 50)
|
||||
self.assertEqual(len(msgs), 1, msgs) # rate-limited report
|
||||
|
||||
|
||||
class EndToEnd(unittest.TestCase):
|
||||
|
||||
def test_datagrams_become_lines(self):
|
||||
d = tempfile.mkdtemp(prefix='hlr-e2e-')
|
||||
try:
|
||||
sock_path = os.path.join(d, 'log.sock')
|
||||
log_path = os.path.join(d, 'logs', 'access.log')
|
||||
sock = rx.bind_socket(sock_path)
|
||||
self.assertEqual(os.stat(sock_path).st_mode & 0o777, 0o666)
|
||||
w = rx.RotatingWriter(log_path, 1 << 20, 2)
|
||||
stop = threading.Event()
|
||||
t = threading.Thread(target=rx.serve, args=(sock, w, stop), daemon=True)
|
||||
t.start()
|
||||
cl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
cl.setblocking(False)
|
||||
for i in range(20):
|
||||
msg = f'<150>1 t h haproxy 1 - - req {i} host=a.example cip=192.0.2.1'.encode()
|
||||
# The default net.unix.max_dgram_qlen is 10 per netns: a full
|
||||
# queue is EAGAIN (HAProxy drops the line); here, retry.
|
||||
for _ in range(100):
|
||||
try:
|
||||
cl.sendto(msg, sock_path)
|
||||
break
|
||||
except BlockingIOError:
|
||||
time.sleep(0.01)
|
||||
deadline = time.time() + 5
|
||||
lines = []
|
||||
while time.time() < deadline:
|
||||
time.sleep(0.2)
|
||||
w.flush()
|
||||
if os.path.exists(log_path):
|
||||
with open(log_path) as fh:
|
||||
lines = fh.read().splitlines()
|
||||
if len(lines) == 20:
|
||||
break
|
||||
stop.set()
|
||||
t.join(3)
|
||||
sock.close()
|
||||
w.close()
|
||||
self.assertEqual(len(lines), 20, lines)
|
||||
self.assertTrue(lines[0].startswith('1 t h haproxy 1 - - req 0 host=a.example'))
|
||||
finally:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
def test_sender_to_dead_receiver_fails_fast_nonblocking(self):
|
||||
"""What HAProxy sees when the receiver is down: an immediate error,
|
||||
never a block (HAProxy then just counts a dropped log line)."""
|
||||
d = tempfile.mkdtemp(prefix='hlr-dead-')
|
||||
try:
|
||||
cl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
cl.setblocking(False)
|
||||
start = time.monotonic()
|
||||
with self.assertRaises(OSError):
|
||||
cl.sendto(b'x', os.path.join(d, 'log.sock'))
|
||||
self.assertLess(time.monotonic() - start, 0.5)
|
||||
finally:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline tests for scripts/assemble-multiarch-index.sh (multi-arch index assembly).
|
||||
|
||||
The script is the publish-side half of the multi-arch pipeline: it receives the
|
||||
amd64 and (optional) arm64 digests that the build steps pushed BY DIGEST and
|
||||
turns them into the tag(s). A stub stands in for `docker buildx imagetools`
|
||||
(env IMAGETOOLS), records every `create` call and answers `inspect` from a
|
||||
table, so no registry or Docker is needed.
|
||||
|
||||
Held invariants:
|
||||
- arm64 failure / cancel / skip -> amd64-only create, exit 0 (tag still published)
|
||||
- a STALE arm64 digest is never attached unless this run's arm64 outcome is success
|
||||
- bad/missing amd64 digest, or amd64 that is not linux/amd64 -> exit 1, NO create
|
||||
- an arm64 digest that does not inspect as arm64 is dropped (amd64-only)
|
||||
- every tag x every repo is created; sources are <repo>@<digest> in that repo
|
||||
|
||||
Running: python3 scripts/test-assemble-multiarch-index.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SCRIPT = os.path.join(HERE, 'assemble-multiarch-index.sh')
|
||||
A = 'sha256:' + 'a' * 64
|
||||
B = 'sha256:' + 'b' * 64
|
||||
C = 'sha256:' + 'c' * 64
|
||||
|
||||
STUB = r'''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
a = sys.argv[1:]
|
||||
log = os.environ['STUB_LOG']
|
||||
table = json.loads(os.environ['STUB_ARCH'])
|
||||
if a[0] == 'inspect':
|
||||
ref = [x for x in a[1:] if '@sha256:' in x][0]
|
||||
arch = table.get(ref.split('@', 1)[1])
|
||||
if arch is None:
|
||||
sys.exit(1)
|
||||
print(json.dumps({'architecture': arch, 'os': 'linux'}))
|
||||
elif a[0] == 'create':
|
||||
open(log, 'a').write(json.dumps(a[1:]) + '\n')
|
||||
else:
|
||||
sys.exit(2)
|
||||
'''
|
||||
|
||||
|
||||
class AssembleTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.stub = os.path.join(self.tmp.name, 'stub')
|
||||
with open(self.stub, 'w') as f:
|
||||
f.write(STUB)
|
||||
os.chmod(self.stub, os.stat(self.stub).st_mode | stat.S_IEXEC)
|
||||
self.log = os.path.join(self.tmp.name, 'creates.log')
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def run_script(self, args, arch=None):
|
||||
env = dict(os.environ, IMAGETOOLS=self.stub, STUB_LOG=self.log,
|
||||
STUB_ARCH=json.dumps(arch if arch is not None else {A: 'amd64', B: 'arm64'}))
|
||||
p = subprocess.run(['bash', SCRIPT] + args, env=env, capture_output=True, text=True)
|
||||
creates = []
|
||||
if os.path.exists(self.log):
|
||||
creates = [json.loads(l) for l in open(self.log)]
|
||||
return p, creates
|
||||
|
||||
BASE = ['--repo', 'r.example/o/img', '--tag', 'latest']
|
||||
|
||||
def test_both_platforms(self):
|
||||
p, c = self.run_script(['--amd64', A, '--arm64', B, '--arm64-outcome', 'success'] + self.BASE)
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertEqual(c, [['-t', 'r.example/o/img:latest', 'r.example/o/img@' + A, 'r.example/o/img@' + B]])
|
||||
self.assertIn('platforms=linux/amd64,linux/arm64', p.stdout)
|
||||
|
||||
def test_arm64_failure_is_amd64_only_even_with_a_digest(self):
|
||||
# A stale digest (e.g. left over from an earlier run) must not be attached.
|
||||
for outcome in ('failure', 'cancelled', 'skipped', ''):
|
||||
if os.path.exists(self.log):
|
||||
os.remove(self.log)
|
||||
p, c = self.run_script(['--amd64', A, '--arm64', B, '--arm64-outcome', outcome] + self.BASE)
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertEqual(c, [['-t', 'r.example/o/img:latest', 'r.example/o/img@' + A]], outcome)
|
||||
self.assertIn('::warning::', p.stdout)
|
||||
self.assertIn('platforms=linux/amd64\n', p.stdout)
|
||||
|
||||
def test_no_arm64_args_is_amd64_only(self):
|
||||
p, c = self.run_script(['--amd64', A] + self.BASE)
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertEqual(c, [['-t', 'r.example/o/img:latest', 'r.example/o/img@' + A]])
|
||||
|
||||
def test_success_with_empty_or_malformed_arm64_digest(self):
|
||||
for bad in ('', 'sha256:short', 'latest'):
|
||||
if os.path.exists(self.log):
|
||||
os.remove(self.log)
|
||||
p, c = self.run_script(['--amd64', A, '--arm64', bad, '--arm64-outcome', 'success'] + self.BASE)
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertEqual(len(c[0]), 3, bad) # -t, ref, one source
|
||||
|
||||
def test_arm64_digest_that_is_not_arm64_is_dropped(self):
|
||||
p, c = self.run_script(['--amd64', A, '--arm64', C, '--arm64-outcome', 'success'] + self.BASE,
|
||||
arch={A: 'amd64', C: 'amd64'})
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertEqual(c, [['-t', 'r.example/o/img:latest', 'r.example/o/img@' + A]])
|
||||
|
||||
def test_arm64_equal_to_amd64_is_dropped(self):
|
||||
p, c = self.run_script(['--amd64', A, '--arm64', A, '--arm64-outcome', 'success'] + self.BASE)
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertEqual(len(c[0]), 3)
|
||||
|
||||
def test_amd64_problems_publish_nothing(self):
|
||||
for amd in ('', 'nope', B): # B inspects as arm64
|
||||
p, c = self.run_script(['--amd64', amd, '--arm64', B, '--arm64-outcome', 'success'] + self.BASE)
|
||||
self.assertNotEqual(p.returncode, 0, amd)
|
||||
self.assertEqual(c, [], amd)
|
||||
p, c = self.run_script(['--amd64', C, '--arm64-outcome', 'failure'] + self.BASE, arch={})
|
||||
self.assertNotEqual(p.returncode, 0) # not inspectable
|
||||
self.assertEqual(c, [])
|
||||
|
||||
def test_all_tags_in_all_repos(self):
|
||||
p, c = self.run_script(['--amd64', A, '--arm64', B, '--arm64-outcome', 'success',
|
||||
'--repo', 'gitea.example/o/img', '--repo', 'ghcr.io/o/img',
|
||||
'--tag', 'latest', '--tag', '1.2.3', '--tag', 'abc123'])
|
||||
self.assertEqual(p.returncode, 0, p.stderr)
|
||||
self.assertEqual(len(c), 2)
|
||||
self.assertEqual(c[0], ['-t', 'gitea.example/o/img:latest', '-t', 'gitea.example/o/img:1.2.3',
|
||||
'-t', 'gitea.example/o/img:abc123',
|
||||
'gitea.example/o/img@' + A, 'gitea.example/o/img@' + B])
|
||||
self.assertEqual(c[1][-2:], ['ghcr.io/o/img@' + A, 'ghcr.io/o/img@' + B])
|
||||
|
||||
def test_requires_repo_and_tag_and_rejects_unknown_args(self):
|
||||
for args in (['--amd64', A, '--tag', 't'], ['--amd64', A, '--repo', 'r'], ['--bogus', 'x']):
|
||||
p, c = self.run_script(args)
|
||||
self.assertNotEqual(p.returncode, 0)
|
||||
self.assertEqual(c, [])
|
||||
|
||||
|
||||
class WorkflowWiring(unittest.TestCase):
|
||||
"""The workflow must keep the properties the script relies on."""
|
||||
|
||||
def setUp(self):
|
||||
self.wf = open(os.path.join(HERE, '..', '.gitea', 'workflows', 'build-push-coraza.yaml')).read()
|
||||
|
||||
def test_arm64_step_is_non_fatal_and_digest_comes_from_step_output(self):
|
||||
self.assertIn('continue-on-error: true', self.wf)
|
||||
self.assertIn('steps.arm64.outputs.digest', self.wf)
|
||||
self.assertIn('steps.arm64.outcome', self.wf)
|
||||
self.assertIn('assemble-multiarch-index.sh', self.wf)
|
||||
|
||||
def test_both_builds_push_by_digest_untagged(self):
|
||||
self.assertEqual(self.wf.count('push-by-digest=true'), 2)
|
||||
self.assertNotIn('tags:', self.wf.replace('# tags:', ''))
|
||||
|
||||
def test_no_tag_is_pushed_by_a_build_step(self):
|
||||
# tags exist only via imagetools create (the assemble script)
|
||||
self.assertNotIn('ghcr.io/shadowdao/coraza-spoa:latest\n', self.wf.split('assemble-multiarch-index.sh')[0])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -75,6 +75,9 @@ sys.path.insert(0, MODULE_DIR)
|
||||
# The module opens /var/log/haproxy-manager.log at import time via
|
||||
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
|
||||
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
|
||||
# spawn_haproxy_master() appends the master's stderr here.
|
||||
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
|
||||
os.environ.setdefault('HAPROXY_START_WAIT', '3')
|
||||
_real_file_handler = logging.FileHandler
|
||||
logging.FileHandler = (
|
||||
lambda fn, *a, **kw: _real_file_handler(
|
||||
@@ -225,8 +228,9 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
# haproxy -c -f FILE -> reject FILE containing {BROKEN_TOKEN}
|
||||
# reject any unusable file in $TEST_CERTS_DIR
|
||||
cfg=""
|
||||
start=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in -f) cfg="$2"; shift ;; esac
|
||||
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
|
||||
shift
|
||||
done
|
||||
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
|
||||
@@ -250,6 +254,8 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
fi
|
||||
done
|
||||
fi
|
||||
# real `haproxy -W -db` prints this once the config is loaded
|
||||
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
|
||||
exit 0
|
||||
""")
|
||||
|
||||
|
||||
@@ -56,6 +56,10 @@ sys.path.insert(0, MODULE_DIR)
|
||||
# The module opens /var/log/haproxy-manager.log at import time via
|
||||
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
|
||||
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
|
||||
# spawn_haproxy_master() appends the master's stderr here (default is the
|
||||
# haproxy-logs volume path, not writable unprivileged).
|
||||
os.environ.setdefault('HAPROXY_STDERR_LOG', os.path.join(_LOG_DIR, 'haproxy-stderr.log'))
|
||||
os.environ.setdefault('HAPROXY_START_WAIT', '3')
|
||||
_real_file_handler = logging.FileHandler
|
||||
logging.FileHandler = (
|
||||
lambda fn, *a, **kw: _real_file_handler(
|
||||
@@ -80,14 +84,18 @@ FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
# haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0
|
||||
# haproxy -W -S ... -f FILE (start) -> same validation, then exit 0
|
||||
cfg=""
|
||||
start=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in -f) cfg="$2"; shift ;; esac
|
||||
case "$1" in -f) cfg="$2"; shift ;; -W) start=1 ;; esac
|
||||
shift
|
||||
done
|
||||
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
|
||||
echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2
|
||||
exit 1
|
||||
fi
|
||||
# A real `haproxy -W -db` prints this once the config is loaded;
|
||||
# spawn_haproxy_master() waits for it.
|
||||
[ "$start" = 1 ] && echo "[NOTICE] (1) : Loading success." >&2
|
||||
exit 0
|
||||
""")
|
||||
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression tests for the default splash (unrouted hosts answered pre-WAF).
|
||||
|
||||
Why this file exists
|
||||
--------------------
|
||||
Visiting a WHP server's own hostname (https://whpNN.<domain>/ on 80/443, not
|
||||
the panel port), its bare IP, or any name with no site behind it used to fall
|
||||
through to `default_backend default-backend` -- but only AFTER every
|
||||
http-request rule had run, Coraza's send-spoe-group included. `use_backend`
|
||||
does not short-circuit http-request processing. So the catch-all was WAF
|
||||
inspected with the server-wide rule set, and a browser on the server hostname
|
||||
also carries the control panel's cookies (cookies are not port-scoped), which
|
||||
produced the WAF block page for a host that has nothing behind it to protect.
|
||||
|
||||
The splash answers those requests with `http-request return` BEFORE the SPOE
|
||||
call. Its safety property is that it must NEVER answer a host that routes to
|
||||
a site. That is held by:
|
||||
- routed_hosts.list / routed_wildcards.list being written from the same
|
||||
rows, in the same generate_config() call, as the routing ACLs
|
||||
- the lists being port-stripped + lowercased (a superset of the raw-Host
|
||||
routing ACLs)
|
||||
- the lists being part of the rollback set
|
||||
and it must keep ACME http-01, the blocked-IP page and the suspended page.
|
||||
|
||||
Running
|
||||
-------
|
||||
python3 scripts/test-default-splash.py
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
MODULE_DIR = os.path.abspath(
|
||||
os.environ.get('HAPROXY_MANAGER_DIR',
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
|
||||
)
|
||||
os.chdir(MODULE_DIR)
|
||||
sys.path.insert(0, MODULE_DIR)
|
||||
|
||||
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
|
||||
_real_file_handler = logging.FileHandler
|
||||
logging.FileHandler = (
|
||||
lambda filename, *a, **kw: _real_file_handler(
|
||||
os.path.join(_LOG_DIR, os.path.basename(filename)), *a, **kw)
|
||||
)
|
||||
try:
|
||||
import haproxy_manager as hm # noqa: E402
|
||||
finally:
|
||||
logging.FileHandler = _real_file_handler
|
||||
logging.getLogger('haproxy_manager').setLevel(logging.CRITICAL)
|
||||
|
||||
SPLASH_FILE = os.path.join(MODULE_DIR, 'errors', 'default-splash.html')
|
||||
|
||||
|
||||
def render_listener(**kw):
|
||||
args = dict(crt_path='/etc/haproxy/certs', suspension_enabled=False,
|
||||
coraza_spoe_backend='127.0.0.1:9000', default_splash=True)
|
||||
args.update(kw)
|
||||
return hm.template_env.get_template('hap_listener.tpl').render(**args)
|
||||
|
||||
|
||||
def return_rules(cfg):
|
||||
return [l.strip() for l in cfg.splitlines()
|
||||
if l.strip().startswith('http-request return')
|
||||
and ('default-splash.html' in l or 'robots-unrouted.txt' in l)]
|
||||
|
||||
|
||||
class Template(unittest.TestCase):
|
||||
|
||||
def test_off_renders_nothing(self):
|
||||
cfg = render_listener(default_splash=False)
|
||||
self.assertNotIn('default-splash', cfg)
|
||||
self.assertNotIn('splash_routed_host', cfg)
|
||||
|
||||
def test_404_everywhere_robots_txt_200(self):
|
||||
"""Josh 2026-11-08: unrouted hosts are 404 on EVERY path (incl. "/")
|
||||
so search engines never index the server IP/hostname; /robots.txt is
|
||||
the one 200 and disallows everything."""
|
||||
rules = return_rules(render_listener())
|
||||
self.assertEqual(len(rules), 2, rules)
|
||||
robots, splash = rules
|
||||
self.assertTrue(robots.startswith('http-request return status 200 content-type "text/plain'))
|
||||
self.assertIn(' file /haproxy/errors/robots-unrouted.txt ', robots)
|
||||
self.assertTrue(robots.endswith(' splash_robots'))
|
||||
self.assertTrue(splash.startswith('http-request return status 404 content-type "text/html'))
|
||||
self.assertTrue(splash.endswith(' !splash_robots'))
|
||||
self.assertNotIn('splash_root', render_listener())
|
||||
self.assertRegex(render_listener(), r'acl splash_robots path /robots\.txt\n')
|
||||
|
||||
def test_noindex_and_no_store_headers_on_both(self):
|
||||
for rule in return_rules(render_listener()):
|
||||
self.assertIn('hdr x-robots-tag "noindex, nofollow, noarchive"', rule)
|
||||
self.assertIn('hdr cache-control "no-store"', rule)
|
||||
self.assertIn('hdr x-content-type-options "nosniff"', rule)
|
||||
|
||||
def test_static_file_not_lf_file(self):
|
||||
"""`file` (verbatim), never `lf-file`: nothing from the request may be
|
||||
expanded into the page (Host echo, unique-id, ...)."""
|
||||
for rule in return_rules(render_listener()):
|
||||
self.assertRegex(rule, r' file /haproxy/errors/(default-splash\.html|robots-unrouted\.txt) ')
|
||||
self.assertNotIn('lf-file', rule)
|
||||
|
||||
def test_conditions_keep_acme_blocked_and_suspended_out(self):
|
||||
for susp in (False, True):
|
||||
for rule in return_rules(render_listener(suspension_enabled=susp)):
|
||||
cond = rule.split(' if ', 1)[1].split()
|
||||
self.assertIn('!splash_routed_host', cond)
|
||||
self.assertIn('!splash_acme', cond)
|
||||
self.assertIn('!is_blocked_ip', cond)
|
||||
if susp:
|
||||
self.assertIn('!is_suspended_domain', cond)
|
||||
else:
|
||||
self.assertNotIn('!is_suspended_domain', cond)
|
||||
|
||||
def test_acme_acl_is_the_challenge_prefix(self):
|
||||
self.assertRegex(render_listener(),
|
||||
r'acl splash_acme path_beg /\.well-known/acme-challenge/\n')
|
||||
|
||||
def test_routed_acls_strip_port_and_lowercase(self):
|
||||
cfg = render_listener()
|
||||
self.assertIn('acl splash_routed_host req.hdr(host),field(1,:),lower '
|
||||
'-f /etc/haproxy/routed_hosts.list', cfg)
|
||||
self.assertIn('acl splash_routed_host req.hdr(host),field(1,:),lower '
|
||||
'-m end -f /etc/haproxy/routed_wildcards.list', cfg)
|
||||
|
||||
def test_splash_runs_before_waf_and_after_ip_block_and_suspension(self):
|
||||
for susp in (False, True):
|
||||
cfg = render_listener(suspension_enabled=susp)
|
||||
first_return = cfg.index('file /haproxy/errors/robots-unrouted.txt')
|
||||
self.assertLess(first_return, cfg.index('http-request send-spoe-group coraza'))
|
||||
self.assertLess(first_return, cfg.index('filter spoe engine coraza'))
|
||||
self.assertGreater(first_return, cfg.index('acl is_blocked_ip'))
|
||||
self.assertGreater(first_return, cfg.index('http-request track-sc0'))
|
||||
if susp:
|
||||
self.assertGreater(first_return, cfg.index('acl is_suspended_domain'))
|
||||
|
||||
|
||||
class Page(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
with open(SPLASH_FILE, 'rb') as fh:
|
||||
self.raw = fh.read()
|
||||
self.text = self.raw.decode('utf-8')
|
||||
|
||||
def test_small_enough_for_one_buffer(self):
|
||||
# http-request return payload must fit tune.bufsize (16384) with headers.
|
||||
self.assertLess(len(self.raw), 8192)
|
||||
|
||||
def test_no_links_scripts_or_external_resources(self):
|
||||
low = self.text.lower()
|
||||
for needle in ('href', 'src=', '<script', '<form', '<a ', 'http://', 'https://',
|
||||
'8443', '@import', 'url('):
|
||||
self.assertNotIn(needle, low, needle)
|
||||
|
||||
def test_no_version_or_template_expansions(self):
|
||||
self.assertNotRegex(self.text, r'20\d\d\.\d+\.\d+')
|
||||
self.assertNotIn('{{', self.text)
|
||||
self.assertNotIn('%[', self.text)
|
||||
|
||||
|
||||
class Robots(unittest.TestCase):
|
||||
|
||||
def test_robots_disallows_everything(self):
|
||||
with open(os.path.join(MODULE_DIR, 'errors', 'robots-unrouted.txt')) as fh:
|
||||
self.assertEqual(fh.read(), 'User-agent: *\nDisallow: /\n')
|
||||
|
||||
|
||||
class RoutedLists(unittest.TestCase):
|
||||
|
||||
def test_render_sorted_dedup_lower(self):
|
||||
body = hm.render_routed_list(['B.example.com', 'a.example.com', 'b.example.com ', ''])
|
||||
lines = [l for l in body.splitlines() if not l.startswith('#')]
|
||||
self.assertEqual(lines, ['a.example.com', 'b.example.com'])
|
||||
|
||||
def test_name_with_whitespace_is_dropped_not_split(self):
|
||||
body = hm.render_routed_list(['ok.example.com', 'bad\nevil.example.com', 'a b'])
|
||||
lines = [l for l in body.splitlines() if not l.startswith('#')]
|
||||
self.assertEqual(lines, ['ok.example.com'])
|
||||
|
||||
def test_lists_are_in_the_rollback_set(self):
|
||||
pairs = dict(hm._config_backup_pairs())
|
||||
self.assertIn(hm.ROUTED_HOSTS_PATH, pairs)
|
||||
self.assertIn(hm.ROUTED_WILDCARDS_PATH, pairs)
|
||||
|
||||
|
||||
class GenerateConfigWritesLists(unittest.TestCase):
|
||||
"""Run the real generate_config() with paths redirected and reload stubbed."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix='splash-test-')
|
||||
self.saved = {k: getattr(hm, k) for k in (
|
||||
'DB_FILE', 'HAPROXY_CONFIG_PATH', 'HAPROXY_BACKUP_PATH',
|
||||
'CLUSTER_SECRET_PATH', 'HAPROXY_SOCKET_PATH', 'SSL_CERTS_DIR',
|
||||
'BLOCKED_IPS_MAP_PATH', 'BLOCKED_IPS_MAP_BACKUP_PATH',
|
||||
'CORAZA_SPOE_CONFIG_PATH', 'CORAZA_SPOE_BACKUP_PATH',
|
||||
'ROUTED_HOSTS_PATH', 'ROUTED_HOSTS_BACKUP_PATH',
|
||||
'ROUTED_WILDCARDS_PATH', 'ROUTED_WILDCARDS_BACKUP_PATH',
|
||||
'reload_haproxy_safely', 'create_backup')}
|
||||
j = lambda n: os.path.join(self.tmp, n) # noqa: E731
|
||||
hm.DB_FILE = j('db.sqlite')
|
||||
hm.HAPROXY_CONFIG_PATH = j('haproxy.cfg')
|
||||
hm.HAPROXY_BACKUP_PATH = j('haproxy.cfg.backup')
|
||||
hm.CLUSTER_SECRET_PATH = j('cluster-secret')
|
||||
hm.HAPROXY_SOCKET_PATH = j('haproxy.sock')
|
||||
hm.SSL_CERTS_DIR = j('certs')
|
||||
hm.BLOCKED_IPS_MAP_PATH = j('blocked_ips.map')
|
||||
hm.BLOCKED_IPS_MAP_BACKUP_PATH = j('blocked_ips.map.backup')
|
||||
hm.CORAZA_SPOE_CONFIG_PATH = j('coraza-spoe.cfg')
|
||||
hm.CORAZA_SPOE_BACKUP_PATH = j('coraza-spoe.cfg.backup')
|
||||
hm.ROUTED_HOSTS_PATH = j('routed_hosts.list')
|
||||
hm.ROUTED_HOSTS_BACKUP_PATH = j('routed_hosts.list.backup')
|
||||
hm.ROUTED_WILDCARDS_PATH = j('routed_wildcards.list')
|
||||
hm.ROUTED_WILDCARDS_BACKUP_PATH = j('routed_wildcards.list.backup')
|
||||
hm.reload_haproxy_safely = lambda *a, **kw: (True, 'stubbed')
|
||||
hm.create_backup = lambda *a, **kw: (True, 'unavailable')
|
||||
self.env = {k: os.environ.pop(k, None) for k in (
|
||||
'HAPROXY_DEFAULT_SPLASH', 'HAPROXY_SUSPENSION_ENABLED',
|
||||
'HAPROXY_CORAZA_SPOE_BACKEND')}
|
||||
hm.init_db()
|
||||
with sqlite3.connect(hm.DB_FILE) as conn:
|
||||
cur = conn.cursor()
|
||||
|
||||
def add(domain, backend, wildcard=0):
|
||||
cur.execute('INSERT INTO domains (domain, ssl_enabled, ssl_cert_path, '
|
||||
'template_override, is_wildcard) VALUES (?, 1, NULL, NULL, ?)',
|
||||
(domain, wildcard))
|
||||
did = cur.lastrowid
|
||||
if backend:
|
||||
cur.execute('INSERT INTO backends (name, domain_id, settings) VALUES (?, ?, NULL)',
|
||||
(backend, did))
|
||||
cur.execute('INSERT INTO backend_servers (backend_id, server_name, '
|
||||
'server_address, server_port, server_options) VALUES (?, ?, ?, ?, ?)',
|
||||
(cur.lastrowid, 'web1', '10.0.0.10', 80, 'check'))
|
||||
add('Site.Example.com', 'site')
|
||||
add('*.wild.example.com', 'wild', wildcard=1)
|
||||
add('whp01.example.net', None) # server hostname: cert-only
|
||||
conn.commit()
|
||||
|
||||
def tearDown(self):
|
||||
for k, v in self.saved.items():
|
||||
setattr(hm, k, v)
|
||||
for k, v in self.env.items():
|
||||
os.environ.pop(k, None)
|
||||
if v is not None:
|
||||
os.environ[k] = v
|
||||
import shutil
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def _lines(self, path):
|
||||
with open(path) as fh:
|
||||
return [l for l in fh.read().splitlines() if l and not l.startswith('#')]
|
||||
|
||||
def test_lists_written_from_routed_rows_only(self):
|
||||
hm.generate_config()
|
||||
self.assertEqual(self._lines(hm.ROUTED_HOSTS_PATH), ['site.example.com'])
|
||||
self.assertEqual(self._lines(hm.ROUTED_WILDCARDS_PATH), ['.wild.example.com'])
|
||||
with open(hm.HAPROXY_CONFIG_PATH) as fh:
|
||||
cfg = fh.read()
|
||||
# every exact routing ACL name is in the list (superset property)
|
||||
for name in re.findall(r'acl \S+-acl hdr\(host\) -i (\S+)', cfg):
|
||||
self.assertIn(name.lower(), self._lines(hm.ROUTED_HOSTS_PATH))
|
||||
for base in re.findall(r'acl \S+-acl hdr_end\(host\) -i (\S+)', cfg):
|
||||
self.assertIn(base.lower(), self._lines(hm.ROUTED_WILDCARDS_PATH))
|
||||
self.assertIn('default-splash.html', cfg)
|
||||
|
||||
def test_default_log_target_is_the_in_container_socket(self):
|
||||
"""Access log goes to the receiver's UNIX socket (chroot-relative),
|
||||
never the old host bridge target 172.18.0.1:514."""
|
||||
os.environ.pop('HAPROXY_SYSLOG_TARGET', None)
|
||||
hm.generate_config()
|
||||
with open(hm.HAPROXY_CONFIG_PATH) as fh:
|
||||
cfg = fh.read()
|
||||
self.assertRegex(cfg, r'\n log /log\.sock len 2048 format rfc5424 local2 info\n')
|
||||
self.assertIn('chroot /var/lib/haproxy', cfg)
|
||||
log_lines = [l for l in cfg.splitlines() if l.strip().startswith('log ')
|
||||
and 'global' not in l]
|
||||
self.assertEqual(len(log_lines), 1, log_lines)
|
||||
self.assertNotIn('172.18.0.1', log_lines[0])
|
||||
|
||||
def test_log_target_env_override(self):
|
||||
os.environ['HAPROXY_SYSLOG_TARGET'] = '192.0.2.5:514'
|
||||
try:
|
||||
hm.generate_config()
|
||||
finally:
|
||||
os.environ.pop('HAPROXY_SYSLOG_TARGET', None)
|
||||
with open(hm.HAPROXY_CONFIG_PATH) as fh:
|
||||
self.assertIn('log 192.0.2.5:514 len 2048', fh.read())
|
||||
|
||||
def test_lists_written_even_with_splash_off(self):
|
||||
os.environ['HAPROXY_DEFAULT_SPLASH'] = 'off'
|
||||
hm.generate_config()
|
||||
self.assertTrue(os.path.exists(hm.ROUTED_HOSTS_PATH))
|
||||
self.assertTrue(os.path.exists(hm.ROUTED_WILDCARDS_PATH))
|
||||
with open(hm.HAPROXY_CONFIG_PATH) as fh:
|
||||
self.assertNotIn('default-splash.html', fh.read())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests: gunicorn access logging must not flood the container's stdout.
|
||||
|
||||
Context: with the journald docker log driver every stdout line is a journal
|
||||
entry (default rate limit 10000 msgs/30 s per service could suppress crash-dump
|
||||
lines) and, via Graylog, a stored message. start-up.sh therefore gives :8080 no
|
||||
access log and sends :8000's to a bounded FILE; gunicorn errors stay on stderr.
|
||||
|
||||
Pins (static, from start-up.sh):
|
||||
* no gunicorn invocation uses `--access-logfile -`;
|
||||
* the :8080 invocation has no --access-logfile at all;
|
||||
* the :8000 invocation logs access to ${API_ACCESS_LOG}, errors to `-`.
|
||||
Pins (live, REAL gunicorn, same flag shapes):
|
||||
* a burst of default-page requests (no access log) writes ZERO lines to
|
||||
stdout and none to stderr;
|
||||
* a burst of API requests writes ZERO stdout lines, the access file receives
|
||||
them, and a handler exception still reaches STDERR (error logging intact);
|
||||
* scripts/gunicorn-access-rotate.py keeps the file bounded (copy-truncate,
|
||||
backups capped, rotated copy mode 0640).
|
||||
Runs in the Docker build gate.
|
||||
"""
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import time
|
||||
import unittest
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
try:
|
||||
import gunicorn # noqa: F401
|
||||
except ImportError:
|
||||
sys.stderr.write('SKIP: gunicorn not installed\n')
|
||||
raise SystemExit(77)
|
||||
|
||||
spec = importlib.util.spec_from_file_location('rot', os.path.join(HERE, 'gunicorn-access-rotate.py'))
|
||||
rot = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(rot)
|
||||
|
||||
APP = textwrap.dedent('''\
|
||||
def app(environ, start_response):
|
||||
if environ['PATH_INFO'] == '/boom':
|
||||
raise RuntimeError('handler-exploded')
|
||||
start_response('200 OK', [('Content-Type', 'text/plain'), ('Content-Length', '2')])
|
||||
return [b'ok']
|
||||
''')
|
||||
|
||||
|
||||
def free_port():
|
||||
s = socket.socket()
|
||||
s.bind(('127.0.0.1', 0))
|
||||
p = s.getsockname()[1]
|
||||
s.close()
|
||||
return p
|
||||
|
||||
|
||||
def start_script():
|
||||
with open(os.path.join(HERE, 'start-up.sh')) as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
class StaticFlags(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.src = start_script()
|
||||
# Each invocation = from "gunicorn \" to its app target line.
|
||||
self.inv = re.findall(r'^(?:exec )?gunicorn \\\n(?:.*\\\n)*?.*haproxy_manager:\w+.*$', self.src, re.M)
|
||||
|
||||
def test_two_invocations(self):
|
||||
self.assertEqual(len(self.inv), 2, self.inv)
|
||||
|
||||
def test_no_stdout_access_log(self):
|
||||
for i in self.inv:
|
||||
self.assertNotRegex(i, r'--access-logfile\s+-(\s|$)')
|
||||
|
||||
def test_default_page_has_no_access_log(self):
|
||||
d = [i for i in self.inv if '0.0.0.0:8080' in i]
|
||||
self.assertEqual(len(d), 1)
|
||||
self.assertNotIn('--access-logfile', d[0])
|
||||
self.assertIn('--error-logfile -', d[0])
|
||||
|
||||
def test_api_access_log_is_a_file(self):
|
||||
a = [i for i in self.inv if '0.0.0.0:8000' in i]
|
||||
self.assertEqual(len(a), 1)
|
||||
self.assertIn('--access-logfile "${API_ACCESS_LOG}"', a[0])
|
||||
self.assertIn('--error-logfile -', a[0])
|
||||
|
||||
def test_rotator_started(self):
|
||||
self.assertIn('gunicorn-access-rotate.py', self.src)
|
||||
|
||||
|
||||
class Live(unittest.TestCase):
|
||||
N_DEFAULT = 300
|
||||
N_API = 100
|
||||
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix='gunicorn-access-')
|
||||
with open(os.path.join(self.dir, 'tapp.py'), 'w') as fh:
|
||||
fh.write(APP)
|
||||
|
||||
def run_gunicorn(self, extra, n, boom=False):
|
||||
port = free_port()
|
||||
out = open(os.path.join(self.dir, 'out-%d' % port), 'w+')
|
||||
err = open(os.path.join(self.dir, 'err-%d' % port), 'w+')
|
||||
p = subprocess.Popen(
|
||||
[sys.executable, '-m', 'gunicorn', '--bind', '127.0.0.1:%d' % port,
|
||||
'--workers', '1', '--threads', '4', '--error-logfile', '-', '--log-level', 'info',
|
||||
'--pythonpath', self.dir] + extra + ['tapp:app'],
|
||||
stdout=out, stderr=err)
|
||||
try:
|
||||
for _ in range(100):
|
||||
try:
|
||||
urllib.request.urlopen('http://127.0.0.1:%d/' % port, timeout=2).read()
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(0.2)
|
||||
for _ in range(n):
|
||||
urllib.request.urlopen('http://127.0.0.1:%d/x' % port, timeout=5).read()
|
||||
if boom:
|
||||
try:
|
||||
urllib.request.urlopen('http://127.0.0.1:%d/boom' % port, timeout=5).read()
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
finally:
|
||||
p.terminate()
|
||||
p.wait(10)
|
||||
out.seek(0)
|
||||
err.seek(0)
|
||||
o, e = out.read(), err.read()
|
||||
out.close()
|
||||
err.close()
|
||||
return o, e
|
||||
|
||||
def test_default_page_burst_is_silent(self):
|
||||
out, err = self.run_gunicorn([], self.N_DEFAULT)
|
||||
self.assertEqual(out, '', 'access logging leaked to stdout: %r' % out[:200])
|
||||
self.assertNotIn('GET /x', err)
|
||||
|
||||
def test_api_burst_goes_to_file_not_stdout_and_errors_stay_on_stderr(self):
|
||||
path = os.path.join(self.dir, 'api-access.log')
|
||||
open(path, 'w').close()
|
||||
out, err = self.run_gunicorn(['--access-logfile', path], self.N_API, boom=True)
|
||||
self.assertEqual(out, '', 'access logging leaked to stdout: %r' % out[:200])
|
||||
self.assertNotIn('GET /x', err)
|
||||
with open(path) as fh:
|
||||
lines = fh.read().splitlines()
|
||||
self.assertGreaterEqual(len([l for l in lines if 'GET /x' in l]), self.N_API)
|
||||
self.assertIn('handler-exploded', err) # error logging intact
|
||||
|
||||
|
||||
class Rotation(unittest.TestCase):
|
||||
def test_bounded_and_mode(self):
|
||||
d = tempfile.mkdtemp(prefix='gunicorn-rot-')
|
||||
path = os.path.join(d, 'api-access.log')
|
||||
for _ in range(12):
|
||||
with open(path, 'ab') as fh:
|
||||
fh.write(b'x' * 1000 + b'\n')
|
||||
rot.rotate_if_needed(path, 4000, 3)
|
||||
files = sorted(os.listdir(d))
|
||||
self.assertLessEqual(len(files), 4, files) # live + 3 backups
|
||||
self.assertLessEqual(os.path.getsize(path), 4000 + 1001)
|
||||
self.assertEqual(stat.S_IMODE(os.stat(path + '.1').st_mode), 0o640)
|
||||
self.assertFalse(os.path.exists(path + '.4'))
|
||||
|
||||
def test_missing_file_is_not_fatal(self):
|
||||
self.assertFalse(rot.rotate_if_needed('/nonexistent/api-access.log', 10, 3))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,614 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline tests for scripts/haproxy-stderr-relay.py (runs in the image build)."""
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
spec = importlib.util.spec_from_file_location('relay', os.path.join(HERE, 'haproxy-stderr-relay.py'))
|
||||
relay = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(relay)
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.t = 0.0
|
||||
|
||||
def __call__(self):
|
||||
return self.t
|
||||
|
||||
|
||||
class RelayTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
|
||||
self.path = os.path.join(self.dir, 'haproxy-stderr.log')
|
||||
open(self.path, 'w').close()
|
||||
self.out = io.StringIO()
|
||||
self.clock = Clock()
|
||||
|
||||
def make(self, burst=1000, per_min=120, max_bytes=10**6, backups=3):
|
||||
return relay.Relay(self.path, self.out, burst, per_min, max_bytes, backups, clock=self.clock)
|
||||
|
||||
def append(self, text):
|
||||
with open(self.path, 'a') as fh:
|
||||
fh.write(text)
|
||||
|
||||
def test_history_is_not_replayed_and_new_lines_are_prefixed(self):
|
||||
self.append('old line from a previous container\n')
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(), '')
|
||||
self.append('[ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(),
|
||||
'haproxy-stderr: [ALERT] (10) : Current worker (67) exited with code 134 (Aborted)\n')
|
||||
|
||||
def test_partial_line_is_held_until_complete(self):
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('Thread 1 is abo')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue(), '')
|
||||
self.append('ut to kill the process.\n')
|
||||
r.poll_once()
|
||||
self.assertIn('Thread 1 is about to kill the process.', self.out.getvalue())
|
||||
|
||||
def test_control_characters_cannot_forge_lines(self):
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('a\x1b[31mb\r\x00c\n')
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue().count('\n'), 1)
|
||||
self.assertNotIn('\x1b', self.out.getvalue())
|
||||
|
||||
def test_rate_limit_passes_a_full_dump_then_summarises(self):
|
||||
r = self.make(burst=200, per_min=60)
|
||||
r.poll_once()
|
||||
self.append(''.join('dump line %d\n' % i for i in range(150)))
|
||||
r.poll_once()
|
||||
self.assertEqual(self.out.getvalue().count('\n'), 150)
|
||||
self.out.truncate(0), self.out.seek(0)
|
||||
self.append(''.join('flap %d\n' % i for i in range(500)))
|
||||
r.poll_once()
|
||||
self.assertLessEqual(self.out.getvalue().count('\n'), 52) # ~50 tokens left
|
||||
self.clock.t += 600 # bucket refills
|
||||
self.out.truncate(0), self.out.seek(0)
|
||||
self.append('after storm\n')
|
||||
r.poll_once()
|
||||
r.tick()
|
||||
self.assertIn('[relay] suppressed ', self.out.getvalue())
|
||||
self.assertIn('after storm', self.out.getvalue())
|
||||
|
||||
def test_copytruncate_rotation_bounds_the_file_and_keeps_following(self):
|
||||
r = self.make(max_bytes=100, backups=2)
|
||||
r.poll_once()
|
||||
self.append('x' * 150 + '\n')
|
||||
r.poll_once()
|
||||
r.rotate_if_needed()
|
||||
self.assertEqual(os.path.getsize(self.path), 0)
|
||||
self.assertTrue(os.path.exists(self.path + '.1'))
|
||||
self.append('after rotate\n')
|
||||
r.poll_once()
|
||||
self.assertIn('after rotate', self.out.getvalue())
|
||||
|
||||
def test_missing_file_is_not_an_error(self):
|
||||
os.remove(self.path)
|
||||
r = self.make()
|
||||
r.poll_once()
|
||||
self.append('late\n')
|
||||
r.poll_once()
|
||||
self.assertIn('late', self.out.getvalue())
|
||||
|
||||
|
||||
|
||||
# --- captured scenario: 4 quick reloads (bucket drained) then a forced worker panic ---------------
|
||||
|
||||
def reload_block(pid, n_proxies=100):
|
||||
"""What HAProxy 3.2 writes on stderr for ONE reload (~110 lines on a small config)."""
|
||||
L = ['[NOTICE] (%d) : Reloading HAProxy' % pid,
|
||||
'[NOTICE] (%d) : Initializing new worker (%d)' % (pid, pid + 100),
|
||||
'[NOTICE] (%d) : haproxy version is 3.2.26-1~bpo13+1' % (pid + 100),
|
||||
'[NOTICE] (%d) : path to executable is /usr/sbin/haproxy' % (pid + 100)]
|
||||
for i in range(6):
|
||||
L.append("[WARNING] (%d) : config : parsing [/etc/haproxy/haproxy.cfg:%d] : a 'http-request' rule "
|
||||
"placed after a 'use_backend' rule will still be processed before." % (pid + 100, 800 + i))
|
||||
L.append("[WARNING] (%d) : config: Can't load stats-file '/var/lib/haproxy/stats.dat': cannot open file." % pid)
|
||||
L.append('[NOTICE] (%d) : Loading success.' % pid)
|
||||
for i in range(n_proxies):
|
||||
L.append('[WARNING] (%d) : Proxy be_%d stopped (cumulated conns: FE: 0, BE: 12).' % (pid + 50, i))
|
||||
L.append('[WARNING] (%d) : Former worker (%d) exited with code 0 (Exit)' % (pid, pid + 50))
|
||||
return L
|
||||
|
||||
|
||||
DUMP = [
|
||||
'Thread 1 is about to kill the process (pid 6631).',
|
||||
'* Thread 1 : id=0x7f60efa77400 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
' 1/1 loops=94 ctxsw=234 stuck=0 prof=0 harmless=0 isolated=0',
|
||||
' call trace(15):',
|
||||
' | 0x55c557cd3582 <00 00 00 e8 ae 03 dc ff]: ha_dump_backtrace+0x82/0x43d > main-0x7d0',
|
||||
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
' lock_hist: R:TASK_WQ U:TASK_WQ',
|
||||
' Thread 3 : id=0x7f60ed5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
' Thread 4 : id=0x7f60ecdfe6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
'',
|
||||
"Hint: when reporting this bug to developers, please check if a core file was",
|
||||
" produced, open it with 'gdb', issue 't a a bt full', check that the",
|
||||
'[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)',
|
||||
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy or its dependencies.',
|
||||
'Please check that you are running an up to date and maintained version of haproxy and open a bug report.',
|
||||
'HAProxy version 3.2.26-1~bpo13+1 2026/10/14 - https://haproxy.org/',
|
||||
'Known bugs: http://www.haproxy.org/bugs/bugs-3.2.26.html',
|
||||
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM',
|
||||
'[WARNING] (6328) : All workers exited. Exiting... (134)',
|
||||
]
|
||||
KEYS = ('about to kill', 'Thread 1 :', 'Thread 2 :', 'Thread 3 :', 'Thread 4 :', 'Current worker',
|
||||
'exited with code 134', 'exit-on-failure', 'All workers exited', 'call trace', 'ha_dump_backtrace',
|
||||
'bug in haproxy', 'Please check that you are running', 'produced, open it with')
|
||||
|
||||
|
||||
def scenario(mod, burst=400, drained=True):
|
||||
"""4 quick reloads (>400 lines: the bucket drains) then the panic dump. Returns facts."""
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = mod.Relay('/nonexistent', out, burst, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
lines = []
|
||||
for k in range(4):
|
||||
lines += reload_block(100 * (k + 1))
|
||||
n_before = len(lines)
|
||||
lines += DUMP
|
||||
for l in lines:
|
||||
if l.strip(): # poll_once drops blank lines before _emit
|
||||
r._emit(l)
|
||||
t[0] += 0.003
|
||||
r.tick()
|
||||
return {'out': out.getvalue(), 'err': err.getvalue(), 'reload_lines': n_before, 'bucket': r.tokens, 'r': r}
|
||||
|
||||
|
||||
class CapturedScenarioTests(unittest.TestCase):
|
||||
def test_drained_bucket_then_panic_forwards_every_crash_line(self):
|
||||
f = scenario(relay)
|
||||
self.assertGreater(f['reload_lines'], 400) # would drain a 400-line bucket
|
||||
e = f['err']
|
||||
for k in KEYS:
|
||||
self.assertIn(k, e, k)
|
||||
# the whole dump, line for line (blank lines are dropped by design)
|
||||
got = [l[len(relay.PREFIX):] for l in e.split('\n') if l]
|
||||
self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()])
|
||||
|
||||
def test_dump_is_forwarded_in_full_even_with_an_empty_bucket(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
for l in ('benign %d' % i for i in range(10)):
|
||||
r._emit(l)
|
||||
self.assertEqual(out.getvalue(), '')
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
self.assertEqual(err.getvalue().count('\n'), len([d for d in DUMP if d.strip()]))
|
||||
self.assertEqual(r.suppressed, 10)
|
||||
|
||||
def test_reload_noise_is_collapsed_not_drained(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for k in range(10):
|
||||
for l in reload_block(100 * (k + 1)):
|
||||
r._emit(l)
|
||||
self.assertEqual(r.tokens, 400.0) # bucket untouched by 1000+ noise lines
|
||||
self.assertEqual(r.suppressed, 0)
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
summ = [l for l in out.getvalue().split('\n') if 'reload/start ok' in l]
|
||||
self.assertEqual(len(summ), 10) # exactly one per reload
|
||||
self.assertNotIn('Proxy be_', out.getvalue())
|
||||
self.assertNotIn('parsing [', out.getvalue())
|
||||
self.assertLess(out.getvalue().count('\n'), 15)
|
||||
|
||||
def test_suppressed_lines_are_summarised_per_minute(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for i in range(105):
|
||||
r._emit('flapping server state %d' % i)
|
||||
self.assertNotIn('[relay] suppressed', out.getvalue())
|
||||
t[0] += 61
|
||||
r.tick()
|
||||
self.assertRegex(out.getvalue(), r'\[relay\] suppressed 100 lines')
|
||||
self.assertEqual(r.suppressed, 0)
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
|
||||
def test_benign_lines_go_to_stdout_and_crash_lines_to_stderr(self):
|
||||
# docker's journald driver: stdout -> PRIORITY 6, stderr -> PRIORITY 3, "<N>" is NOT parsed
|
||||
f = scenario(relay)
|
||||
self.assertIn('[relay] reload/start ok', f['out'])
|
||||
self.assertNotIn('Thread', f['out'])
|
||||
self.assertNotIn('Current worker', f['out'])
|
||||
self.assertNotIn('[ALERT]', f['out'])
|
||||
self.assertNotIn('reload/start ok', f['err'])
|
||||
for text in (f['out'], f['err']):
|
||||
for l in text.split('\n'):
|
||||
self.assertFalse(l.startswith('<') or l.startswith(relay.PREFIX + '<'), l)
|
||||
|
||||
def test_clean_worker_exit_is_not_crash_evidence(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r._emit('[WARNING] (1) : Former worker (5) exited with code 0 (Exit)')
|
||||
r._emit('[WARNING] (1) : Former worker (6) exited with code 143 (Terminated)')
|
||||
self.assertEqual(err.getvalue().count('\n'), 1)
|
||||
self.assertIn('code 143', err.getvalue())
|
||||
|
||||
def test_crash_classes_never_limited(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
evidence = ['[ALERT] (6328) : Current worker (6631) exited with code 134 (Aborted)',
|
||||
'[ALERT] (6328) : exit-on-failure: killing every processes with SIGTERM',
|
||||
'[WARNING] (6328) : Current worker (9) exited with code 139 (Segmentation fault)',
|
||||
'[WARNING] (6328) : A worker process unexpectedly died and this can only be explained by a bug in haproxy',
|
||||
'[WARNING] (6328) : All workers exited. Exiting... (134)',
|
||||
' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0',
|
||||
'* Thread 1 : id=0x7f60ef5ff6c0 act=1 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0']
|
||||
for l in evidence:
|
||||
r._emit(l)
|
||||
self.assertEqual(err.getvalue().count('\n'), len(evidence))
|
||||
|
||||
FORGED = [
|
||||
'GET /FATAL/SIGSEGV/segfault HTTP/1.1',
|
||||
'[WARNING] (6328) : Server be_x/ [ALERT] FATAL watchdog is DOWN, reason: Layer4 timeout',
|
||||
'[WARNING] (6328) : Server be/Current worker (1) exited with code 134 is DOWN',
|
||||
'[NOTICE] (6328) : proxy example.com/Thread 1 is about to kill the process (pid 1)',
|
||||
'Host: Current worker (1) exited with code 134 (Aborted)',
|
||||
'Host: [ALERT] (1) : forged alert',
|
||||
'x-forwarded-for: Thread 1 is about to kill the process (pid 1).',
|
||||
'user-agent: * Thread 1 : id=0x7f60ef5ff6c0 act=1',
|
||||
'ha_panic watchdog FATAL SIGABRT',
|
||||
'backend ALERT_Current worker_exited with code 134',
|
||||
'[WARNING] (6328) : Server Former worker (7) exited with code 134 is DOWN',
|
||||
]
|
||||
|
||||
def test_forged_keywords_stay_rate_limited(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
for l in self.FORGED * 50:
|
||||
r._emit(l)
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
self.assertFalse(r.in_dump)
|
||||
self.assertEqual(r.suppressed, len(self.FORGED) * 50)
|
||||
self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # none of it touched the crash budget
|
||||
# a forged header in the middle of normal traffic cannot open a block either
|
||||
r._emit('proxy: Thread 1 is about to kill the process (pid 1).')
|
||||
r._emit('free text that would be a dump line')
|
||||
self.assertFalse(r.in_dump)
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
|
||||
def test_forged_lines_cannot_be_collapsed_away_or_open_blocks(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r._emit("GET /Loading success HTTP/1.1")
|
||||
self.assertEqual(r.routine, 0)
|
||||
self.assertIn('Loading success', out.getvalue())
|
||||
|
||||
def test_flood_of_dump_headers_is_capped_by_the_window_budget(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
for i in range(20000):
|
||||
r._emit('Thread 1 is about to kill the process (pid %d).' % i)
|
||||
r._emit(' Thread 2 : id=0x7f60ef5ff6c0 act=0 glob=0 wq=1 rq=0 tl=0 tlsz=0 rqsz=0')
|
||||
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
|
||||
n = err.getvalue().count('\n')
|
||||
self.assertLessEqual(n, relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
|
||||
self.assertGreater(r.suppressed, 50000) # the rest is counted, not lost silently
|
||||
t[0] += relay.CRASH_WINDOW + 1 # a new window: real evidence flows again
|
||||
before = err.getvalue().count('\n')
|
||||
r._emit('[ALERT] (1) : Current worker (2) exited with code 134 (Aborted)')
|
||||
self.assertEqual(err.getvalue().count('\n'), before + 1)
|
||||
|
||||
def test_bound_closed_block_cannot_rearm_during_cooldown(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
for i in range(relay.DUMP_MAX_LINES + 5):
|
||||
r._emit('never closed %d' % i)
|
||||
self.assertFalse(r.in_dump)
|
||||
r._emit('Thread 1 is about to kill the process (pid 2).') # immediate new header
|
||||
self.assertFalse(r.in_dump)
|
||||
r._emit('call trace line after the second header')
|
||||
self.assertNotIn('call trace line after', err.getvalue())
|
||||
t[0] += relay.DUMP_COOLDOWN + 1 # cooldown over: a real dump opens again
|
||||
r._emit('Thread 1 is about to kill the process (pid 3).')
|
||||
self.assertTrue(r.in_dump)
|
||||
|
||||
def test_hostile_and_very_long_lines(self):
|
||||
r = self.make_relay()
|
||||
r.poll_once()
|
||||
self.append('A' * 500000 + '\n')
|
||||
self.append('\x1b]0;evil\x07 [ALERT] forged\x00\r\n')
|
||||
r.poll_once()
|
||||
outs = self.out.getvalue().split('\n')
|
||||
self.assertTrue(all(len(l) < relay.MAX_LINE + 200 for l in outs))
|
||||
self.assertNotIn('\x1b', self.out.getvalue())
|
||||
self.assertNotIn('\x00', self.out.getvalue())
|
||||
self.assertEqual(self.out.getvalue().count('\n'), 2)
|
||||
|
||||
def test_unterminated_flood_is_bounded_in_memory(self):
|
||||
r = self.make_relay()
|
||||
r.poll_once()
|
||||
for _ in range(3):
|
||||
self.append('B' * 400000) # never a newline
|
||||
r.poll_once()
|
||||
self.assertLessEqual(len(r.pending), relay.MAX_PENDING)
|
||||
|
||||
def test_unclosed_dump_block_cannot_disable_the_limiter(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
for i in range(relay.DUMP_MAX_LINES + 500):
|
||||
r._emit('never closed %d' % i)
|
||||
self.assertFalse(r.in_dump)
|
||||
self.assertLessEqual(err.getvalue().count('\n'), relay.DUMP_MAX_LINES + 3)
|
||||
self.assertGreaterEqual(r.suppressed, 400)
|
||||
|
||||
def test_dump_block_is_closed_by_time_bound(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
t[0] += relay.DUMP_MAX_SECONDS + 1
|
||||
r._emit('later benign line')
|
||||
self.assertFalse(r.in_dump)
|
||||
self.assertNotIn('later benign', err.getvalue())
|
||||
|
||||
|
||||
# --- review finding I2: tenant-triggered ALERT lines must not spend crash evidence --------
|
||||
TENANT_LINES = [
|
||||
"[ALERT] (10) : backend 'be_tenant1' has no server available!",
|
||||
"[WARNING] (10) : Server be_tenant1/s1 is DOWN, reason: Layer4 connection problem, info: \"Connection refused\", check duration: 0ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.",
|
||||
"[WARNING] (10) : Server be_tenant1/s1 is UP, reason: Layer4 check passed, check duration: 0ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue.",
|
||||
"[NOTICE] (10) : Server be_tenant1/s1 is going DOWN for maintenance.",
|
||||
]
|
||||
|
||||
def test_tenant_alert_flood_then_panic_still_forwards_the_full_dump_at_stderr(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for i in range(1200):
|
||||
r._emit(self.TENANT_LINES[0])
|
||||
t[0] += 0.001
|
||||
t[0] += 5.0
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
got = [l[len(relay.PREFIX):] for l in err.getvalue().split('\n') if l]
|
||||
self.assertEqual(got, [d.rstrip() for d in DUMP if d.strip()])
|
||||
self.assertIn('exited with code 134', err.getvalue())
|
||||
self.assertNotIn('exited with code 134', out.getvalue())
|
||||
self.assertNotIn('has no server available', err.getvalue())
|
||||
|
||||
def test_tenant_lines_are_their_own_rate_limited_stdout_class(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for i in range(1200):
|
||||
r._emit(self.TENANT_LINES[i % len(self.TENANT_LINES)])
|
||||
self.assertEqual(err.getvalue(), '') # never PRIORITY 3
|
||||
self.assertLessEqual(out.getvalue().count('\n'), relay.TENANT_BURST + 2)
|
||||
self.assertGreater(out.getvalue().count('\n'), 0) # but not silently dropped
|
||||
self.assertEqual(r.tokens, 400.0) # general bucket untouched
|
||||
self.assertEqual(r.budget_left, relay.CRASH_BUDGET) # crash budget untouched
|
||||
self.assertEqual(r.dump_budget_left, relay.DUMP_BUDGET)
|
||||
self.assertGreater(r.tenant_suppressed, 1000)
|
||||
t[0] += relay.SUMMARY_EVERY + 1
|
||||
r.tick()
|
||||
self.assertRegex(out.getvalue(), r'\[relay\] suppressed \d+ tenant-driven server/backend state lines')
|
||||
self.assertEqual(r.tenant_suppressed, 0)
|
||||
|
||||
def test_other_alerts_are_not_crash_evidence(self):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = relay.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r._emit('[ALERT] (12) : config : unable to load')
|
||||
r._emit("[ALERT] (12) : backend 'x' has no server available!")
|
||||
self.assertEqual(err.getvalue(), '')
|
||||
self.assertEqual(out.getvalue().count("\n"), 2) # both reach stdout (rate-limited classes), neither stderr
|
||||
|
||||
|
||||
def test_dump_block_has_a_budget_ordinary_evidence_cannot_spend(self):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = relay.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
for i in range(relay.CRASH_BUDGET + 500): # e.g. a worker crash loop
|
||||
r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i)
|
||||
self.assertEqual(r.budget_left, 0)
|
||||
before = err.getvalue().count('\n')
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
self.assertEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()]))
|
||||
|
||||
def test_rotated_backup_is_mode_0640(self):
|
||||
r = relay.Relay(self.path, io.StringIO(), 1000, 120, 10, 3, clock=lambda: 0.0)
|
||||
os.chmod(self.path, 0o644)
|
||||
self.append('x' * 100 + '\n')
|
||||
r.rotate_if_needed()
|
||||
self.assertEqual(os.stat(self.path + '.1').st_mode & 0o777, 0o640)
|
||||
|
||||
# shared helpers for the file-driven tests
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix='hap-stderr-relay-')
|
||||
self.path = os.path.join(self.dir, 'haproxy-stderr.log')
|
||||
open(self.path, 'w').close()
|
||||
self.out = io.StringIO()
|
||||
|
||||
def make_relay(self):
|
||||
return relay.Relay(self.path, self.out, 1000, 120, 10 ** 6, 3, clock=lambda: 0.0)
|
||||
|
||||
def append(self, text):
|
||||
with open(self.path, 'a') as fh:
|
||||
fh.write(text)
|
||||
|
||||
|
||||
# --- mutants: the same scenario must FAIL when a guarantee is removed (private copies) ------------
|
||||
|
||||
def _scenario_ok(mod):
|
||||
f = scenario(mod)
|
||||
got = [l[len(mod.PREFIX):] for l in f['err'].split('\n') if l]
|
||||
crash_ok = all(k in f['err'] for k in KEYS) and got == [d.rstrip() for d in DUMP if d.strip()]
|
||||
collapse_ok = f['out'].count('reload/start ok') == 4 and 'Proxy be_' not in f['out'] and f['bucket'] > 300
|
||||
prio_ok = 'Thread' not in f['out'] and 'reload/start ok' not in f['err']
|
||||
return crash_ok, collapse_ok, prio_ok
|
||||
|
||||
|
||||
class MutantTests(unittest.TestCase):
|
||||
SRC = os.path.join(HERE, 'haproxy-stderr-relay.py')
|
||||
|
||||
def load_mutant(self, old, new):
|
||||
with open(self.SRC) as fh:
|
||||
src = fh.read()
|
||||
self.assertIn(old, src, 'mutation target vanished: update the test')
|
||||
mutated = src.replace(old, new, 1)
|
||||
self.assertNotEqual(mutated, src)
|
||||
d = tempfile.mkdtemp(prefix='hap-relay-mutant-') # private copy, the real file is untouched
|
||||
path = os.path.join(d, 'mutant.py')
|
||||
with open(path, 'w') as fh:
|
||||
fh.write(mutated)
|
||||
sp = importlib.util.spec_from_file_location('mutant', path)
|
||||
m = importlib.util.module_from_spec(sp)
|
||||
sp.loader.exec_module(m)
|
||||
return m
|
||||
|
||||
def test_unmutated_copy_passes_everything(self):
|
||||
self.assertEqual(_scenario_ok(relay), (True, True, True))
|
||||
|
||||
def test_mutant_limiter_applied_to_crash_lines(self):
|
||||
m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):', 'if False:')
|
||||
crash_ok, _, _ = _scenario_ok(m)
|
||||
self.assertFalse(crash_ok)
|
||||
|
||||
def test_mutant_dump_block_not_tracked(self):
|
||||
m = self.load_mutant('if not self.in_dump and DUMP_START_RE.match(line) and now >= self.cooldown_until:', 'if False:')
|
||||
crash_ok, _, _ = _scenario_ok(m)
|
||||
self.assertFalse(crash_ok)
|
||||
|
||||
def test_mutant_collapse_disabled(self):
|
||||
m = self.load_mutant('if ROUTINE_RE.match(line):', 'if False:')
|
||||
_, collapse_ok, _ = _scenario_ok(m)
|
||||
self.assertFalse(collapse_ok)
|
||||
|
||||
def test_mutant_crash_lines_on_stdout(self):
|
||||
m = self.load_mutant('self._write(self.err, line)', 'self._write(self.out, line)')
|
||||
_, _, prio_ok = _scenario_ok(m)
|
||||
self.assertFalse(prio_ok)
|
||||
|
||||
def test_mutant_dump_bound_removed(self):
|
||||
m = self.load_mutant('DUMP_MAX_LINES = 500 ', 'DUMP_MAX_LINES = 10 ** 9 ')
|
||||
err = io.StringIO()
|
||||
r = m.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
for i in range(m.DUMP_MAX_LINES if False else 800):
|
||||
r._emit('never closed %d' % i)
|
||||
self.assertTrue(r.in_dump) # real module closes at 500: see test_unclosed_dump_block...
|
||||
|
||||
def _forged_run(self, mod):
|
||||
err = io.StringIO()
|
||||
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
for l in CapturedScenarioTests.FORGED * 20:
|
||||
r._emit(l)
|
||||
return err.getvalue()
|
||||
|
||||
def test_forged_run_clean_on_real_module(self):
|
||||
self.assertEqual(self._forged_run(relay), '')
|
||||
|
||||
def test_mutant_anchors_removed(self):
|
||||
m = self.load_mutant('if self.in_dump or CRASH_RE.match(line):',
|
||||
"if self.in_dump or re.search(r'ALERT|FATAL|Current worker|about to kill|Thread \\d+ : id=', line):")
|
||||
self.assertNotEqual(self._forged_run(m), '')
|
||||
|
||||
def _flood(self, mod):
|
||||
err, t = io.StringIO(), [0.0]
|
||||
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
r.tokens = 0.0
|
||||
for i in range(5000):
|
||||
r._emit('Thread 1 is about to kill the process (pid %d).' % i)
|
||||
r._emit('[WARNING] (1) : All workers exited. Exiting... (134)')
|
||||
return err.getvalue().count('\n')
|
||||
|
||||
def test_flood_capped_on_real_module(self):
|
||||
self.assertLessEqual(self._flood(relay), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
|
||||
|
||||
def test_mutant_budget_removed(self):
|
||||
m = self.load_mutant('if self._budget(now, kind):', 'if True:')
|
||||
self.assertGreater(self._flood(m), relay.CRASH_BUDGET + relay.DUMP_BUDGET + 5)
|
||||
|
||||
def _rearm(self, mod):
|
||||
r = mod.Relay('/nonexistent', io.StringIO(), 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=io.StringIO())
|
||||
r.tokens = 0.0
|
||||
r._emit('Thread 1 is about to kill the process (pid 1).')
|
||||
for i in range(relay.DUMP_MAX_LINES + 5):
|
||||
r._emit('x %d' % i)
|
||||
r._emit('Thread 1 is about to kill the process (pid 2).')
|
||||
return r.in_dump
|
||||
|
||||
def test_mutant_cooldown_removed(self):
|
||||
self.assertFalse(self._rearm(relay))
|
||||
m = self.load_mutant(' and now >= self.cooldown_until:', ':')
|
||||
self.assertTrue(self._rearm(m))
|
||||
|
||||
def test_mutant_summary_removed(self):
|
||||
m = self.load_mutant("self._write(self.out, '[relay] suppressed %d lines", "self._noop = ('[relay] suppressed %d lines")
|
||||
out, t = io.StringIO(), [0.0]
|
||||
r = m.Relay('/nonexistent', out, 5, 60, 1 << 30, 3, clock=lambda: t[0], err=io.StringIO())
|
||||
for i in range(105):
|
||||
r._emit('x %d' % i)
|
||||
t[0] += 61
|
||||
r.tick()
|
||||
self.assertNotIn('[relay] suppressed', out.getvalue())
|
||||
|
||||
|
||||
def _tenant_then_panic_ok(self, mod):
|
||||
out, err, t = io.StringIO(), io.StringIO(), [0.0]
|
||||
r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: t[0], err=err)
|
||||
for i in range(1200):
|
||||
r._emit("[ALERT] (10) : backend 'be_tenant1' has no server available!")
|
||||
t[0] += 0.001
|
||||
t[0] += 5.0
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
got = [l[len(mod.PREFIX):] for l in err.getvalue().split('\n') if l]
|
||||
return got == [d.rstrip() for d in DUMP if d.strip()] and 'no server available' not in err.getvalue()
|
||||
|
||||
def test_tenant_then_panic_clean_on_real_module(self):
|
||||
self.assertTrue(self._tenant_then_panic_ok(relay))
|
||||
|
||||
def test_mutant_wide_alert_exemption_restored(self):
|
||||
m = self.load_mutant("\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:Current worker", "\\[ALERT\\]\\s+\\(\\d+\\)\\s*:\\s*(?:.|Current worker")
|
||||
self.assertFalse(self._tenant_then_panic_ok(m))
|
||||
|
||||
def test_mutant_tenant_class_removed(self):
|
||||
m = self.load_mutant('if TENANT_RE.match(line):', 'if False:')
|
||||
self.assertFalse(self._tenant_then_panic_ok(m) and self._tenant_stdout_ok(m))
|
||||
|
||||
def _tenant_stdout_ok(self, mod):
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = mod.Relay('/nonexistent', out, 400, 120, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
for i in range(1200):
|
||||
r._emit("[WARNING] (10) : Server be_t/s1 is DOWN, reason: Layer4 timeout")
|
||||
return r.tokens == 400.0 and out.getvalue().count('\n') <= mod.TENANT_BURST + 2
|
||||
|
||||
def test_mutant_shared_budget(self):
|
||||
m = self.load_mutant("'dump' if is_dump else 'crash'", "'crash'")
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
r = m.Relay('/nonexistent', out, 1, 1, 1 << 30, 3, clock=lambda: 0.0, err=err)
|
||||
r.tokens = 0.0
|
||||
for i in range(m.CRASH_BUDGET + 500):
|
||||
r._emit('[ALERT] (10) : Current worker (%d) exited with code 134 (Aborted)' % i)
|
||||
before = err.getvalue().count('\n')
|
||||
for l in DUMP:
|
||||
if l.strip():
|
||||
r._emit(l)
|
||||
self.assertNotEqual(err.getvalue().count('\n') - before, len([d for d in DUMP if d.strip()]))
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline tests for scripts/haproxy-supervisor.sh (pidof / python are stubbed on PATH)."""
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SCRIPT = os.path.join(HERE, 'haproxy-supervisor.sh')
|
||||
|
||||
PIDOF = """#!/bin/sh
|
||||
# "haproxy" is alive iff the start stamp exists and is younger than $LIFE seconds
|
||||
[ -f "$STATE/started" ] || exit 1
|
||||
start=$(cat "$STATE/started")
|
||||
now=$(date +%s.%N)
|
||||
awk -v s="$start" -v n="$now" -v l="$LIFE" 'BEGIN{exit !(n - s < l)}'
|
||||
"""
|
||||
PYTHON = """#!/bin/sh
|
||||
# stand-in for ensure_haproxy.py: stamp the start, record it
|
||||
date +%s.%N > "$STATE/started"
|
||||
date +%s.%N >> "$STATE/starts"
|
||||
exit ${ENSURE_RC:-0}
|
||||
"""
|
||||
|
||||
|
||||
class SupervisorTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.d = tempfile.mkdtemp(prefix='hap-sup-')
|
||||
self.state = os.path.join(self.d, 'state')
|
||||
os.mkdir(self.state)
|
||||
for name, body in (('pidof', PIDOF), ('python', PYTHON)):
|
||||
p = os.path.join(self.d, name)
|
||||
with open(p, 'w') as fh:
|
||||
fh.write(body)
|
||||
os.chmod(p, os.stat(p).st_mode | stat.S_IXUSR)
|
||||
|
||||
def run_sup(self, seconds, **env):
|
||||
e = dict(os.environ, PATH=self.d + ':' + os.environ['PATH'], STATE=self.state)
|
||||
e.setdefault('LIFE', '0')
|
||||
e.update(env)
|
||||
try:
|
||||
subprocess.run(['bash', SCRIPT], env=e, timeout=seconds, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
return 'exited'
|
||||
except subprocess.TimeoutExpired:
|
||||
return 'running' # still looping when the clock ran out = alive
|
||||
|
||||
def starts(self):
|
||||
try:
|
||||
with open(os.path.join(self.state, 'starts')) as fh:
|
||||
return [float(x) for x in fh.read().split()]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
def test_non_integer_interval_does_not_kill_the_supervisor(self):
|
||||
self.assertEqual(self.run_sup(8, HAPROXY_SUPERVISOR_INTERVAL='0.5', ENSURE_RC='1'), 'running')
|
||||
self.assertGreaterEqual(len(self.starts()), 2) # fell back to the default interval and kept trying
|
||||
|
||||
def test_garbage_interval_and_max_fall_back(self):
|
||||
self.assertEqual(self.run_sup(4, HAPROXY_SUPERVISOR_INTERVAL='abc', HAPROXY_SUPERVISOR_MAX_BACKOFF='x'), 'running')
|
||||
self.assertGreaterEqual(len(self.starts()), 1)
|
||||
|
||||
def test_crash_loop_backs_off_2_5_10_capped(self):
|
||||
# haproxy dies immediately after every successful start (LIFE=0) -> quick deaths
|
||||
self.run_sup(14, HAPROXY_SUPERVISOR_INTERVAL='1', HAPROXY_SUPERVISOR_BACKOFF_STEPS='1 2 3',
|
||||
HAPROXY_SUPERVISOR_QUICK_DEATH='5', HAPROXY_SUPERVISOR_STABLE='8',
|
||||
HAPROXY_SUPERVISOR_MAX_BACKOFF='3')
|
||||
s = self.starts()
|
||||
gaps = [round(b - a, 1) for a, b in zip(s, s[1:])]
|
||||
self.assertGreaterEqual(len(s), 4, gaps)
|
||||
# gap = interval(1) + backoff step: 2, 3, 4, then capped at 1+3 = 4
|
||||
self.assertLess(gaps[0], gaps[1] + 0.3, gaps)
|
||||
self.assertGreaterEqual(gaps[1], 3.0 - 0.3, gaps)
|
||||
self.assertGreaterEqual(gaps[2], 4.0 - 0.3, gaps)
|
||||
self.assertLessEqual(max(gaps), 4.0 + 0.8, gaps)
|
||||
|
||||
def test_stable_haproxy_resets_the_backoff(self):
|
||||
# alive for 2s after each start; QUICK_DEATH=1 so a 2s life is NOT quick: no backoff growth
|
||||
self.run_sup(9, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='2', HAPROXY_SUPERVISOR_BACKOFF_STEPS='3 3 3',
|
||||
HAPROXY_SUPERVISOR_QUICK_DEATH='1', HAPROXY_SUPERVISOR_STABLE='8',
|
||||
HAPROXY_SUPERVISOR_MAX_BACKOFF='3')
|
||||
s = self.starts()
|
||||
gaps = [b - a for a, b in zip(s, s[1:])]
|
||||
self.assertGreaterEqual(len(s), 3, gaps)
|
||||
self.assertLess(max(gaps), 3.6, gaps) # never the 3 s backoff on top
|
||||
|
||||
def test_running_haproxy_is_left_alone(self):
|
||||
with open(os.path.join(self.state, 'started'), 'w') as fh:
|
||||
fh.write(str(time.time()))
|
||||
self.assertEqual(self.run_sup(3, HAPROXY_SUPERVISOR_INTERVAL='1', LIFE='1000'), 'running')
|
||||
self.assertEqual(self.starts(), [])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -128,6 +128,9 @@ STUB_FILES = {
|
||||
# `lf-file` on the Coraza deny rule; loaded at parse time. Present in the
|
||||
# image (COPY errors /haproxy/errors), stubbed for anything else.
|
||||
'/haproxy/errors/403-waf.html': '<html><body>blocked %[unique-id]</body></html>\n',
|
||||
# `file` on the default-splash return rules; loaded at parse time too.
|
||||
'/haproxy/errors/default-splash.html': '<html><body>splash</body></html>\n',
|
||||
'/haproxy/errors/robots-unrouted.txt': 'User-agent: *\nDisallow: /\n',
|
||||
}
|
||||
|
||||
# (suspension_enabled, coraza_spoe_backend) combinations to render + validate.
|
||||
@@ -137,6 +140,12 @@ SCENARIOS = (
|
||||
'HAPROXY_SUSPENSION_ENABLED': 'true',
|
||||
'HAPROXY_CORAZA_SPOE_BACKEND': '127.0.0.1:9000',
|
||||
}),
|
||||
# The default splash is ON unless switched off; render the off shape too.
|
||||
('splash-off', {
|
||||
'HAPROXY_SUSPENSION_ENABLED': 'true',
|
||||
'HAPROXY_CORAZA_SPOE_BACKEND': '127.0.0.1:9000',
|
||||
'HAPROXY_DEFAULT_SPLASH': 'off',
|
||||
}),
|
||||
)
|
||||
|
||||
|
||||
@@ -298,7 +307,8 @@ def render(scenario_name, env_overrides, workdir):
|
||||
seed_database(os.path.join(scenario_dir, 'haproxy_config.db'))
|
||||
|
||||
saved_env = {}
|
||||
for key in ('HAPROXY_SUSPENSION_ENABLED', 'HAPROXY_CORAZA_SPOE_BACKEND'):
|
||||
for key in ('HAPROXY_SUSPENSION_ENABLED', 'HAPROXY_CORAZA_SPOE_BACKEND',
|
||||
'HAPROXY_DEFAULT_SPLASH'):
|
||||
saved_env[key] = os.environ.pop(key, None)
|
||||
os.environ.update(env_overrides)
|
||||
|
||||
@@ -330,6 +340,8 @@ def assert_scenario_branches(scenario_name, config_path, env_overrides):
|
||||
'HAPROXY_SUSPENSION_ENABLED' in env_overrides),
|
||||
'coraza': ('filter spoe engine coraza',
|
||||
'HAPROXY_CORAZA_SPOE_BACKEND' in env_overrides),
|
||||
'splash': ('file /haproxy/errors/default-splash.html',
|
||||
env_overrides.get('HAPROXY_DEFAULT_SPLASH') != 'off'),
|
||||
}
|
||||
for label, (needle, should_be_present) in expected.items():
|
||||
present = needle in text
|
||||
|
||||
+23
-32
@@ -2,42 +2,33 @@
|
||||
# Global settings
|
||||
#---------------------------------------------------------------------
|
||||
global
|
||||
# ACCESS LOG DESTINATION.
|
||||
# ACCESS LOG DESTINATION: a UNIX datagram socket INSIDE this container.
|
||||
#
|
||||
# This used to be `log 127.0.0.1 local2`, which was a silent black hole:
|
||||
# 127.0.0.1 is the CONTAINER's own loopback, nothing has ever listened on
|
||||
# udp/514 in the container netns, and there is no /dev/log in the image.
|
||||
# Every access log line -- ~1.5M/day across the whole edge -- was written
|
||||
# to a socket with no receiver and dropped. Nothing errored, nothing
|
||||
# warned, and `haproxy -c` was perfectly happy. The cost only shows up
|
||||
# during an incident: per-IP 429s, tarpits, wp-admin gate redirects,
|
||||
# WAF 403s and `silent-drop`s left no record anywhere, so the edge could
|
||||
# not be asked what it had rejected. Only aggregate stick-table counters
|
||||
# survived.
|
||||
# scripts/haproxy-log-receiver.py (supervised by start-up.sh) reads it and
|
||||
# writes a size-capped, rotating /var/log/haproxy/access.log in the
|
||||
# haproxy-logs volume. No network listener anywhere.
|
||||
#
|
||||
# Now points at the DOCKER BRIDGE GATEWAY, where the host's rsyslog has an
|
||||
# imudp listener bound (installed idempotently by WHP's
|
||||
# setup-haproxy-logrotate.sh, which also writes the logrotate stanza).
|
||||
# The host writes local2 to /var/log/haproxy.log and stops it there, so it
|
||||
# does not also flood /var/log/messages or the Graylog forwarder.
|
||||
# History: `log 127.0.0.1 local2` was a black hole (nothing listened in
|
||||
# the container netns); its successor, the docker bridge gateway
|
||||
# 172.18.0.1:514 with a host rsyslog imudp listener, had no listener on
|
||||
# the fleet either (probable cause, not measured: rsyslog binds that
|
||||
# address once at boot, before docker has created the bridge, and does
|
||||
# not retry). Either way per-IP
|
||||
# 429s, tarpits, gate redirects and WAF 403s left no record.
|
||||
#
|
||||
# WHY NOT `log stdout format raw local0`: it is INCOMPATIBLE with the
|
||||
# `daemon` keyword below, and incompatible SILENTLY. Verified on the
|
||||
# pinned 3.0.11 binary: with `daemon` set, a `log stdout` config serves
|
||||
# traffic normally and emits ZERO log lines, and `haproxy -c` returns 0
|
||||
# with no error and no warning -- so the CI config gate
|
||||
# (scripts/validate-rendered-config.py) cannot catch it either. Making it
|
||||
# work means dropping `daemon` / adding -db so haproxy stays in the
|
||||
# foreground, which in turn breaks the three synchronous
|
||||
# `subprocess.run(['haproxy', '-W', ...], check=True)` launch sites in
|
||||
# haproxy_manager.py (they would block until the 180s timeout and then be
|
||||
# killed). That is a change to the exact code path whose failure mode is
|
||||
# "container Up, ports 80/443 never bound, every site down, /health still
|
||||
# 200". Not worth it for a logging change.
|
||||
# WHY NOT `log stdout`: silently emits NOTHING with `daemon` (below) on
|
||||
# the pinned 3.0.11 (`haproxy -c` still passes), and dropping `daemon`
|
||||
# breaks the synchronous `haproxy -W` launch sites in haproxy_manager.py.
|
||||
# Also, `docker logs haproxy-manager` feeds WHP's AI log monitor, which
|
||||
# must not get a per-request firehose.
|
||||
#
|
||||
# UDP means a dead listener degrades to dropped log lines, never to a
|
||||
# stalled or failing request path -- the correct failure direction for an
|
||||
# edge fronting ~60 customer sites.
|
||||
# The path is relative to the `chroot` below: HAProxy resolves it after
|
||||
# chrooting, so `/log.sock` here is /var/lib/haproxy/log.sock on disk,
|
||||
# which is where the receiver binds (HAPROXY_LOG_SOCKET).
|
||||
#
|
||||
# Sends are non-blocking: a dead receiver or a full socket buffer drops
|
||||
# log lines, never stalls a request. HAPROXY_SYSLOG_TARGET still
|
||||
# overrides the target (`<ip>:<port>` for a remote collector).
|
||||
#
|
||||
# len 2048 accommodates the enriched log-format in hap_listener.tpl
|
||||
# (URL + User-Agent + UUID); the default 1024 would truncate long ones.
|
||||
|
||||
@@ -616,6 +616,47 @@ frontend web
|
||||
http-request set-path /suspended if is_suspended_domain
|
||||
use_backend default-backend if is_suspended_domain
|
||||
{%- endif %}
|
||||
{%- if default_splash %}
|
||||
|
||||
# Default splash for traffic that is NOT routed to any site: the server's
|
||||
# own hostname (registered cert-only, no backend), a bare IP, a stale or
|
||||
# mistyped name. HAProxy answers it ITSELF with a static page and the
|
||||
# request goes no further -- in particular it never reaches the Coraza
|
||||
# SPOE call below.
|
||||
#
|
||||
# Why: `use_backend default-backend` does not short-circuit anything --
|
||||
# HAProxy runs EVERY http-request rule (Coraza included) before it picks
|
||||
# a backend. So the catch-all used to be WAF-inspected with the full
|
||||
# server-wide rule set like customer traffic, and a browser visiting the
|
||||
# server hostname also carries the control panel's cookies (cookies are
|
||||
# not port-scoped: whp01:8443 cookies are sent to whp01:443). Anything
|
||||
# Coraza matched there showed the WAF block page for a host that has no
|
||||
# site behind it to protect. A fixed static reply has nothing to protect.
|
||||
#
|
||||
# "Routed" is decided from two lists generate_config() writes from the
|
||||
# SAME domain rows it renders the routing ACLs from (exact names, and
|
||||
# ".base" suffixes for wildcards), port stripped and lowercased. They
|
||||
# are a SUPERSET of what routes -- the routing ACLs below compare the raw
|
||||
# Host (port included) -- so a host that routes is never splashed; the
|
||||
# worst a mismatch can do is send an unrouted request through the WAF to
|
||||
# default-backend, which is the pre-splash behaviour. Both lists are part
|
||||
# of the rollback set (_config_backup_pairs) so they cannot drift from
|
||||
# the haproxy.cfg that is actually loaded.
|
||||
#
|
||||
# Kept OUT of the splash: ACME http-01 (the server hostname's own cert is
|
||||
# issued through it), blocked IPs (blocked-ip page) and suspended hosts
|
||||
# (suspended page). Rate limits, the IP block and the wp-* gates above
|
||||
# still apply. EVERY path is 404 with the same page, plus
|
||||
# X-Robots-Tag noindex and /robots.txt "Disallow: /", so search engines
|
||||
# never index the server's IP or hostname as a site.
|
||||
# Disable with HAPROXY_DEFAULT_SPLASH=off (back to default-backend).
|
||||
acl splash_routed_host req.hdr(host),field(1,:),lower -f /etc/haproxy/routed_hosts.list
|
||||
acl splash_routed_host req.hdr(host),field(1,:),lower -m end -f /etc/haproxy/routed_wildcards.list
|
||||
acl splash_acme path_beg /.well-known/acme-challenge/
|
||||
acl splash_robots path /robots.txt
|
||||
http-request return status 200 content-type "text/plain; charset=utf-8" hdr cache-control "no-store" hdr x-robots-tag "noindex, nofollow, noarchive" hdr x-content-type-options "nosniff" file /haproxy/errors/robots-unrouted.txt if !splash_routed_host !splash_acme !is_blocked_ip{% if suspension_enabled %} !is_suspended_domain{% endif %} splash_robots
|
||||
http-request return status 404 content-type "text/html; charset=utf-8" hdr cache-control "no-store" hdr x-robots-tag "noindex, nofollow, noarchive" hdr x-content-type-options "nosniff" file /haproxy/errors/default-splash.html if !splash_routed_host !splash_acme !is_blocked_ip{% if suspension_enabled %} !is_suspended_domain{% endif %} !splash_robots
|
||||
{%- endif %}
|
||||
|
||||
# CVE-2026-109618 (GHSA-w8hw-w9fx-j79w, PHP "RCE via OOM corruption"),
|
||||
# vector (a): a multipart/form-data body with no declared length is not
|
||||
|
||||
Reference in new issue
Block a user