From b6a62e7f9f748b171985e03d937204b66b0f123a Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Sat, 22 Aug 2026 10:48:49 -0700 Subject: [PATCH] fix(security-stats): stop reporting counters the stick tables never stored /api/security/stats and scripts/show-tarpit-ips.sh reported "Scan Count", "offense count" and BLOCKED/TARPITTED status parsed from gpc0/gpc1. No stick table in this repo has ever stored a general-purpose counter -- the `web` table stores conn_cur, conn_rate(10s), http_req_rate(10s), http_err_rate(30s), and the two brute-force tables store http_req_rate(60s). Every one of those figures was fabricated, and an operator was making decisions on them. Three independent silences kept it alive: * `int(parts[3])` on a positional split hit `exp=368842`, raised ValueError, and the loop `continue`d -- so the endpoint always answered `active_threats: 0` with an empty list. Live on whp01 it also reported parts[0], the `0x...:` allocation pointer, as the source IP. * The command was sent to /tmp/haproxy-cli WITHOUT the `@1` worker prefix. That is the MASTER CLI socket, which answers "Unknown command: 'show' ..." -- and socat still exits 0, so the `returncode != 0` guard never fired. `total_tracked_ips` was the line count of that help text (8) while the real table held 388 entries. * The shell consumers wrote `gpc0=${gpc0:-0}`, rendering a field that does not exist as a confident zero. Report what the tables actually store, rather than adding gpc counters to make the old semantics real. Adding them would mean editing hap_listener.tpl -- the one change here with a silent-total-outage failure mode -- to rebuild enforcement history that the edge access log (shipped 2026.08.8, on the host at /var/log/haproxy.log) already records per request, with status codes, termination states and request references the stick table could never hold. * haproxy_manager.py: STICK_TABLE_FIELD_CONTRACT names what each table stores. haproxy_cli() sends worker commands with `@1`, falls back to the bare form for a plain stats socket, and inspects the RESPONSE BODY because socat's exit status is worthless here. parse_stick_table_entry() reads name=value / name(window_ms)=value pairs by NAME, never by position. read_stick_table() RAISES -- naming the field -- when a row is missing a contract field, instead of defaulting it to 0. * /api/security/stats returns the four real counters with their windows, the true `used:` count, and no invented threat_level/blocked/offense_count. Fewer numbers, all of them real. * scripts/show-edge-ip-rates.sh replaces the fabricated report; the four expected fields are declared once as EXPECTED_FIELDS and drive the parser. show-tarpit-ips.sh becomes a shim that explains why its numbers are gone and points at where tarpit events actually live. * monitor-attacks.sh loses fourteen fabricated "threat" categories and a composite threat score, all permanently zero; its access-log section now says the log is on the host instead of silently printing nothing. * haproxy_tarpit_config.txt -- the never-shipped design sketch these counters were copied from -- gets a NOT IMPLEMENTED banner. * scripts/test-stick-table-contract.py (offline, 21 tests) holds the templates' `store` clauses, STICK_TABLE_FIELD_CONTRACT and every consumer to each other, and asserts each loud-failure path against the real captured responses. Template and consumers can no longer drift apart quietly. No template is touched, so haproxy.cfg is unchanged. Verified on whp01: total_tracked_ips now tracks `used:` exactly (511 vs the table's 511, was 8 vs 388), and per-IP values match `show table web key ` field for field. haproxy PIDs unmoved, `haproxy -c` warnings unchanged, five customer sites HTTP 200. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 22 ++ VERSION | 2 +- haproxy_manager.py | 326 ++++++++++++++---- haproxy_tarpit_config.txt | 34 ++ scripts/monitor-attacks.sh | 248 +++++++------- scripts/show-edge-ip-rates.sh | 314 ++++++++++++++++++ scripts/show-tarpit-ips.sh | 148 ++------- scripts/test-stick-table-contract.py | 475 +++++++++++++++++++++++++++ 8 files changed, 1273 insertions(+), 296 deletions(-) create mode 100755 scripts/show-edge-ip-rates.sh create mode 100644 scripts/test-stick-table-contract.py diff --git a/CLAUDE.md b/CLAUDE.md index eaf3850..02c7f8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,8 +7,30 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ### Testing - **API Testing**: `./scripts/test-api.sh` - Tests all API endpoints with optional authentication - **Certificate Request Testing**: `./scripts/test-certificate-request.sh` - Tests certificate generation endpoints +- **Stick-table contract**: `python3 scripts/test-stick-table-contract.py` - offline; holds the templates' `store` clauses, `STICK_TABLE_FIELD_CONTRACT`, and every consumer to each other. Run it after touching any `stick-table` line. - **Manual Testing**: Run `curl` commands against `http://localhost:8000` endpoints as shown in README.md +### Reading stick tables (and why it is easy to get silently wrong) + +`/tmp/haproxy-cli` is HAProxy's **master** CLI socket. Worker commands +(`show table`, `show map`, `add map`, ...) need an `@1` prefix. Without it +HAProxy answers `Unknown command: 'show', ...` **and socat still exits 0** — so +an exit-status check passes and the help text gets parsed as data. Always use +`haproxy_cli(cmd, worker=True)` in Python, which inspects the response body. + +Stick-table entries are `name=value` / `name(window_ms)=value` pairs, not fixed +columns; the first token is an allocation pointer (`0x...:`), not the key. Parse +by NAME, and treat a missing field as an ERROR — never default it to `0`. The +`web` table stores only `conn_cur`, `conn_rate`, `http_req_rate`, +`http_err_rate`; it holds **no history and no counter of past blocks**. What was +actually denied/tarpitted is in the edge access log on the **host** at +`/var/log/haproxy.log` (shipped 2026.08.8), not in any stick table. + +This is written down because `/api/security/stats` and `show-tarpit-ips.sh` +reported "Scan Count"/"BLOCKED" figures parsed from `gpc0`/`gpc1` — fields no +stick table has ever stored — for their entire existence. See the header of +`haproxy_tarpit_config.txt` and the contract test. + ### Running the Application - **Docker Build**: `docker build -t haproxy-manager .` - **Local Development**: `python haproxy_manager.py` (requires HAProxy, certbot, and dependencies installed) diff --git a/VERSION b/VERSION index 510ab47..7200918 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2026.08.8 +2026.08.9 diff --git a/haproxy_manager.py b/haproxy_manager.py index e4db235..893fa40 100644 --- a/haproxy_manager.py +++ b/haproxy_manager.py @@ -1872,74 +1872,282 @@ def sync_blocked_ips(): log_operation('sync_blocked_ips', False, str(e)) return jsonify({'status': 'error', 'message': str(e)}), 500 +# --------------------------------------------------------------------------- +# HAProxy runtime API (stick tables) +# +# WHY THIS SECTION IS SO DEFENSIVE +# -------------------------------- +# The previous /api/security/stats read `gpc0` and `gpc1` out of `show table +# web` and reported them as "scan count" / "offense count" / "blocked". No +# stick table in this repo has EVER stored a general-purpose counter — see +# STICK_TABLE_FIELD_CONTRACT below and the `store` clauses in +# templates/hap_listener.tpl and templates/hap_security_tables.tpl. Three +# separate silences let that survive: +# +# 1. `int(parts[3])` on a positional split raised ValueError on `exp=368842` +# and the loop just `continue`d, so every row was skipped and the endpoint +# always answered `active_threats: 0` with an empty list. An operator +# reading that saw "no threats" and could not tell it apart from "the +# parser is broken". +# 2. The command was sent WITHOUT a worker prefix. /tmp/haproxy-cli is the +# MASTER socket; `show table web` there is answered with "Unknown command: +# 'show', but maybe one of the following ones is a better match: ..." -- +# and socat still exits 0, so `result.returncode != 0` never fired. The +# reported `total_tracked_ips` was literally the number of lines in that +# help text minus one (8), while the real table held 388 entries. +# 3. The shell consumers defaulted every missing field to 0 (`${gpc0:-0}`), +# so a field that does not exist rendered as a confident zero. +# +# Rules for anything added here, all three aimed at the same failure mode: +# * Read the CONTRACT, not positions. Stick-table output is `name=value` / +# `name(window)=value` pairs whose order and presence follow the template's +# `store` clause. Positional indexing silently reads the wrong column the +# moment that clause changes. +# * NEVER default a missing field to a number. A field the table does not +# store must surface as an error naming the field, not as 0. +# * NEVER trust socat's exit status. HAProxy reports command errors in the +# response BODY and the socket still closes cleanly. Use haproxy_cli(). +# +# scripts/test-stick-table-contract.py holds STICK_TABLE_FIELD_CONTRACT, the +# rendered templates and the shell consumers to each other, and fails if any +# one of them drifts. +# --------------------------------------------------------------------------- + +# What each stick table ACTUALLY stores, per its `store` clause. The single +# source of truth for every consumer in this repo. Keep in sync with the +# templates -- the contract test enforces that, in both directions. +STICK_TABLE_FIELD_CONTRACT = { + 'web': ('conn_cur', 'conn_rate', 'http_req_rate', 'http_err_rate'), + 'wp_bruteforce': ('http_req_rate',), + 'xmlrpc_bruteforce': ('http_req_rate',), +} + +# Metadata every stick-table entry carries regardless of the `store` clause. +STICK_TABLE_ENTRY_META = ('key', 'use', 'exp', 'shard') + +# HAProxy answers a rejected runtime command in the response body and the +# socket still closes 0. These are the prefixes it uses. +_HAPROXY_CLI_ERROR_MARKERS = ( + 'Unknown command', + 'No such table', + 'Permission denied', + "Can't find the specified process", + 'unknown process', + 'Missing ', +) + +_STICK_TABLE_TOKEN_RE = re.compile(r'^([a-z_][a-z0-9_]*)(?:\(([^)]*)\))?=(.*)$') +_STICK_TABLE_HEADER_RE = re.compile( + r'^#\s*table:\s*(?P[^,]+),\s*type:\s*(?P[^,]+),\s*' + r'size:\s*(?P\d+),\s*used:\s*(?P\d+)') + + +class HaproxyCliError(RuntimeError): + """A runtime-API command was rejected, timed out, or answered nothing. + + Exists so a rejected command cannot be mistaken for an empty result. That + distinction is the whole point of this module's stick-table code. + """ + + +def _haproxy_socket_path(): + return HAPROXY_SOCKET_PATH if os.path.exists(HAPROXY_SOCKET_PATH) else '/tmp/haproxy-cli' + + +def _cli_response_is_error(text): + if text is None: + return True + head = text.lstrip() + return any(head.startswith(marker) for marker in _HAPROXY_CLI_ERROR_MARKERS) + + +def _cli_send(command, socket_path, timeout): + proc = subprocess.run( + ['socat', 'stdio', socket_path], + input=command + '\n', capture_output=True, text=True, timeout=timeout) + if proc.returncode != 0: + raise HaproxyCliError( + 'socat failed talking to %s (exit %d): %s' + % (socket_path, proc.returncode, (proc.stderr or '').strip())) + return proc.stdout + + +def haproxy_cli(command, worker=False, timeout=None): + """Send one runtime-API command and return its response, or raise. + + `worker=True` marks a command that only the WORKER answers (show table, + show map, add map, ...). On this deployment the socket is HAProxy's MASTER + CLI, where those need an `@1` prefix; on a plain stats socket they must NOT + have one. Rather than guessing from configuration that can change under us, + try the prefixed form and fall back -- and raise if BOTH are rejected, + instead of returning HAProxy's help text as if it were data. + """ + socket_path = _haproxy_socket_path() + timeout = timeout if timeout is not None else DEFAULT_SUBPROCESS_TIMEOUT + attempts = (['@1 ' + command, command] if worker else [command]) + failures = [] + for attempt in attempts: + try: + out = _cli_send(attempt, socket_path, timeout) + except subprocess.TimeoutExpired: + raise HaproxyCliError('timed out after %ss running %r on %s' + % (timeout, attempt, socket_path)) + if out.strip() and not _cli_response_is_error(out): + return out + failures.append('%r -> %r' % (attempt, out.strip()[:200] or '')) + raise HaproxyCliError( + 'HAProxy rejected %r on %s (socat exited 0 -- the rejection is in the ' + 'response body, which is exactly why this is checked): %s' + % (command, socket_path, '; '.join(failures))) + + +def parse_stick_table_entry(line): + """{field: {'value': str, 'window_ms': int|None}} for one `show table` row. + + Parses `name=value` / `name(window)=value` pairs by NAME. The leading + `0x...:` allocation pointer is skipped -- reading it as the key is how the + old code came to report memory addresses as IP addresses. + """ + fields = {} + for token in line.split(): + m = _STICK_TABLE_TOKEN_RE.match(token) + if not m: + continue # the 0x...: pointer, or anything else unnamed + name, window, value = m.group(1), m.group(2), m.group(3) + fields[name] = { + 'value': value, + 'window_ms': int(window) if window and window.isdigit() else None, + } + return fields + + +def read_stick_table(table): + """(header dict, [(raw line, parsed fields)]) for a stick table. + + Raises HaproxyCliError if the response is not a stick-table dump, or if any + row is missing a field the contract says the table stores. A field the + table does not carry is an ERROR here, never a zero. + """ + expected = STICK_TABLE_FIELD_CONTRACT.get(table) + if expected is None: + raise HaproxyCliError( + 'no field contract for stick table %r; add it to ' + 'STICK_TABLE_FIELD_CONTRACT (and to the templates) first' % table) + + raw = haproxy_cli('show table %s' % table, worker=True) + lines = raw.strip().split('\n') + header = _STICK_TABLE_HEADER_RE.match(lines[0]) if lines else None + if not header: + raise HaproxyCliError( + 'response to `show table %s` is not a stick-table dump; first line ' + 'was %r' % (table, lines[0][:200] if lines else '')) + + entries = [] + for line in lines[1:]: + if not line.strip() or line.lstrip().startswith('#'): + continue + fields = parse_stick_table_entry(line) + if 'key' not in fields: + raise HaproxyCliError( + 'stick-table row for %r has no key= field: %r' % (table, line[:200])) + missing = [f for f in expected if f not in fields] + if missing: + raise HaproxyCliError( + 'stick table %r no longer stores %s -- STICK_TABLE_FIELD_CONTRACT ' + 'and the `store` clause in templates/hap_listener.tpl have drifted ' + 'apart. Present: %s. Offending row: %r' + % (table, ', '.join(missing), + ', '.join(sorted(fields)), line[:200])) + entries.append((line, fields)) + + return { + 'name': header.group('name'), + 'type': header.group('type'), + 'size': int(header.group('size')), + 'used': int(header.group('used')), + }, entries + + @app.route('/api/security/stats', methods=['GET']) @require_api_key def get_security_stats(): - """Get current security statistics from HAProxy stick table""" + """Per-source connection and request rates from the `web` stick table. + + Reports ONLY what the table stores: conn_cur, conn_rate, http_req_rate and + http_err_rate, each with the window HAProxy is actually counting over. It + deliberately does NOT classify a "threat level" or report a "blocked" flag: + the thresholds live in templates/hap_listener.tpl and a copy here would be + a second source of truth free to drift, which is the class of bug this + endpoint used to be. Sources at or over a limit are visible from the rates + themselves, and the enforcement that actually happened -- 429s, tarpits + (termination state PT), WAF denials -- is in the edge access log on the + HOST at /var/log/haproxy.log, which records per-request outcomes the stick + table never held. + + Query params: + limit max sources returned (default 50) + min_req_rate only sources at or above this http_req_rate (default 1, + i.e. sources with current activity; pass 0 for all) + """ try: - if os.path.exists(HAPROXY_SOCKET_PATH): - socket_path = HAPROXY_SOCKET_PATH - else: - socket_path = '/tmp/haproxy-cli' + limit = max(1, min(int(request.args.get('limit', 50)), 1000)) + min_req_rate = max(0, int(request.args.get('min_req_rate', 1))) + except ValueError: + return jsonify({'status': 'error', + 'message': 'limit and min_req_rate must be integers'}), 400 - # Get stick table data - cmd = f'echo "show table web" | socat stdio {socket_path}' - result = subprocess.run(cmd, shell=True, capture_output=True, text=True) - - if result.returncode != 0: - return jsonify({'status': 'error', 'message': 'Failed to get stick table data'}), 500 - - # Parse stick table output - lines = result.stdout.strip().split('\n') - threats = [] - - for line in lines[1:]: # Skip header - parts = line.split() - if len(parts) >= 8: - ip = parts[0] - try: - gpc0 = int(parts[3]) if len(parts) > 3 else 0 - gpc1 = int(parts[4]) if len(parts) > 4 else 0 - req_rate = int(parts[5]) if len(parts) > 5 else 0 - err_rate = int(parts[6]) if len(parts) > 6 else 0 - conn_rate = int(parts[7]) if len(parts) > 7 else 0 - - # Only include IPs with significant activity - if gpc0 > 0 or gpc1 > 0 or req_rate > 30 or err_rate > 5 or conn_rate > 10: - threat_level = 'low' - if gpc1 > 2: - threat_level = 'critical' - elif gpc0 > 0 or err_rate > 10: - threat_level = 'high' - elif req_rate > 40 or conn_rate > 15: - threat_level = 'medium' - - threats.append({ - 'ip': ip, - 'blocked': gpc0 > 0, - 'repeat_offender': gpc1 > 2, - 'offense_count': gpc1, - 'request_rate': req_rate, - 'error_rate': err_rate, - 'connection_rate': conn_rate, - 'threat_level': threat_level - }) - except (ValueError, IndexError): - continue - - # Sort by threat level - threats.sort(key=lambda x: (x['offense_count'], x['error_rate'], x['request_rate']), reverse=True) - - return jsonify({ - 'status': 'success', - 'total_tracked_ips': len(lines) - 1, - 'active_threats': len(threats), - 'threats': threats[:50] # Limit to top 50 - }) + try: + header, entries = read_stick_table('web') + except HaproxyCliError as e: + log_operation('get_security_stats', False, str(e)) + return jsonify({'status': 'error', 'message': str(e)}), 502 except Exception as e: log_operation('get_security_stats', False, str(e)) return jsonify({'status': 'error', 'message': str(e)}), 500 + counters = STICK_TABLE_FIELD_CONTRACT['web'] + windows = {} + sources = [] + active = 0 + for line, fields in entries: + row = {'ip': fields['key']['value']} + for name in counters: + # A non-numeric counter means HAProxy's output format changed under + # us. Say so; do not coerce it to 0 and report it as a measurement. + try: + row[name] = int(fields[name]['value']) + except ValueError: + msg = ('stick table web reported a non-numeric %s=%r; the ' + '`show table` output format has changed. Row: %r' + % (name, fields[name]['value'], line[:200])) + log_operation('get_security_stats', False, msg) + return jsonify({'status': 'error', 'message': msg}), 502 + if fields[name]['window_ms'] is not None: + windows.setdefault(name, fields[name]['window_ms']) + if any(row[name] > 0 for name in counters): + active += 1 + if row['http_req_rate'] >= min_req_rate: + sources.append(row) + + sources.sort(key=lambda r: (r['http_req_rate'], r['http_err_rate'], + r['conn_rate'], r['conn_cur']), reverse=True) + + return jsonify({ + 'status': 'success', + 'table': header['name'], + 'table_size': header['size'], + 'total_tracked_ips': header['used'], + 'sources_with_activity': active, + 'counters': list(counters), + 'counter_windows_ms': windows, + 'returned': len(sources[:limit]), + 'min_req_rate': min_req_rate, + 'sources': sources[:limit], + 'note': ('Current per-source rates only. The stick table stores no ' + 'history and no counter of past blocks; enforcement events ' + 'are in the edge access log on the host at /var/log/haproxy.log.'), + }) + @app.route('/api/security/temporary-block', methods=['POST']) @require_api_key def temporary_block(): diff --git a/haproxy_tarpit_config.txt b/haproxy_tarpit_config.txt index 0c16413..ce85c12 100644 --- a/haproxy_tarpit_config.txt +++ b/haproxy_tarpit_config.txt @@ -1,3 +1,37 @@ +# ============================================================================= +# NOT IMPLEMENTED. THIS FILE IS A DESIGN SKETCH THAT WAS NEVER SHIPPED. +# ============================================================================= +# +# Nothing here is deployed, has ever been deployed, or is rendered into +# haproxy.cfg. The real edge config is generated from templates/*.tpl. Compare: +# +# THIS FILE proposes: store gpc0,gpc1,gpc2,http_err_rate(30s),... +# plus sc-inc-gpc0/1/2 scan-escalation rules +# templates/hap_listener.tpl ACTUALLY has: +# store conn_cur,conn_rate(10s),http_req_rate(10s),http_err_rate(30s) +# +# There is no gpc0, no gpc1, no gpc2, and no scan-escalation state anywhere on +# the edge, and never has been. +# +# WHY THE BANNER. This file was mistaken for the shipped config. Two consumers +# -- /api/security/stats in haproxy_manager.py and scripts/show-tarpit-ips.sh -- +# were written to parse gpc0/gpc1 out of `show table web`, defaulting missing +# fields to 0. The result was a "Scan Count" column and BLOCKED/TARPITTED +# statuses that were pure fabrication, presented to an operator as fact, for +# the entire life of both tools. Fixed 2026-08-22; scripts/test-stick-table- +# contract.py now fails if any consumer's field expectations and the templates' +# `store` clauses ever drift apart again. +# +# IF YOU WANT TO REVIVE ANY OF THIS: the counters must be added to a template +# `store` clause and to STICK_TABLE_FIELD_CONTRACT in haproxy_manager.py first. +# Adding them to a consumer alone produces confident zeros, not data. Note also +# that sc0/sc1/sc2 are all in use and HAProxy's tune.stick-counters defaults to +# 3, and that since 2026.08.8 the edge has real per-request access logging on +# the host at /var/log/haproxy.log -- which records what was actually denied, +# tarpitted and rate-limited, with request references. That log is a better +# source for most of what this sketch was reaching for. +# ============================================================================= + global daemon log stdout local0 info diff --git a/scripts/monitor-attacks.sh b/scripts/monitor-attacks.sh index 0d9bdbe..329264a 100755 --- a/scripts/monitor-attacks.sh +++ b/scripts/monitor-attacks.sh @@ -1,136 +1,148 @@ -#!/bin/bash +#!/usr/bin/env bash +# +# monitor-attacks.sh — HAProxy edge activity monitor. +# +# Two sections, both fed from real data: +# 1. Current per-IP rates, from the `web` stick table (delegated to +# show-edge-ip-rates.sh — there is exactly one stick-table parser). +# 2. Recent enforcement events, from the HAProxy access log. +# +# HISTORY / WHY THIS IS SHORTER THAN IT USED TO BE +# The previous version printed a "Threat Intelligence Dashboard" with +# fourteen categories (auth_fail, authz_fail, scanner, sql_inj, traversal, +# wp_brute, admin_scan, shell_att, repeat_off, manual_bl, auto_bl, +# glitch_rate, ...) and a composite "threat score", all parsed out of +# gpc(0), gpc(1), gpc(3), gpc(12), gpc(13) and glitch_rate(300s). NONE of +# those fields exist: the `web` table stores only conn_cur, conn_rate, +# http_req_rate and http_err_rate. Every category was permanently 0 and the +# whole dashboard printed nothing while implying it was watching. All of it +# has been deleted rather than "fixed" — there was no data source to fix it +# against. +# +# Usage: monitor-attacks.sh [live] +# Env: LOG_FILE= access log to read (default /var/log/haproxy.log) +# LOG_LINES= how many trailing log lines to scan (default 500) -# Real-time attack monitoring for HAProxy -# Shows blocked requests and suspicious activity +set -uo pipefail -LOG_FILE="/var/log/haproxy.log" -SOCKET="/tmp/haproxy-cli" +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +LOG_FILE="${LOG_FILE:-/var/log/haproxy.log}" +LOG_LINES="${LOG_LINES:-500}" -echo "===================================================" -echo "HAProxy Security Monitor - Real-time Attack Detection" -echo "===================================================" -echo "" - -# Function to show current threats with HAProxy 3.0.11 metrics -show_threats() { - echo "HAProxy 3.0.11 Threat Intelligence Dashboard:" - echo "show table web" | socat stdio "$SOCKET" 2>/dev/null | \ - awk 'NR>1 { - # Parse the stick table output for array-based GPC values - ip = $1 - # Look for GPC array values in the data - auth_fail = 0 - authz_fail = 0 - rate_viol = 0 - scanner = 0 - sql_inj = 0 - traversal = 0 - wp_brute = 0 - admin_scan = 0 - shell_att = 0 - repeat_off = 0 - manual_bl = 0 - auto_bl = 0 - glitch_rate = 0 - threat_score = 0 - - # Extract relevant metrics (simplified parsing) - if ($0 ~ /gpc\(0\)=([0-9]+)/) { - match($0, /gpc\(0\)=([0-9]+)/, arr); auth_fail = arr[1] - } - if ($0 ~ /gpc\(1\)=([0-9]+)/) { - match($0, /gpc\(1\)=([0-9]+)/, arr); authz_fail = arr[1] - } - if ($0 ~ /gpc\(3\)=([0-9]+)/) { - match($0, /gpc\(3\)=([0-9]+)/, arr); scanner = arr[1] - } - if ($0 ~ /gpc\(12\)=([0-9]+)/) { - match($0, /gpc\(12\)=([0-9]+)/, arr); repeat_off = arr[1] - } - if ($0 ~ /gpc\(13\)=([0-9]+)/) { - match($0, /gpc\(13\)=([0-9]+)/, arr); manual_bl = arr[1] - } - if ($0 ~ /glitch_rate\(300s\)=([0-9]+)/) { - match($0, /glitch_rate\(300s\)=([0-9]+)/, arr); glitch_rate = arr[1] - } - - # Calculate composite threat score (simplified) - threat_score = auth_fail*10 + authz_fail*8 + scanner*12 + repeat_off*25 + manual_bl*100 - - # Only show IPs with significant threat indicators - if (auth_fail > 0 || authz_fail > 0 || scanner > 0 || repeat_off > 0 || manual_bl > 0 || glitch_rate > 0) { - threat_level = "LOW" - if (threat_score >= 100) threat_level = "CRITICAL" - else if (threat_score >= 50) threat_level = "HIGH" - else if (threat_score >= 20) threat_level = "MEDIUM" - - printf "%-15s [%8s] Score:%-3d Auth:%-2d Authz:%-2d Scanner:%-1d Repeat:%-1d Glitch:%-2d\n", - ip, threat_level, threat_score, auth_fail, authz_fail, scanner, repeat_off, glitch_rate - } - }' | head -15 - - echo "" - echo "Top HTTP/2 Protocol Violators:" - echo "show table web" | socat stdio "$SOCKET" 2>/dev/null | \ - awk 'NR>1 && $0 ~ /glitch/ { - if ($0 ~ /glitch_rate\(300s\)=([0-9]+)/) { - match($0, /glitch_rate\(300s\)=([0-9]+)/, arr) - if (arr[1] > 2) { - printf "%-15s glitch_rate:%-3s\n", $1, arr[1] - } - } - }' | head -5 - echo "---------------------------------------------------" +# --- Section 1: current rates (real stick-table data) ----------------------- +show_rates() { + "$SCRIPT_DIR/show-edge-ip-rates.sh" "$@" } -# Function to show recent blocks +# --- Section 2: recent enforcement events (real access-log data) ------------ show_recent_blocks() { - echo "Recent Blocked Requests:" - tail -100 "$LOG_FILE" 2>/dev/null | \ - grep -E "(bot_scanner|scan_admin|scan_shells|sql_injection|directory_traversal|rate_abuse|tarpit|denied|403)" | \ - tail -10 | \ - awk '{ - if (match($0, /[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+/)) { - ip = substr($0, RSTART, RLENGTH) - gsub(/:.*/, "", ip) - reason = "" - if ($0 ~ /bot_scanner/) reason = "BOT_SCANNER" - else if ($0 ~ /scan_admin/) reason = "ADMIN_SCAN" - else if ($0 ~ /scan_shells/) reason = "SHELL_SCAN" - else if ($0 ~ /sql_injection/) reason = "SQL_INJECTION" - else if ($0 ~ /directory_traversal/) reason = "DIR_TRAVERSAL" - else if ($0 ~ /rate_abuse/) reason = "RATE_ABUSE" - else if ($0 ~ /tarpit/) reason = "TARPIT" - else if ($0 ~ /denied/) reason = "DENIED" - else if ($0 ~ /403/) reason = "BLOCKED" - printf "[%s] %-15s %s\n", strftime("%H:%M:%S"), ip, reason + echo "Recent enforcement events (last $LOG_LINES log lines):" + echo + + if [ ! -f "$LOG_FILE" ] || [ ! -r "$LOG_FILE" ]; then + cat <' /var/log/haproxy.log | tail -50 # one client IP + grep -a 'id=' /var/log/haproxy.log # one request reference + # (the UUID on the block page) + tail -f /var/log/haproxy.log | grep -aE ' (PT|PR)--' # live + + Or point this script at a copy: LOG_FILE=/path/to/haproxy.log $0 +MSG + echo + return 0 + fi + + printf "%-15s %-16s %-4s %-5s %-28s %s\n" "TIME" "CLIENT IP" "CODE" "TERM" "HOST" "REQUEST / REQUEST-ID" + printf "%s\n" "-----------------------------------------------------------------------------------------------------" + + local found + found=$(tail -n "$LOG_LINES" "$LOG_FILE" 2>/dev/null | awk ' + { + status = ""; term = ""; cip = ""; host = ""; id = ""; ts = ""; req = "" + + # %tr is bracketed: [22/Aug/2026:10:11:12.345] -> keep HH:MM:SS + # No {n} interval expressions here: not every awk in a slim Debian + # image supports them. Spelled out instead. + if (match($0, /\[[0-9][0-9]\/[A-Za-z][A-Za-z][A-Za-z]\/[0-9][0-9][0-9][0-9]:[0-9][0-9]:[0-9][0-9]:[0-9][0-9]/)) { + ts = substr($0, RSTART + 13, 8) + } + + # Anchor on the %TR/%Tw/%Tc/%Tr/%Ta timers block: %ST follows it, and + # the termination state (%tsc) is 4 fields further on (%B %CC %CS %tsc). + for (i = 1; i <= NF; i++) { + if ($i ~ /^[+-]?[0-9]+\/[+-]?[0-9]+\/[+-]?[0-9]+\/[+-]?[0-9]+\/[+-]?[0-9]+$/) { + status = $(i + 1) + term = $(i + 5) + break } - }' - echo "" + } + + # Only enforcement outcomes: tarpit (PT--), deny (PR--), 403, 429. + if (!(term ~ /^PT/ || term ~ /^PR/ || status == "403" || status == "429")) next + + for (i = 1; i <= NF; i++) { + if (substr($i, 1, 4) == "cip=") cip = substr($i, 5) + if (substr($i, 1, 5) == "host=") host = substr($i, 6) + if (substr($i, 1, 3) == "id=") id = substr($i, 4) + } + + if (match($0, /"[A-Z]+ [^"]*"/)) { + req = substr($0, RSTART + 1, RLENGTH - 2) + if (length(req) > 42) req = substr(req, 1, 41) "..." + } + + if (cip == "") cip = "-" + if (host == "") host = "-" + if (term == "") term = "-" + if (status == "") status = "-" + printf "%-15s %-16s %-4s %-5s %-28s %s\n", ts, cip, status, term, host, req + if (id != "" && id != "-") printf "%-15s %s\n", "", " id=" id + n++ + } + END { if (n == 0) print "(no tarpit/deny/403/429 events in the scanned window)" } + ') + printf '%s\n' "$found" + echo + echo "TERM = HAProxy termination state: PT-- tarpit, PR-- deny (incl. WAF/rate limit)." + echo "id= = request reference; it is printed on the block page and is how a" + echo " customer support ticket correlates to an exact request here." } -# Monitor mode selection -if [ "$1" == "live" ]; then - echo "Live monitoring mode - Press Ctrl+C to exit" - echo "" +banner() { + echo "===================================================" + echo "HAProxy Edge Monitor - $(date '+%Y-%m-%d %H:%M:%S')" + echo "===================================================" + echo +} +if [ "${1:-}" = "live" ]; then + echo "Live monitoring mode - Press Ctrl+C to exit" while true; do clear - echo "===================================================" - echo "HAProxy Security Monitor - $(date '+%Y-%m-%d %H:%M:%S')" - echo "===================================================" - echo "" - show_threats - echo "" + banner + show_rates || true + echo show_recent_blocks sleep 5 done else - # Single run mode - show_threats - echo "" + banner + rc=0 + show_rates || rc=$? + echo show_recent_blocks - echo "" - echo "Tip: Run with 'live' parameter for continuous monitoring" + echo + echo "Tip: run with 'live' for a refreshing view." echo "Usage: $0 [live]" -fi \ No newline at end of file + # Propagate a stick-table read failure: if the rates section could not be + # produced, this run did NOT report what it claims to report. + exit "$rc" +fi diff --git a/scripts/show-edge-ip-rates.sh b/scripts/show-edge-ip-rates.sh new file mode 100755 index 0000000..f74200c --- /dev/null +++ b/scripts/show-edge-ip-rates.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash +# +# show-edge-ip-rates.sh — real, current per-IP rate counters from the HAProxy +# `web` stick table. +# +# WHAT THIS CAN TELL YOU +# The `web` stick table (templates/hap_listener.tpl) stores exactly four +# counters per client IP: +# conn_cur, conn_rate(10s), http_req_rate(10s), http_err_rate(30s) +# Those are INSTANTANEOUS values — the current concurrency and the current +# sliding-window rates. This script prints them, and nothing else. +# +# WHAT THIS CANNOT TELL YOU +# * Who has been tarpitted, denied, or rate-limited. The stick table stores +# NO history and NO counter of past enforcement actions. It has no gpc0 / +# gpc1 / gpc(N) / gpc_rate / glitch_rate columns at all — any tool that +# claims to read them from this table is fabricating numbers. +# * Anything about an IP that has gone quiet: entries expire after 10m. +# +# Real enforcement events live in the HAProxy ACCESS LOG, which is on the +# DOCKER HOST at /var/log/haproxy.log (it does NOT exist inside this +# container). The log-format carries the HAProxy termination state plus +# cip= (real client IP), host=, ua= and id= (the request UUID shown on the +# block page, which correlates with customer support tickets). +# +# Ready to run ON THE HOST: +# # last 20 tarpitted (PT--) or denied (PR--) requests +# grep -aE ' (PT|PR)--' /var/log/haproxy.log | tail -20 +# # everything HAProxy answered 429/403 to, newest last +# grep -aE ' (429|403) ' /var/log/haproxy.log | tail -20 +# # everything for one client IP +# grep -a 'cip=203.0.113.7' /var/log/haproxy.log | tail -50 +# # look up one request reference from a support ticket +# grep -a 'id=' /var/log/haproxy.log +# +# USAGE +# show-edge-ip-rates.sh [-a|--all] +# -a, --all also show rows whose counters are all zero (off by default: +# a table with hundreds of idle entries is pure noise) +# +# ENVIRONMENT +# SHOW_ALL=1 same as --all +# HAPROXY_SOCKET= override the CLI socket (default /tmp/haproxy-cli) +# HAPROXY_TABLE_DUMP= +# parse a previously captured `show table web` dump +# from a file instead of talking to the socket. +# Supported seam for offline analysis of a captured +# support bundle, and for testing this parser. +# +# NOTE ON THE SOCKET +# /tmp/haproxy-cli is HAProxy's MASTER CLI socket, so worker commands need an +# `@1` prefix. Without it HAProxy answers "Unknown command: 'show' ..." AND +# socat still exits 0 — so exit status is worthless here and this script +# inspects the RESPONSE BODY instead. + +set -euo pipefail + +SOCKET="${HAPROXY_SOCKET:-/tmp/haproxy-cli}" +TABLE="web" + +# Fields this script expects the `web` stick table to store. Keep on ONE line +# in this exact NAME=(a b c d) shape — the contract test greps for it, and the +# parser below is driven entirely by it. +EXPECTED_FIELDS=(conn_cur conn_rate http_req_rate http_err_rate) + +# Which of EXPECTED_FIELDS to sort on (descending). Falls back to the first +# field if this name is not in the list. +SORT_FIELD="http_req_rate" + +SHOW_ALL="${SHOW_ALL:-0}" +while [ $# -gt 0 ]; do + case "$1" in + -a|--all) SHOW_ALL=1 ;; + -h|--help) sed -n '2,60p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown argument: $1" >&2; echo "Usage: $0 [-a|--all]" >&2; exit 2 ;; + esac + shift +done + +die() { echo "ERROR: $*" >&2; exit 1; } + +# First non-blank line of a blob. +# +# Deliberately NOT `printf ... | sed -n '/./{p;q;}'`. sed quits after the first +# match and closes the pipe; on a real 550-entry table dump printf is still +# writing and takes SIGPIPE, so under `set -o pipefail` the whole command +# substitution returns 141 and `set -e` kills the script -- silently, with no +# output at all. That is the same class of failure this script exists to stop +# hiding, so it does not get to happen here. A plain read loop has no pipeline +# and no early close. +first_nonblank() { + local line + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + *[![:space:]]*) printf '%s\n' "$line"; return 0 ;; + esac + done </dev/null +} + +# ---------------------------------------------------------------- fetch data +BODY="" +SOURCE="" +if [ -n "${HAPROXY_TABLE_DUMP:-}" ]; then + [ -r "$HAPROXY_TABLE_DUMP" ] || die "HAPROXY_TABLE_DUMP is set but '$HAPROXY_TABLE_DUMP' is not readable." + BODY=$(cat "$HAPROXY_TABLE_DUMP") + SOURCE="file $HAPROXY_TABLE_DUMP" +else + [ -S "$SOCKET" ] || die "HAProxy CLI socket not found at $SOCKET (is HAProxy running, and are you inside the haproxy-manager container?)" + command -v socat >/dev/null 2>&1 || die "socat is not installed; cannot talk to $SOCKET" + + # Master socket form first, then the plain stats-socket form. + BODY=$(send_cmd "@1 show table $TABLE" || true) + SOURCE="socket $SOCKET (@1 show table $TABLE)" + if [ -z "${BODY//[[:space:]]/}" ] || body_is_rejected "$BODY"; then + FALLBACK=$(send_cmd "show table $TABLE" || true) + if [ -n "${FALLBACK//[[:space:]]/}" ] && ! body_is_rejected "$FALLBACK"; then + BODY="$FALLBACK" + SOURCE="socket $SOCKET (show table $TABLE)" + else + echo "ERROR: HAProxy rejected BOTH '@1 show table $TABLE' and 'show table $TABLE'." >&2 + echo " @1 response : $(first_nonblank "$BODY")" >&2 + echo " bare response: $(first_nonblank "$FALLBACK")" >&2 + echo " Check the socket is HAProxy's CLI and that the table '$TABLE' exists" >&2 + echo " (a config reload without the frontend would drop it)." >&2 + exit 1 + fi + fi +fi + +# ------------------------------------------------------------- header checks +HEADER=$(first_nonblank "$BODY") +case "$HEADER" in + "# table: $TABLE,"*) : ;; + *) + echo "ERROR: unexpected first line from '$SOURCE'." >&2 + echo " expected it to start with: # table: $TABLE," >&2 + echo " got : $HEADER" >&2 + exit 1 + ;; +esac + +TBL_SIZE=$(printf '%s\n' "$HEADER" | sed -n 's/.*size:\([0-9]*\).*/\1/p') +TBL_USED=$(printf '%s\n' "$HEADER" | sed -n 's/.*used:\([0-9]*\).*/\1/p') +[ -n "$TBL_SIZE" ] || TBL_SIZE="?" +[ -n "$TBL_USED" ] || TBL_USED="?" + +# ------------------------------------------------------------------- parsing +# awk emits: +# W \t : ... (one line, from first row) +# R \t \t \t +# and exits 1 after reporting any row missing an expected field. +PARSED="" +if ! PARSED=$(printf '%s\n' "$BODY" | awk -v fieldlist="${EXPECTED_FIELDS[*]}" -v sortfield="$SORT_FIELD" ' +BEGIN { + nf = split(fieldlist, F, " ") + sortidx = 1 + for (i = 1; i <= nf; i++) if (F[i] == sortfield) sortidx = i + wprinted = 0 +} +/^#/ { next } +!/key=/ { next } +{ + split("", val, " "); split("", win, " ") + for (i = 1; i <= NF; i++) { + tok = $i + p = index(tok, "=") + if (p == 0) continue + lhs = substr(tok, 1, p - 1) + rhs = substr(tok, p + 1) + b = index(lhs, "(") + if (b > 0) { + nm = substr(lhs, 1, b - 1) + win[nm] = substr(lhs, b + 1, length(lhs) - b - 1) + } else { + nm = lhs + win[nm] = "" + } + val[nm] = rhs + } + + missing = "" + for (i = 1; i <= nf; i++) if (!(F[i] in val)) missing = missing (missing == "" ? "" : ", ") F[i] + if (missing != "") { + printf "ERROR: stick table row is missing expected field(s): %s\n", missing > "/dev/stderr" + printf " offending row: %s\n", $0 > "/dev/stderr" + printf " this script expects the web table to store: %s\n", fieldlist > "/dev/stderr" + print " Those expectations and the templates/hap_listener.tpl `store` clause have DRIFTED." > "/dev/stderr" + print " Fix one or the other; refusing to print 0 for a counter HAProxy never reported." > "/dev/stderr" + exit 1 + } + if (!("key" in val)) { + printf "ERROR: stick table row has no key= field: %s\n", $0 > "/dev/stderr" + exit 1 + } + + if (!wprinted) { + line = "W" + for (i = 1; i <= nf; i++) { + w = win[F[i]] + if (w ~ /^[0-9]+$/) w = sprintf("%g", w / 1000); else w = "-" + line = line "\t" F[i] ":" w + } + print line + wprinted = 1 + } + + nonzero = 0 + row = "" + for (i = 1; i <= nf; i++) { + v = val[F[i]] + if (v + 0 != 0) nonzero = 1 + row = row "\t" v + } + printf "R\t%s\t%s\t%d%s\n", val[F[sortidx]] + 0, val["key"], nonzero, row +} +'); then + exit 1 +fi + +# ------------------------------------------------------------------ printing +WINSPEC=$(printf '%s\n' "$PARSED" | sed -n 's/^W\t//p' || true) + +echo "===================================================================" +echo " HAProxy edge IP rates — table '$TABLE' (current values only)" +echo "===================================================================" +echo "Source : $SOURCE" +echo "Tracked : ${TBL_USED} of ${TBL_SIZE} slots in use" +if [ "$SHOW_ALL" = "1" ]; then + echo "Filter : showing ALL tracked IPs" +else + echo "Filter : showing only IPs with a non-zero counter (use --all for every row)" +fi +echo + +# Column headers, with each counter's window rendered in SECONDS (HAProxy +# reports the window in milliseconds, e.g. conn_rate(10000) = 10s). +HDR=$(printf "%-18s" "IP Address") +i=0 +for f in "${EXPECTED_FIELDS[@]}"; do + w=$(printf '%s\n' "$WINSPEC" | tr '\t' '\n' | sed -n "s/^${f}://p") + if [ -n "$w" ] && [ "$w" != "-" ]; then + label="${f}/${w}s" + else + label="$f" + fi + HDR="$HDR $(printf '%18s' "$label")" + i=$((i + 1)) +done +echo "$HDR" +printf '%s\n' "$HDR" | sed 's/./-/g' + +ROWS=$(printf '%s\n' "$PARSED" | sed -n 's/^R\t//p' || true) +shown=0 +if [ -n "$ROWS" ]; then + while IFS=$'\t' read -r sortkey ip nonzero rest; do + [ -n "${ip:-}" ] || continue + if [ "$SHOW_ALL" != "1" ] && [ "$nonzero" = "0" ]; then + continue + fi + line=$(printf "%-18s" "$ip") + oldifs="$IFS"; IFS=$'\t' + # shellcheck disable=SC2086 + set -- $rest + IFS="$oldifs" + for v in "$@"; do + line="$line $(printf '%18s' "$v")" + done + echo "$line" + shown=$((shown + 1)) + done < <(printf '%s\n' "$ROWS" | sort -t"$(printf '\t')" -k1,1nr) +fi + +if [ "$shown" -eq 0 ]; then + if [ "$SHOW_ALL" = "1" ]; then + echo "(no IPs currently tracked)" + else + echo "(no IP currently has a non-zero counter — re-run with --all to list idle entries)" + fi +fi + +echo +echo "===================================================================" +echo "These are CURRENT values. The table keeps no history and no record of" +echo "past tarpits/denials. For actual enforcement events, read the access" +echo "log ON THE DOCKER HOST (it does not exist in this container):" +echo " grep -aE ' (PT|PR)--' /var/log/haproxy.log | tail -20 # tarpit / deny" +echo " grep -aE ' (429|403) ' /var/log/haproxy.log | tail -20 # rate-limit / block" +echo " grep -a 'cip=' /var/log/haproxy.log | tail -50 # one client" +echo " grep -a 'id=' /var/log/haproxy.log # one request reference" +echo +echo "Operator actions (via the MASTER CLI socket — the @1 prefix is required):" +echo " printf '@1 show table $TABLE key \\n' | socat stdio $SOCKET" +echo " printf '@1 set table $TABLE key data.http_req_rate 0\\n' | socat stdio $SOCKET" +echo " printf '@1 clear table $TABLE key \\n' | socat stdio $SOCKET # drop one entry" +echo " printf '@1 clear table $TABLE\\n' | socat stdio $SOCKET # drop ALL entries" +echo "===================================================================" diff --git a/scripts/show-tarpit-ips.sh b/scripts/show-tarpit-ips.sh index 491df09..6b22be9 100755 --- a/scripts/show-tarpit-ips.sh +++ b/scripts/show-tarpit-ips.sh @@ -1,123 +1,35 @@ -#!/bin/bash - -# Script to display IPs that have been tarpitted by HAProxy 3.0 -# Uses HAProxy stats socket to query stick-table data +#!/usr/bin/env bash # -# Usage in Docker container: -# docker exec -it haproxy-manager /haproxy/scripts/show-tarpit-ips.sh +# DEPRECATED SHIM — kept so existing docs/runbooks/muscle memory keep working. +# +# This script used to print a "Tarpitted IPs Report" with a "Scan Count" and a +# BLOCKED / SILENT-DROP / TARPIT status per IP, all derived from gpc0 and gpc1 +# stick-table columns. Those columns DO NOT EXIST: the `web` table +# (templates/hap_listener.tpl) stores only conn_cur, conn_rate, http_req_rate +# and http_err_rate. The old parser defaulted every missing field to 0, so the +# whole report was fabricated — every IP showed "Scan Count 0 / Normal" +# regardless of what it was actually doing. +# +# The stick table also keeps NO history, so nothing in it can identify who was +# tarpitted. Real enforcement events live in the access log ON THE DOCKER HOST +# at /var/log/haproxy.log (it does not exist inside this container): +# grep -aE ' (PT|PR)--' /var/log/haproxy.log | tail -20 # tarpit / deny +# grep -aE ' (429|403) ' /var/log/haproxy.log | tail -20 # rate-limit / block +# +# What IS knowable from the stick table — the current per-IP rates — is printed +# by show-edge-ip-rates.sh, which this shim now runs. -SOCKET="/tmp/haproxy-cli" +set -euo pipefail -# Check if socket exists -if [ ! -S "$SOCKET" ]; then - echo "Error: HAProxy socket not found at $SOCKET" - echo "Make sure HAProxy is running with stats socket enabled" - exit 1 -fi +cat >&2 <<'NOTE' +NOTE: show-tarpit-ips.sh is deprecated and cannot report tarpits. + The HAProxy stick table stores no history and no gpc0/gpc1 counters, so + the old "Scan Count"/"BLOCKED" columns were fabricated numbers. + Actual tarpit/deny events are in /var/log/haproxy.log ON THE HOST: + grep -aE ' (PT|PR)--' /var/log/haproxy.log | tail -20 + Running show-edge-ip-rates.sh instead (current rates, real values): -echo "===================================================================" -echo " HAProxy Tarpitted IPs Report " -echo "===================================================================" -echo -echo "Showing IPs tracked in the stick-table with scan detection counters:" -echo "(gpc0 = total scan attempts, gpc1 = escalation level)" -echo +NOTE -# In HAProxy 3.0, we need to use the proper process prefix -# The web frontend table is in the worker process, not master -# First check which process has the table -# Note: grep for actual worker line, not the header -PROCESS_ID=$(echo "show proc" | socat stdio "$SOCKET" 2>/dev/null | grep -E '^[0-9]+.*worker' | awk '{print $1}' | head -1) - -if [ -z "$PROCESS_ID" ]; then - echo "Error: Could not find HAProxy worker process" - echo "Try: echo 'show proc' | socat stdio $SOCKET" - exit 1 -fi - -# Show stick-table entries from the web frontend using the worker process -# Use printf to avoid bash history expansion issues with ! -printf "@!%s show table web\n" "${PROCESS_ID}" | socat stdio "$SOCKET" 2>/dev/null | { - # Skip the header line - read header - - # Check if we got an error or empty response - if echo "$header" | grep -q "No such table"; then - echo "Error: Table 'web' not found. HAProxy may need to be reloaded." - exit 1 - fi - - has_data=false - echo "IP Address | Scan Count | Level | HTTP Err Rate | Status" - echo "---------------------|------------|-------|---------------|------------------" - - # Process each line - while IFS= read -r line; do - # Skip empty lines and comments - if [ -z "$line" ] || echo "$line" | grep -q "^#"; then - continue - fi - - # HAProxy 3.0 format: 0x... key= use=... exp=... gpc0=... gpc1=... http_err_rate(10s)=... - if echo "$line" | grep -q "key="; then - has_data=true - - # Extract IP and counters - ip=$(echo "$line" | grep -o 'key=[^ ]*' | cut -d'=' -f2) - gpc0=$(echo "$line" | grep -o 'gpc0=[0-9]*' | cut -d'=' -f2) - gpc1=$(echo "$line" | grep -o 'gpc1=[0-9]*' | cut -d'=' -f2) - err_rate=$(echo "$line" | grep -o 'http_err_rate([^)]*=[0-9]*' | grep -o '[0-9]*$') - - # Set defaults if values are empty - gpc0=${gpc0:-0} - gpc1=${gpc1:-0} - err_rate=${err_rate:-0} - - # Determine status based on scan count and escalation - status="" - if [ "$gpc0" -ge 100 ]; then - status="BLOCKED (429)" - elif [ "$gpc0" -ge 60 ]; then - status="SILENT-DROP" - elif [ "$gpc0" -ge 40 ]; then - if [ "$gpc1" -ge 2 ]; then - status="SILENT-DROP (repeat)" - else - status="TARPIT 10s" - fi - elif [ "$gpc0" -ge 25 ]; then - status="TARPIT 10s" - else - status="Normal" - fi - - # Format output - printf "%-20s | %10s | %5s | %13s | %s\n" "$ip" "$gpc0" "$gpc1" "$err_rate/10s" "$status" - fi - done - - if [ "$has_data" = false ]; then - echo "(No IPs currently tracked - table is empty)" - fi -} - -echo -echo "===================================================================" -echo "Legend:" -echo " - Scan Count 25-39: Low scanner → TARPIT 10s delay" -echo " - Scan Count 40-59: Medium scanner → TARPIT 10s (1st), SILENT-DROP (repeat)" -echo " - Scan Count 60-99: High scanner → SILENT-DROP (immediate disconnect)" -echo " - Scan Count 100+: Critical scanner → BLOCKED (429 response)" -echo " - Burst (5+ in 10s): → TARPIT 10s (1st), SILENT-DROP (repeat)" -echo "===================================================================" -echo "Note: Only counts suspicious scripts/configs, NOT missing images/fonts/CSS" -echo "Note: IPs are tracked for 1 hour since last activity" -echo -echo "To clear a specific IP from the table:" -echo " printf '@!${PROCESS_ID} del table web key \\n' | socat stdio $SOCKET" -echo -echo "To clear all entries:" -echo " printf '@!${PROCESS_ID} clear table web\\n' | socat stdio $SOCKET" -echo -echo "Debug: Worker PID is ${PROCESS_ID}" -echo \ No newline at end of file +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec "$SCRIPT_DIR/show-edge-ip-rates.sh" "$@" diff --git a/scripts/test-stick-table-contract.py b/scripts/test-stick-table-contract.py new file mode 100644 index 0000000..7199679 --- /dev/null +++ b/scripts/test-stick-table-contract.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +"""Contract test: what the stick tables STORE vs what the consumers READ. + +Why this file exists +-------------------- +`/api/security/stats` and `scripts/show-tarpit-ips.sh` spent their whole +existence reporting "Scan Count", "offense count" and "BLOCKED" figures parsed +out of `gpc0` and `gpc1`. **No stick table in this repo has ever stored a +general-purpose counter.** Every one of those numbers was fabricated, and an +operator was making decisions on them. + +Nothing caught it, because each layer failed silently in a different way: + + * `int(parts[3])` on a positional split hit `exp=368842`, raised ValueError, + and the loop `continue`d -- so the endpoint answered `active_threats: 0` + with an empty list. "No threats" and "the parser is broken" looked + identical. + * The command went to the MASTER CLI socket without the `@1` worker prefix. + HAProxy answered `Unknown command: 'show', but maybe one of the following + ones is a better match: ...` and **socat still exited 0**, so the + `returncode != 0` guard never fired. The reported `total_tracked_ips` was + the line count of that help text (8) while the real table held 388 entries. + * The shell consumers wrote `gpc0=${gpc0:-0}`, so a field that does not exist + rendered as a confident zero. + +The durable fix is not "parse better" -- it is making the template and its +consumers unable to drift apart without something going red. That is this file. + +What it enforces +---------------- + 1. `STICK_TABLE_FIELD_CONTRACT` in haproxy_manager.py equals, exactly and in + both directions, the `store` clauses in the rendered templates. + 2. Every shell consumer's `EXPECTED_FIELDS=(...)` array equals the contract. + 3. A captured sample of REAL `show table` output from the live edge parses to + exactly the contract's fields plus the entry metadata -- so the contract + describes reality, not just itself. + 4. The loud-failure behaviour: `read_stick_table()` RAISES on a rejected + command, on a non-table response, and on a row missing a contract field. + It must never answer zeros. Guard 4 is the one that would have caught the + original bug on day one. + 5. No `store` clause names a general-purpose counter, and no template tracks + one -- the state this repo is actually in, asserted rather than assumed. + +Assertions about the TEMPLATES go through `rule_lines()`, which strips comments +before matching. These templates quote their own rules in prose at length; a +bare `assertIn` over the rendered text passes just as happily against a rule +that has been commented out. Same lesson, and same helper, as +scripts/test-wpadmin-gate.py. + +Runs fully offline -- no HAProxy, no socket, no network. + +Running +------- + python3 scripts/test-stick-table-contract.py +""" + +import os +import re +import sys +import logging +import tempfile +import unittest + +MODULE_DIR = os.path.abspath( + os.environ.get('HAPROXY_MANAGER_DIR', + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +) +os.chdir(MODULE_DIR) +sys.path.insert(0, MODULE_DIR) + +_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-') +_real_file_handler = logging.FileHandler +logging.FileHandler = ( + lambda filename, *a, **kw: _real_file_handler( + os.path.join(_LOG_DIR, os.path.basename(filename)), *a, **kw) +) + +import haproxy_manager # noqa: E402 + + +# --------------------------------------------------------------------------- +# A captured sample of REAL output, so this test can assert against reality +# without a live socket. +# +# Provenance: `echo "@1 show table web" | socat stdio /tmp/haproxy-cli` inside +# the haproxy-manager container on whp01, 2026-08-22, HAProxy 3.0.11. Rows +# trimmed for length; format byte-for-byte as emitted. +# +# Note what is and is NOT here: no gpc0, no gpc1, no gpc(N), no gpc_rate, no +# glitch_rate. Note also that field windows come back in MILLISECONDS +# (`conn_rate(10000)`), not the `10s` the template writes -- a consumer that +# labels the raw number "10s" is off by a factor of 1000. +# --------------------------------------------------------------------------- +LIVE_TABLE_SAMPLE = """\ +# table: web, type: ip, size:204800, used:388 +0x7f6e5447e728: key=17.58.57.102 use=0 exp=368842 shard=0 conn_rate(10000)=0 conn_cur=0 http_req_rate(10000)=0 http_err_rate(30000)=0 +0x7f6e541cf7c8: key=43.173.182.9 use=0 exp=171404 shard=0 conn_rate(10000)=0 conn_cur=0 http_req_rate(10000)=0 http_err_rate(30000)=0 +0x7f6e5547f528: key=95.129.255.180 use=0 exp=590639 shard=0 conn_rate(10000)=1 conn_cur=0 http_req_rate(10000)=1 http_err_rate(30000)=0 +0x7f6e5447e488: key=5.9.105.254 use=0 exp=556277 shard=0 conn_rate(10000)=0 conn_cur=0 http_req_rate(10000)=0 http_err_rate(30000)=1 +""" + +# The verbatim reply the MASTER CLI socket gives to an unprefixed worker +# command. Captured the same way. socat exits 0 on this -- which is the entire +# reason haproxy_cli() inspects the body. +MASTER_SOCKET_REJECTION = """\ +Unknown command: 'show', but maybe one of the following ones is a better match: + show cli level : display the level of the current CLI session + show cli sockets : dump list of cli sockets + show proc : show processes status + show startup-logs : report logs emitted during HAProxy startup + show version : show version of the current process + help [] : list matching or all commands + prompt [timed] : toggle interactive mode with prompt + quit : disconnect +""" + +# Anything matching these in a `store` clause is a general-purpose counter. +GPC_PATTERN = re.compile(r'\bgpc|glitch') + +# Shell consumers that must declare their field expectations as a single +# EXPECTED_FIELDS array, and the table each one reads. +SHELL_CONSUMERS = { + 'scripts/show-edge-ip-rates.sh': 'web', +} + +EXPECTED_FIELDS_RE = re.compile(r'^\s*EXPECTED_FIELDS=\(([^)]*)\)\s*$', re.M) + + +def rule_lines(cfg, needle): + """Comment-stripped config lines containing `needle`. + + A line that is entirely a comment is dropped; a line mixing config with a + trailing comment is truncated at the first ' #' before matching. Without + this, every assertion below would pass against a rule that had been + commented out but whose text survived in the surrounding prose -- and these + templates quote their own rules in prose constantly. See + scripts/test-wpadmin-gate.py, where a mutation audit proved the point. + """ + out = [] + for raw in cfg.split('\n'): + stripped = raw.strip() + if stripped and not stripped.startswith('#'): + code = stripped.split(' #', 1)[0].rstrip() + if code and needle in code: + out.append(code) + return out + + +def render_listener(): + return haproxy_manager.template_env.get_template('hap_listener.tpl').render( + crt_path='/etc/haproxy/certs', + suspension_enabled=False, + coraza_spoe_backend=None, + ) + + +def render_security_tables(): + return haproxy_manager.template_env.get_template('hap_security_tables.tpl').render() + + +def stick_tables_from(cfg): + """{table name: (field, ...)} for every stick-table declared in `cfg`. + + A stick-table takes the name of the frontend/backend/listen section that + declares it -- that name is what `show table ` wants, so the section + header is part of the contract, not incidental. Section headers and + stick-table lines are both read comment-stripped. + """ + tables = {} + section = None + for raw in cfg.split('\n'): + stripped = raw.strip() + if not stripped or stripped.startswith('#'): + continue + code = stripped.split(' #', 1)[0].rstrip() + header = re.match(r'^(frontend|backend|listen)\s+(\S+)', code) + if header: + section = header.group(2) + continue + if code.startswith('stick-table'): + store = re.search(r'\bstore\s+(\S+)', code) + if not store: + raise AssertionError( + 'stick-table in section %r has no `store` clause: %r' + % (section, code)) + if section is None: + raise AssertionError( + 'stick-table declared outside any section: %r' % code) + # store is a comma-separated list; each item is name or name(window) + fields = tuple(item.split('(')[0] + for item in store.group(1).split(',')) + tables[section] = fields + return tables + + +def all_template_stick_tables(): + tables = {} + for cfg in (render_listener(), render_security_tables()): + for name, fields in stick_tables_from(cfg).items(): + if name in tables: + raise AssertionError('stick table %r declared twice' % name) + tables[name] = fields + return tables + + +class StickTableContract(unittest.TestCase): + """Guard 1 + 5: the templates and the Python contract, held together.""" + + def setUp(self): + self.templates = all_template_stick_tables() + self.contract = haproxy_manager.STICK_TABLE_FIELD_CONTRACT + + def test_templates_actually_declare_stick_tables(self): + """Guard the guard: an empty parse would make every other check vacuous.""" + self.assertTrue(self.templates, + 'parsed no stick tables out of the templates at all -- ' + 'stick_tables_from() is broken, not the templates') + + def test_same_table_names(self): + self.assertEqual( + sorted(self.templates), sorted(self.contract), + 'STICK_TABLE_FIELD_CONTRACT and the templates disagree on WHICH ' + 'stick tables exist. Add/remove the table in both places.') + + def test_same_fields_per_table(self): + for table in sorted(self.templates): + with self.subTest(table=table): + self.assertEqual( + sorted(self.templates[table]), + sorted(self.contract.get(table, ())), + "stick table %r stores %s but STICK_TABLE_FIELD_CONTRACT " + "claims %s. Whichever is wrong, a consumer is about to read " + "a field that is never populated -- which is the bug this " + "test exists for." % (table, + list(self.templates[table]), + list(self.contract.get(table, ())))) + + def test_web_table_is_the_one_the_api_reads(self): + self.assertIn('web', self.contract) + self.assertIn('web', self.templates) + + def test_no_general_purpose_counters_are_stored(self): + """The state of the world today, asserted rather than assumed. + + If a gpc/glitch counter is ever genuinely added to a template, this + test is the place to update -- and updating it forces whoever does so + to also add the field to STICK_TABLE_FIELD_CONTRACT (test_same_fields_ + per_table) and to the shell consumers (test_shell_consumers_match_ + contract). That chain is the point: a counter cannot appear in a + consumer without existing in the table, and cannot appear in the table + without the consumers being updated. + """ + for table, fields in self.templates.items(): + for field in fields: + self.assertIsNone( + GPC_PATTERN.search(field), + 'stick table %r now stores %r. Update this test, ' + 'STICK_TABLE_FIELD_CONTRACT, and every consumer.' + % (table, field)) + + def test_track_sc_counters_have_a_table_each(self): + """Every `track-scN ... table X` names a table that really exists. + + A typo here is invisible to `haproxy -c` only in the sense that it is + NOT -- but it is invisible to the consumers, which would query a table + that is never written. + """ + cfg = render_listener() + for line in rule_lines(cfg, 'track-sc'): + named = re.search(r'\btable\s+(\S+)', line) + if named: + self.assertIn( + named.group(1), self.templates, + 'track-sc rule references undeclared table %r: %r' + % (named.group(1), line)) + + def test_sc_counter_indices_fit_haproxys_limit(self): + """sc0/sc1/sc2 are all HAProxy gives us by default. + + `tune.stick-counters` defaults to 3. A `track-sc3` without raising it + is a config-time failure, and the templates' own comments assume the + limit -- so assert it rather than leaving it as folklore. + """ + cfg = render_listener() + '\n' + render_security_tables() + raised = rule_lines(cfg, 'tune.stick-counters') + limit = 3 + if raised: + limit = int(re.search(r'(\d+)', raised[-1]).group(1)) + for line in rule_lines(cfg, 'track-sc'): + idx = int(re.search(r'track-sc(\d+)', line).group(1)) + self.assertLess( + idx, limit, + 'track-sc%d exceeds tune.stick-counters (%d): %r' + % (idx, limit, line)) + + +class ShellConsumerContract(unittest.TestCase): + """Guard 2: the shell consumers cannot drift from the contract.""" + + def test_shell_consumers_match_contract(self): + for path, table in sorted(SHELL_CONSUMERS.items()): + with self.subTest(script=path): + full = os.path.join(MODULE_DIR, path) + self.assertTrue(os.path.exists(full), + '%s is missing; it is a declared consumer of ' + 'stick table %r' % (path, table)) + with open(full) as fh: + src = fh.read() + m = EXPECTED_FIELDS_RE.search(src) + self.assertIsNotNone( + m, '%s must declare its field expectations once as a ' + 'single-line `EXPECTED_FIELDS=(a b c)` array so this ' + 'test can hold it to the template' % path) + declared = sorted(m.group(1).split()) + self.assertEqual( + declared, + sorted(haproxy_manager.STICK_TABLE_FIELD_CONTRACT[table]), + '%s reads %s but stick table %r stores %s' + % (path, declared, table, + sorted(haproxy_manager.STICK_TABLE_FIELD_CONTRACT[table]))) + + def test_retired_script_no_longer_parses_phantom_counters(self): + """show-tarpit-ips.sh may explain gpc0/gpc1; it may not extract them. + + The shim is allowed -- encouraged -- to name the fields in prose so an + operator who runs it learns why its numbers went away. What it must not + do is go back to pulling values out of them. + """ + path = os.path.join(MODULE_DIR, 'scripts/show-tarpit-ips.sh') + if not os.path.exists(path): + self.skipTest('show-tarpit-ips.sh has been removed outright') + with open(path) as fh: + lines = fh.readlines() + for raw in lines: + stripped = raw.strip() + if not stripped or stripped.startswith('#'): + continue + code = stripped.split(' #', 1)[0] + self.assertIsNone( + re.search(r"grep -o ['\"]?gpc|gpc[0-9]*=\$|sc_get_gpc|sc-inc-gpc", code), + 'show-tarpit-ips.sh is extracting a general-purpose counter ' + 'again: %r' % stripped) + + +class LiveSampleParses(unittest.TestCase): + """Guard 3: the contract describes real HAProxy output, not just itself.""" + + def test_header_parses(self): + header, entries = self._read() + self.assertEqual(header['name'], 'web') + self.assertEqual(header['type'], 'ip') + self.assertEqual(header['size'], 204800) + self.assertEqual(header['used'], 388) + self.assertEqual(len(entries), 4) + + def test_sample_fields_are_exactly_contract_plus_metadata(self): + expected = set(haproxy_manager.STICK_TABLE_FIELD_CONTRACT['web']) + expected |= set(haproxy_manager.STICK_TABLE_ENTRY_META) + _, entries = self._read() + for line, fields in entries: + self.assertEqual( + set(fields), expected, + 'real `show table web` output carries %s, contract+metadata ' + 'expects %s. Row: %r' + % (sorted(fields), sorted(expected), line)) + + def test_key_is_the_ip_not_the_allocation_pointer(self): + """The original bug read parts[0] -- the `0x...:` pointer -- as the IP.""" + _, entries = self._read() + ips = [f['key']['value'] for _, f in entries] + self.assertIn('95.129.255.180', ips) + for ip in ips: + self.assertFalse(ip.startswith('0x'), + 'parsed a memory address as an IP: %r' % ip) + + def test_windows_are_milliseconds(self): + """HAProxy reports `conn_rate(10000)` for a `conn_rate(10s)` store. + + Asserted because labelling that raw 10000 as "10s" (or as seconds) is + an easy and completely silent way to be wrong by 1000x in the panel. + """ + _, entries = self._read() + _, fields = entries[0] + self.assertEqual(fields['conn_rate']['window_ms'], 10000) + self.assertEqual(fields['http_req_rate']['window_ms'], 10000) + self.assertEqual(fields['http_err_rate']['window_ms'], 30000) + self.assertIsNone(fields['conn_cur']['window_ms'], + 'conn_cur is a gauge, not a rate; it has no window') + + def test_values_are_the_real_ones(self): + _, entries = self._read() + by_ip = {f['key']['value']: f for _, f in entries} + self.assertEqual(by_ip['95.129.255.180']['http_req_rate']['value'], '1') + self.assertEqual(by_ip['5.9.105.254']['http_err_rate']['value'], '1') + self.assertEqual(by_ip['17.58.57.102']['http_req_rate']['value'], '0') + + def _read(self): + return _read_table_from(LIVE_TABLE_SAMPLE) + + +class FailsLoudly(unittest.TestCase): + """Guard 4: every way this can go wrong must raise, never return zeros. + + This is the guard that would have caught the original bug immediately. Each + case below is a real response the old code accepted silently. + """ + + def test_master_socket_rejection_is_not_data(self): + """The exact reply that used to be reported as `total_tracked_ips: 8`.""" + with self.assertRaises(haproxy_manager.HaproxyCliError) as ctx: + _read_table_from(MASTER_SOCKET_REJECTION) + self.assertIn('not a stick-table dump', str(ctx.exception)) + + def test_socat_exit_zero_does_not_mean_success(self): + """haproxy_cli() must reject on the BODY, not the exit status. + + socat returns 0 for every response above -- the rejection is only ever + visible in the text. + """ + self.assertTrue( + haproxy_manager._cli_response_is_error(MASTER_SOCKET_REJECTION)) + self.assertTrue( + haproxy_manager._cli_response_is_error('No such table\n')) + self.assertTrue( + haproxy_manager._cli_response_is_error('Permission denied\n')) + self.assertFalse( + haproxy_manager._cli_response_is_error(LIVE_TABLE_SAMPLE)) + + def test_missing_contract_field_raises_and_names_it(self): + """A field the table stopped storing must not silently become 0.""" + degraded = LIVE_TABLE_SAMPLE.replace(' http_err_rate(30000)=0', '') + with self.assertRaises(haproxy_manager.HaproxyCliError) as ctx: + _read_table_from(degraded) + msg = str(ctx.exception) + self.assertIn('http_err_rate', msg, + 'the error must name the missing field') + self.assertIn('drifted', msg, + 'the error must say what actually went wrong') + + def test_empty_response_raises(self): + with self.assertRaises(haproxy_manager.HaproxyCliError): + _read_table_from('') + + def test_row_without_key_raises(self): + broken = LIVE_TABLE_SAMPLE.replace('key=17.58.57.102 ', '') + with self.assertRaises(haproxy_manager.HaproxyCliError) as ctx: + _read_table_from(broken) + self.assertIn('no key=', str(ctx.exception)) + + def test_unknown_table_raises_before_touching_the_socket(self): + """Querying a table with no contract is a programming error, not a 0.""" + with self.assertRaises(haproxy_manager.HaproxyCliError) as ctx: + haproxy_manager.read_stick_table('does_not_exist') + self.assertIn('no field contract', str(ctx.exception)) + + def test_empty_table_is_not_an_error(self): + """A table with zero entries is a legitimate, distinguishable result.""" + header, entries = _read_table_from( + '# table: web, type: ip, size:204800, used:0\n') + self.assertEqual(header['used'], 0) + self.assertEqual(entries, []) + + +def _read_table_from(response, table='web'): + """Run read_stick_table() against a canned response instead of a socket.""" + real = haproxy_manager.haproxy_cli + haproxy_manager.haproxy_cli = lambda cmd, worker=False, timeout=None: response + try: + return haproxy_manager.read_stick_table(table) + finally: + haproxy_manager.haproxy_cli = real + + +if __name__ == '__main__': + unittest.main(verbosity=2)