diff --git a/VERSION b/VERSION index 0657c95..8ac114e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2026.07.3 +2026.08.1 diff --git a/haproxy_manager.py b/haproxy_manager.py index a70f812..b7ef7ef 100644 --- a/haproxy_manager.py +++ b/haproxy_manager.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta import json import ipaddress import shutil +import stat import tempfile import threading import time @@ -105,6 +106,12 @@ HAPROXY_CONFIG_PATH = '/etc/haproxy/haproxy.cfg' HAPROXY_BACKUP_PATH = '/etc/haproxy/haproxy.cfg.backup' BLOCKED_IPS_MAP_PATH = '/etc/haproxy/blocked_ips.map' BLOCKED_IPS_MAP_BACKUP_PATH = '/etc/haproxy/blocked_ips.map.backup' +# Coraza SPOE engine file. `haproxy -c` parses this too (the frontend's +# `filter spoe engine coraza config ` line points at it), so it is part +# of the same restorable config set as haproxy.cfg — rolling back haproxy.cfg +# while leaving a broken coraza-spoe.cfg behind still fails validation. +CORAZA_SPOE_CONFIG_PATH = '/etc/haproxy/coraza-spoe.cfg' +CORAZA_SPOE_BACKUP_PATH = '/etc/haproxy/coraza-spoe.cfg.backup' HAPROXY_SOCKET_PATH = '/var/run/haproxy.sock' SSL_CERTS_DIR = '/etc/haproxy/certs' # Stable per-host secret for QUIC Retry/address-validation tokens. Lives in the @@ -1795,6 +1802,21 @@ def generate_config(): config_parts = [] + # Snapshot the last-known-good config BEFORE anything below touches a + # file in /etc/haproxy. Everything this function writes (haproxy.cfg, + # blocked_ips.map, coraza-spoe.cfg) is validated as one set by + # `haproxy -c`, so the rollback point has to predate the first of them. + # Taking it here (rather than inside reload_haproxy_safely(), which runs + # after the writes) is what makes rollback real - see create_backup(). + backup_ok, backup_status = create_backup() + if not backup_ok: + # Could not even attempt a snapshot (I/O error). Writing a new + # config now would leave us with no way back, so refuse. + raise Exception( + "Refusing to regenerate config: failed to back up the current " + "configuration, so a failed change could not be rolled back" + ) + # Optional Coraza WAF integration. When HAPROXY_CORAZA_SPOE_BACKEND is # set on the haproxy-manager container, we render an extra TCP backend # pointing at a coraza-spoa sidecar AND inject a `filter spoe ...` line @@ -1984,25 +2006,21 @@ backend default-backend # how the file was authored. if not coraza_spoe_cfg.endswith('\n'): coraza_spoe_cfg += '\n' - coraza_spoe_path = '/etc/haproxy/coraza-spoe.cfg' - with open(coraza_spoe_path, 'w') as f: - f.write(coraza_spoe_cfg) - logger.info(f"Coraza SPOE engine config written to {coraza_spoe_path} " + write_config_atomically(CORAZA_SPOE_CONFIG_PATH, coraza_spoe_cfg) + logger.info(f"Coraza SPOE engine config written to " + f"{CORAZA_SPOE_CONFIG_PATH} " f"(SPOA target: {coraza_spoe_backend})") - # Write complete configuration to tmp - temp_config_path = "/etc/haproxy/haproxy.cfg" - config_content = '\n'.join(config_parts) logger.debug("Generated HAProxy configuration") - # Write complete configuration to tmp - # Write new configuration to file - with open(HAPROXY_CONFIG_PATH, 'w') as f: - f.write(config_content) - + # Write new configuration to file (atomically - a truncated haproxy.cfg + # is as fatal as an invalid one). The rollback point was taken above, + # before this write. + write_config_atomically(HAPROXY_CONFIG_PATH, config_content) + # Use safe reload with validation and rollback - success, message = reload_haproxy_safely() + success, message = reload_haproxy_safely(backup_status=backup_status) if success: logger.info("Configuration generated and HAProxy reloaded safely") log_operation('generate_config', True, 'Configuration generated and HAProxy reloaded safely') @@ -2019,63 +2037,306 @@ backend default-backend traceback.print_exc() raise -def create_backup(): - """Create backup of current config and map files""" +# --------------------------------------------------------------------------- +# Config backup / rollback +# --------------------------------------------------------------------------- +# Rollback only works if the backup predates the write it is supposed to undo. +# Until 2026-08 create_backup() ran from inside reload_haproxy_safely(), i.e. +# AFTER generate_config() had already overwritten haproxy.cfg — so the "backup" +# was a copy of the new (possibly broken) config and restore_backup() restored +# the same broken bytes. The advertised rollback was a no-op and a fatal +# haproxy.cfg persisted on disk, where start_haproxy() refuses to launch (the +# June 2026 missing-template incident). create_backup() must now be called by +# the writer, BEFORE the first byte is written. + +# Statuses returned by create_backup() that mean a rollback target exists. +_ROLLBACK_AVAILABLE_STATUSES = ('created', 'kept_previous') + + +def _files_identical(path_a, path_b): + """Byte-compare two files. + + Deliberately not filecmp.cmp(): it memoises on (size, mtime), and + shutil.copy2() preserves mtime, so a stale cache entry could report a + changed config as unchanged. These files are small; read them. + """ try: - if os.path.exists(HAPROXY_CONFIG_PATH): - shutil.copy2(HAPROXY_CONFIG_PATH, HAPROXY_BACKUP_PATH) - if os.path.exists(BLOCKED_IPS_MAP_PATH): - shutil.copy2(BLOCKED_IPS_MAP_PATH, BLOCKED_IPS_MAP_BACKUP_PATH) - logger.info("Backups created successfully") - return True + if os.path.getsize(path_a) != os.path.getsize(path_b): + return False + with open(path_a, 'rb') as fa, open(path_b, 'rb') as fb: + while True: + chunk_a = fa.read(65536) + chunk_b = fb.read(65536) + if chunk_a != chunk_b: + return False + if not chunk_a: + return True + except OSError: + return False + + +def _config_set_matches_backup(): + """True if every live config file is byte-identical to its backup copy. + + After a successful reload the live set has already been recorded as + known-good (see promote_current_config_to_backup()), which is the common + case at the start of the next generation. Recognising it lets create_backup() + skip both the re-validation and the copy - worth doing because + `haproxy -c` on an edge with hundreds of certificates is not free and + generate_config() runs synchronously inside customer-facing API calls. + """ + for live_path, backup_path in _config_backup_pairs(): + if os.path.exists(live_path) != os.path.exists(backup_path): + return False + if (os.path.exists(live_path) + and not _files_identical(live_path, backup_path)): + return False + return True + + +def _config_backup_pairs(): + """(live, backup) pairs forming one restorable config set. + + Built at call time rather than at import so the module-level path constants + stay patchable (tests, alternate deployments). + """ + return ( + (HAPROXY_CONFIG_PATH, HAPROXY_BACKUP_PATH), + (BLOCKED_IPS_MAP_PATH, BLOCKED_IPS_MAP_BACKUP_PATH), + (CORAZA_SPOE_CONFIG_PATH, CORAZA_SPOE_BACKUP_PATH), + ) + + +def write_config_atomically(path, content): + """Write content to path via temp file + rename. + + A half-written haproxy.cfg (disk full, container killed mid-write) is just + as fatal as an invalid one and is invisible to the caller. os.replace() is + atomic within a filesystem, so the file on disk is always either the whole + old config or the whole new one — never a truncated hybrid. This also keeps + the "existing config is already broken" case from being self-inflicted. + """ + directory = os.path.dirname(path) or '.' + # Preserve the mode of the file we are replacing; mkstemp defaults to 0600 + # and HAProxy config files are conventionally 0644. + try: + mode = stat.S_IMODE(os.stat(path).st_mode) + except OSError: + mode = 0o644 + fd, tmp_path = tempfile.mkstemp( + dir=directory, prefix=os.path.basename(path) + '.', suffix='.tmp' + ) + try: + with os.fdopen(fd, 'w') as f: + f.write(content) + f.flush() + os.fsync(f.fileno()) + os.chmod(tmp_path, mode) + os.replace(tmp_path, path) + except Exception: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +def create_backup(require_valid=True): + """Snapshot the CURRENT on-disk config set as the rollback point. + + MUST be called BEFORE the new configuration is written — see the module + comment above. Calling it afterwards silently disarms rollback. + + require_valid=True (default) refuses to promote a config that HAProxy + already rejects. Backing up a broken config would make "rollback" mean + "restore a different broken config"; keeping the older, validated backup + instead means a rollback always lands on something HAProxy will actually + start with. Cost is one `haproxy -c` run per config generation. + + Returns (ok, status): + ok=False, status='error' - the copy itself failed; caller decides. + status='created' - backup now holds the current config. + status='kept_previous' - current config missing or invalid; the + existing (older, good) backup was kept. + status='unavailable' - nothing to roll back to at all (first + run, or broken config and no prior + backup). Rollback is NOT possible. + """ + try: + snapshot_ok = True + reason = None + + if not os.path.exists(HAPROXY_CONFIG_PATH): + snapshot_ok = False + reason = 'no existing HAProxy config on disk (first run?)' + elif _config_set_matches_backup(): + # The backup already IS the current config, recorded when it last + # loaded successfully. Nothing to copy and nothing to re-validate. + logger.debug("Config backup already matches the live config") + return True, 'created' + elif require_valid: + status, msg = validate_config_file(HAPROXY_CONFIG_PATH) + if status == 'invalid': + snapshot_ok = False + reason = f'current config on disk does not validate: {msg}' + elif status == 'unavailable': + # The validator itself could not run (no haproxy binary, etc). + # That is NOT evidence the config is bad, and refusing to back + # up would leave us with no rollback target at all, so fall + # back to last-written semantics and say so loudly. + logger.warning( + f"Could not verify current config before backup ({msg}); " + "backing it up unverified" + ) + + if not snapshot_ok: + if os.path.exists(HAPROXY_BACKUP_PATH): + logger.warning( + f"Not refreshing config backup: {reason}. Keeping the " + f"existing backup at {HAPROXY_BACKUP_PATH} as the rollback " + "target." + ) + return True, 'kept_previous' + logger.error( + f"No config backup could be taken: {reason}, and no previous " + f"backup exists at {HAPROXY_BACKUP_PATH}. ROLLBACK IS NOT " + "AVAILABLE for this configuration change." + ) + return True, 'unavailable' + + for live_path, backup_path in _config_backup_pairs(): + if os.path.exists(live_path): + shutil.copy2(live_path, backup_path) + logger.info("Backup of last-known-good config created successfully") + return True, 'created' except Exception as e: logger.error(f"Failed to create backup: {e}") - return False + return False, 'error' -def restore_backup(): - """Restore from backup files""" +def promote_current_config_to_backup(): + """Record the live config as the known-good rollback target. + + Called ONLY after the config has both validated and been loaded by HAProxy, + so "backup" really means "the last configuration this box was running". + Must never be called before a reload attempt: doing so would make the + backup a copy of the config we may still have to roll back from - the same + class of bug as backing up after the write. + + Without this, a box whose very first generation succeeded has no rollback + target at all until its second successful generation, and any corruption of + haproxy.cfg in between leaves nothing to recover to. + """ try: - if os.path.exists(HAPROXY_BACKUP_PATH): - shutil.copy2(HAPROXY_BACKUP_PATH, HAPROXY_CONFIG_PATH) - if os.path.exists(BLOCKED_IPS_MAP_BACKUP_PATH): - shutil.copy2(BLOCKED_IPS_MAP_BACKUP_PATH, BLOCKED_IPS_MAP_PATH) - logger.info("Backups restored successfully") + for live_path, backup_path in _config_backup_pairs(): + if os.path.exists(live_path): + shutil.copy2(live_path, backup_path) + logger.debug("Known-good config backup updated after successful reload") return True except Exception as e: - logger.error(f"Failed to restore backup: {e}") + # Non-fatal: the config is live and working, we just failed to record + # it. Loud, because the next change now has a staler rollback target. + logger.error(f"Failed to record known-good config backup: {e}") return False -def validate_haproxy_config(): - """Validate HAProxy configuration file""" - try: - result = subprocess.run(['haproxy', '-c', '-f', HAPROXY_CONFIG_PATH], - capture_output=True, text=True) - if result.returncode == 0: - logger.info("HAProxy configuration validation passed") - return True, None - else: - error_msg = f"HAProxy configuration validation failed: {result.stderr}" - logger.error(error_msg) - return False, error_msg - except Exception as e: - error_msg = f"Error validating HAProxy config: {e}" - logger.error(error_msg) - return False, error_msg -def reload_haproxy_safely(): - """Safely reload HAProxy with validation and rollback""" +def restore_backup(): + """Restore the backed-up config set over the live files. + + Returns (restored, message). restored=False means NOTHING was rolled back + and the live config is still whatever the failed change left on disk — + callers MUST surface that difference, it is the difference between "we + recovered" and "this edge is sitting on a config HAProxy will not load". + """ + if not os.path.exists(HAPROXY_BACKUP_PATH): + msg = (f"No config backup at {HAPROXY_BACKUP_PATH} - cannot roll back; " + f"{HAPROXY_CONFIG_PATH} still holds the failed configuration") + logger.critical(msg) + return False, msg try: - # Create backup before changes - if not create_backup(): - return False, "Failed to create backup" - + for live_path, backup_path in _config_backup_pairs(): + if os.path.exists(backup_path): + shutil.copy2(backup_path, live_path) + msg = f"Configuration restored from backup ({HAPROXY_BACKUP_PATH})" + logger.info(msg) + return True, msg + except Exception as e: + msg = (f"Failed to restore backup: {e} - {HAPROXY_CONFIG_PATH} may hold " + "a broken configuration") + logger.critical(msg) + return False, msg + + +def validate_config_file(config_path): + """Run `haproxy -c` against config_path. + + Returns (status, message) with status one of: + 'valid' - HAProxy parsed the file successfully + 'invalid' - HAProxy rejected it (message carries stderr) + 'unavailable' - the validator could not be run at all (binary missing, + timeout, ...). Deliberately distinct from 'invalid': + it tells us nothing about the config. + """ + try: + result = subprocess.run(['haproxy', '-c', '-f', config_path], + capture_output=True, text=True) + except Exception as e: + return 'unavailable', f"Error validating HAProxy config: {e}" + if result.returncode == 0: + return 'valid', None + return 'invalid', f"HAProxy configuration validation failed: {result.stderr}" + + +def validate_haproxy_config(): + """Validate the live HAProxy configuration file. Returns (is_valid, error).""" + status, message = validate_config_file(HAPROXY_CONFIG_PATH) + if status == 'valid': + logger.info("HAProxy configuration validation passed") + return True, None + logger.error(message) + return False, message + +def reload_haproxy_safely(backup_status=None): + """Safely reload HAProxy with validation and rollback. + + PRECONDITION: the caller must already have called create_backup() BEFORE + writing the new config, and pass the status it returned. This function runs + after the new config is on disk, so it cannot take a meaningful backup + itself — doing so is exactly the bug this contract exists to prevent. + + backup_status=None means the caller did not take a pre-write backup. We do + NOT create one here (that would overwrite a genuinely good backup with the + unverified new config); we log it and fall back to whatever backup already + exists on disk. + """ + try: + if backup_status is None: + logger.error( + "reload_haproxy_safely() called without a pre-write backup " + "status - rollback will fall back to whatever backup already " + "exists on disk. Callers must call create_backup() BEFORE " + "writing the new configuration." + ) + elif backup_status not in _ROLLBACK_AVAILABLE_STATUSES: + logger.warning( + f"Proceeding with reload without a rollback target " + f"(backup status: {backup_status})" + ) + # Validate new configuration is_valid, error_msg = validate_haproxy_config() if not is_valid: # Restore backup on validation failure - restore_backup() + restored, restore_msg = restore_backup() + if not restored: + logger.critical( + "Config validation failed AND rollback was not possible - " + f"{HAPROXY_CONFIG_PATH} holds an invalid configuration that " + "HAProxy will refuse to start with" + ) + return False, (f"Config validation failed: {error_msg} | " + f"ROLLBACK FAILED: {restore_msg}") return False, f"Config validation failed: {error_msg}" - + # Attempt reload if is_process_running('haproxy'): # Use HAProxy stats socket for graceful reload @@ -2094,20 +2355,28 @@ def reload_haproxy_safely(): if reload_result.returncode == 0: logger.info("HAProxy reloaded successfully") + # Now - and only now - is this config known good. + promote_current_config_to_backup() return True, "HAProxy reloaded successfully" else: # Reload failed, restore backup - restore_backup() - # Try to reload with backup config - subprocess.run('echo "reload" | socat stdio /tmp/haproxy-cli', - shell=True, capture_output=True) + restored, restore_msg = restore_backup() + if restored: + # Try to reload with the restored (known-good) config + subprocess.run( + 'echo "reload" | socat stdio /tmp/haproxy-cli', + shell=True, capture_output=True) error_msg = f"HAProxy reload failed: {reload_result.stderr}" + if not restored: + error_msg += f" | ROLLBACK FAILED: {restore_msg}" logger.error(error_msg) return False, error_msg except Exception as e: # Critical error during reload, restore backup - restore_backup() + restored, restore_msg = restore_backup() error_msg = f"Critical error during reload: {e}" + if not restored: + error_msg += f" | ROLLBACK FAILED: {restore_msg}" logger.error(error_msg) return False, error_msg else: @@ -2118,11 +2387,15 @@ def reload_haproxy_safely(): check=True, capture_output=True, text=True ) logger.info("HAProxy started successfully") + # Now - and only now - is this config known good. + promote_current_config_to_backup() return True, "HAProxy started successfully" except subprocess.CalledProcessError as e: # Start failed, restore backup - restore_backup() + restored, restore_msg = restore_backup() error_msg = f"Failed to start HAProxy: {e.stderr}" + if not restored: + error_msg += f" | ROLLBACK FAILED: {restore_msg}" logger.error(error_msg) return False, error_msg except Exception as e: diff --git a/scripts/test-config-rollback.py b/scripts/test-config-rollback.py new file mode 100755 index 0000000..ad84b97 --- /dev/null +++ b/scripts/test-config-rollback.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Regression tests for HAProxy config backup / rollback ordering. + +Why this file exists +-------------------- +generate_config() used to write the new haproxy.cfg and only THEN call +reload_haproxy_safely() -> create_backup(), so the "backup" was a copy of the +config that had just been written. On a validation failure restore_backup() +restored the identical broken bytes: the advertised rollback was a no-op and a +fatal haproxy.cfg stayed on disk, where start_haproxy() refuses to launch. + +These tests pin the ordering invariant (backup predates the write) and the +observable end-to-end behaviour (after a failed validation the file on disk is +the previous working config and HAProxy will start with it). + +Running +------- + python3 scripts/test-config-rollback.py # tests the repo checkout + HAPROXY_MANAGER_DIR=/some/other/tree \ + python3 scripts/test-config-rollback.py # tests another tree + +The repo has no Python test framework (scripts/test-*.sh are curl-based +integration scripts against a running API), so this is a self-contained +stdlib-unittest script - no pytest, no venv, no new dependencies beyond the +application's own requirements.txt (Flask/Jinja2/psutil), which are already +present in the container image. + +No HAProxy binary is required: a stub `haproxy` is put on PATH that mimics +`haproxy -c -f ` by rejecting any config containing the token +__BROKEN__, which is how the tests inject an invalid configuration. +""" + +import os +import sys +import shutil +import sqlite3 +import logging +import tempfile +import textwrap +import unittest + +BROKEN_TOKEN = '__BROKEN__' + +MODULE_DIR = os.path.abspath( + os.environ.get('HAPROXY_MANAGER_DIR', + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +) + +# haproxy_manager builds its Jinja2 environment from the relative path +# Path('templates'), so it has to be imported with the module dir as cwd. +os.chdir(MODULE_DIR) +sys.path.insert(0, MODULE_DIR) + +# The module opens /var/log/haproxy-manager.log at import time via +# logging.FileHandler. Redirect that one call so the suite runs unprivileged. +_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-') +_real_file_handler = logging.FileHandler +logging.FileHandler = ( + lambda fn, *a, **kw: _real_file_handler( + os.path.join(_LOG_DIR, os.path.basename(fn)), *a, **kw) +) +try: + import haproxy_manager as hm +except ImportError as exc: # pragma: no cover - environment problem, not a failure + sys.stderr.write( + f"SKIP: cannot import haproxy_manager ({exc}).\n" + "Install the application requirements first: pip install -r requirements.txt\n" + ) + raise SystemExit(77) +finally: + logging.FileHandler = _real_file_handler + +logging.getLogger('haproxy_manager').setLevel(logging.CRITICAL) + +FAKE_HAPROXY = textwrap.dedent(f"""\ + #!/bin/sh + # Test stub for the haproxy binary. + # haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0 + # haproxy -W -S ... -f FILE (start) -> same validation, then exit 0 + cfg="" + while [ $# -gt 0 ]; do + case "$1" in -f) cfg="$2"; shift ;; esac + shift + done + if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then + echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2 + exit 1 + fi + exit 0 +""") + + +class RollbackTestCase(unittest.TestCase): + """Base fixture: an isolated fake /etc/haproxy plus a stub haproxy binary.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix='haproxy-rollback-test-') + self.addCleanup(shutil.rmtree, self.tmp, True) + + bindir = os.path.join(self.tmp, 'bin') + os.makedirs(bindir) + stub = os.path.join(bindir, 'haproxy') + with open(stub, 'w') as fh: + fh.write(FAKE_HAPROXY) + os.chmod(stub, 0o755) + self._old_path = os.environ['PATH'] + os.environ['PATH'] = bindir + os.pathsep + self._old_path + self.addCleanup(lambda: os.environ.__setitem__('PATH', self._old_path)) + + self.etc = os.path.join(self.tmp, 'etc') + os.makedirs(self.etc) + + overrides = { + 'DB_FILE': os.path.join(self.etc, 'haproxy_config.db'), + 'HAPROXY_CONFIG_PATH': os.path.join(self.etc, 'haproxy.cfg'), + 'HAPROXY_BACKUP_PATH': os.path.join(self.etc, 'haproxy.cfg.backup'), + 'BLOCKED_IPS_MAP_PATH': os.path.join(self.etc, 'blocked_ips.map'), + 'BLOCKED_IPS_MAP_BACKUP_PATH': os.path.join(self.etc, 'blocked_ips.map.backup'), + 'CLUSTER_SECRET_PATH': os.path.join(self.etc, 'cluster-secret'), + 'SSL_CERTS_DIR': os.path.join(self.etc, 'certs'), + 'HAPROXY_SOCKET_PATH': os.path.join(self.etc, 'haproxy.sock'), + # Added by the rollback fix; older trees do not have it. + 'CORAZA_SPOE_CONFIG_PATH': os.path.join(self.etc, 'coraza-spoe.cfg'), + 'CORAZA_SPOE_BACKUP_PATH': os.path.join(self.etc, 'coraza-spoe.cfg.backup'), + } + self._saved = {} + for name, value in overrides.items(): + self._saved[name] = getattr(hm, name, None) + setattr(hm, name, value) + self.addCleanup(self._restore_globals) + os.makedirs(hm.SSL_CERTS_DIR) + + # log_operation() appends to a hardcoded /var/log path. Injecting `open` + # into the module namespace shadows the builtin for that module only + # (module globals are searched before builtins), so the real + # log_operation code still runs. + real_open = open + log_dir = self.tmp + + def _redirecting_open(path, *args, **kwargs): + if isinstance(path, str) and path.startswith('/var/log/'): + path = os.path.join(log_dir, os.path.basename(path)) + return real_open(path, *args, **kwargs) + + hm.open = _redirecting_open + self.addCleanup(lambda: hm.__dict__.pop('open', None)) + + hm.init_db() + + def _restore_globals(self): + for name, value in self._saved.items(): + if value is None: + hm.__dict__.pop(name, None) + else: + setattr(hm, name, value) + + # -- helpers --------------------------------------------------------- + def add_domain(self, domain, backend_name, address='10.0.0.1'): + with sqlite3.connect(hm.DB_FILE) as conn: + cur = conn.cursor() + cur.execute('INSERT INTO domains (domain, ssl_enabled) VALUES (?, 0)', + (domain,)) + domain_id = cur.lastrowid + cur.execute('INSERT INTO backends (name, domain_id) VALUES (?, ?)', + (backend_name, domain_id)) + backend_id = cur.lastrowid + cur.execute( + 'INSERT INTO backend_servers ' + '(backend_id, server_name, server_address, server_port) ' + 'VALUES (?, ?, ?, ?)', + (backend_id, 'srv1', address, 8080)) + conn.commit() + + def block_ip(self, ip): + with sqlite3.connect(hm.DB_FILE) as conn: + conn.execute('INSERT INTO blocked_ips (ip_address, reason) VALUES (?, ?)', + (ip, 'test')) + conn.commit() + + def read(self, path): + with open(path) as fh: + return fh.read() + + def config_is_loadable(self): + """True if HAProxy would accept the config currently on disk.""" + import subprocess + return subprocess.run( + ['haproxy', '-c', '-f', hm.HAPROXY_CONFIG_PATH], + capture_output=True).returncode == 0 + + def generate_good_config(self): + self.add_domain('good.example.com', 'good_backend') + hm.generate_config() + self.assertTrue(self.config_is_loadable(), + 'fixture precondition: first generated config must be valid') + return self.read(hm.HAPROXY_CONFIG_PATH) + + def break_the_config(self): + """Queue a domain whose rendered backend the validator rejects.""" + self.add_domain('bad.example.com', BROKEN_TOKEN + '_backend', '10.0.0.2') + + +class TestBackupOrdering(RollbackTestCase): + + def test_backup_is_taken_before_the_new_config_is_written(self): + """The ordering invariant, asserted directly. + + Whatever create_backup() sees on disk must be the OLD config; if the + write happens first the backup is a copy of the new config and rollback + is meaningless. + """ + good = self.generate_good_config() + + seen = {} + real_create_backup = hm.create_backup + + def spy(*args, **kwargs): + seen['config_on_disk'] = self.read(hm.HAPROXY_CONFIG_PATH) + return real_create_backup(*args, **kwargs) + + hm.create_backup = spy + self.addCleanup(setattr, hm, 'create_backup', real_create_backup) + + self.add_domain('second.example.com', 'second_backend', '10.0.0.3') + hm.generate_config() + + self.assertIn('config_on_disk', seen, + 'create_backup() was never called during generate_config()') + self.assertEqual( + seen['config_on_disk'], good, + 'create_backup() ran AFTER the new config was written - the backup ' + 'is a copy of the new config, so rollback cannot undo anything') + + def test_backup_tracks_the_last_known_good_config(self): + """After a change that validated AND loaded, the backup is that config. + + The rollback target is "the last configuration HAProxy actually ran", + not "the file that happened to be there last time". + """ + good = self.generate_good_config() + self.add_domain('second.example.com', 'second_backend', '10.0.0.3') + hm.generate_config() + + live = self.read(hm.HAPROXY_CONFIG_PATH) + self.assertNotEqual(live, good, 'fixture sanity: the new config should differ') + self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), live, + 'the successful config was not recorded as known-good') + + def test_backup_is_not_promoted_when_the_change_fails(self): + """A config that never loaded must not become the rollback target.""" + good = self.generate_good_config() + self.break_the_config() + with self.assertRaises(Exception): + hm.generate_config() + + self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good, + 'a config that failed validation was promoted to backup') + + +class TestRollbackEndToEnd(RollbackTestCase): + + def test_failed_validation_leaves_the_last_good_config_on_disk(self): + good = self.generate_good_config() + + self.break_the_config() + with self.assertRaises(Exception): + hm.generate_config() + + on_disk = self.read(hm.HAPROXY_CONFIG_PATH) + self.assertNotIn(BROKEN_TOKEN, on_disk, + 'the rejected config is still on disk - rollback was a no-op') + self.assertEqual(on_disk, good, + 'on-disk config is not byte-identical to the last good one') + + def test_haproxy_would_still_start_after_a_failed_change(self): + """The operational consequence: the edge can still come up.""" + self.generate_good_config() + self.break_the_config() + with self.assertRaises(Exception): + hm.generate_config() + + self.assertTrue(self.config_is_loadable(), + 'HAProxy would refuse to start with the config left on disk') + with self.assertLogs('haproxy_manager', level='INFO') as captured: + hm.start_haproxy() + self.assertTrue( + any('HAProxy started successfully' in line for line in captured.output), + f'start_haproxy() did not succeed after rollback: {captured.output}') + + def test_blocked_ips_map_is_rolled_back_too(self): + """generate_config() rewrites the map file before writing haproxy.cfg.""" + self.block_ip('192.0.2.10') + self.generate_good_config() + good_map = self.read(hm.BLOCKED_IPS_MAP_PATH) + + self.block_ip('198.51.100.20') + self.break_the_config() + with self.assertRaises(Exception): + hm.generate_config() + + self.assertEqual(self.read(hm.BLOCKED_IPS_MAP_PATH), good_map, + 'blocked IPs map was not rolled back with the config') + + def test_first_run_failure_reports_that_rollback_was_impossible(self): + """No prior config: there is nothing to restore, and that must be said. + + A missing backup must never be reported as a successful restore, and it + must never be turned into "restore an empty file". + """ + self.break_the_config() + with self.assertRaises(Exception) as ctx: + hm.generate_config() + + self.assertIn('ROLLBACK FAILED', str(ctx.exception), + 'a failed change with no backup was not reported as such') + self.assertFalse(os.path.exists(hm.HAPROXY_BACKUP_PATH), + 'a backup was fabricated from the broken config') + # The broken config is deliberately left in place: start_haproxy() can + # then detect it and try to regenerate. It must not be blanked. + self.assertGreater(os.path.getsize(hm.HAPROXY_CONFIG_PATH), 0, + 'config file was emptied instead of left for diagnosis') + + +class TestBackupPrimitives(RollbackTestCase): + + def test_restore_backup_distinguishes_missing_backup_from_success(self): + restored, message = hm.restore_backup() + self.assertFalse(restored, + 'restore_backup() reported success with no backup present') + self.assertIn('cannot roll back', message.lower()) + + good = self.generate_good_config() + with open(hm.HAPROXY_CONFIG_PATH, 'w') as fh: + fh.write('scribbled over\n') + + restored, message = hm.restore_backup() + self.assertTrue(restored, message) + self.assertEqual(self.read(hm.HAPROXY_CONFIG_PATH), good) + + def test_a_successful_generation_records_a_rollback_target(self): + """Even the first-ever generation must leave something to roll back to.""" + good = self.generate_good_config() + self.assertTrue( + os.path.exists(hm.HAPROXY_BACKUP_PATH), + 'after a successful reload there is still no known-good backup') + self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good) + + def test_a_broken_current_config_does_not_replace_a_good_backup(self): + """The known-good marker. + + If the config already on disk is broken (previous failed write, manual + edit), snapshotting it would make "rollback" mean "restore a different + broken config". The older validated backup must survive. + """ + good = self.generate_good_config() + self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good, + 'fixture: a good backup should exist by now') + + with open(hm.HAPROXY_CONFIG_PATH, 'w') as fh: + fh.write(f'garbage {BROKEN_TOKEN} config\n') + + ok, status = hm.create_backup() + self.assertTrue(ok) + self.assertEqual(status, 'kept_previous') + self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good, + 'a broken config overwrote the known-good backup') + + def test_reload_does_not_take_its_own_backup(self): + """reload_haproxy_safely() runs after the write, so it must not back up.""" + good = self.generate_good_config() + with open(hm.HAPROXY_CONFIG_PATH, 'w') as fh: + fh.write(f'broken {BROKEN_TOKEN}\n') + + success, message = hm.reload_haproxy_safely(backup_status='created') + + self.assertFalse(success) + self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good, + 'reload_haproxy_safely() overwrote the good backup') + self.assertEqual(self.read(hm.HAPROXY_CONFIG_PATH), good, + 'reload_haproxy_safely() did not roll the config back') + + def test_unchanged_config_is_not_revalidated(self): + """Fast path: if the backup already is the live config, do no work. + + generate_config() runs inside customer-facing API calls and + `haproxy -c` is expensive on an edge with hundreds of certificates. + """ + self.generate_good_config() + + calls = [] + real_validate = hm.validate_config_file + hm.validate_config_file = lambda path: (calls.append(path), + real_validate(path))[1] + self.addCleanup(setattr, hm, 'validate_config_file', real_validate) + + ok, status = hm.create_backup() + self.assertTrue(ok) + self.assertEqual(status, 'created') + self.assertEqual(calls, [], + 'the unchanged live config was re-validated needlessly') + + def test_fast_path_does_not_hide_a_drifted_broken_config(self): + """If the live config drifted from the backup, the gate must still run.""" + good = self.generate_good_config() + with open(hm.HAPROXY_CONFIG_PATH, 'w') as fh: + fh.write(f'hand edited {BROKEN_TOKEN}\n') + + ok, status = hm.create_backup() + self.assertTrue(ok) + self.assertEqual(status, 'kept_previous', + 'a drifted broken config was silently accepted') + self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good) + + def test_backup_set_covers_every_file_generate_config_writes(self): + pairs = dict(hm._config_backup_pairs()) + for path in (hm.HAPROXY_CONFIG_PATH, hm.BLOCKED_IPS_MAP_PATH, + hm.CORAZA_SPOE_CONFIG_PATH): + self.assertIn(path, pairs, + f'{path} is written by generate_config() but is not ' + 'part of the backed-up config set') + + def test_coraza_spoe_config_round_trips(self): + self.generate_good_config() + with open(hm.CORAZA_SPOE_CONFIG_PATH, 'w') as fh: + fh.write('spoe-good\n') + hm.create_backup() + with open(hm.CORAZA_SPOE_CONFIG_PATH, 'w') as fh: + fh.write('spoe-broken\n') + restored, message = hm.restore_backup() + self.assertTrue(restored, message) + self.assertEqual(self.read(hm.CORAZA_SPOE_CONFIG_PATH), 'spoe-good\n') + + +class TestAtomicWrite(RollbackTestCase): + + def test_write_is_atomic_and_preserves_mode(self): + path = os.path.join(self.etc, 'atomic.cfg') + with open(path, 'w') as fh: + fh.write('old') + os.chmod(path, 0o644) + + hm.write_config_atomically(path, 'new content\n') + + self.assertEqual(self.read(path), 'new content\n') + self.assertEqual(oct(os.stat(path).st_mode & 0o777), oct(0o644)) + leftovers = [n for n in os.listdir(self.etc) if n.endswith('.tmp')] + self.assertEqual(leftovers, [], f'temp files left behind: {leftovers}') + + def test_failed_write_leaves_the_previous_file_intact(self): + path = os.path.join(self.etc, 'atomic.cfg') + with open(path, 'w') as fh: + fh.write('old content\n') + + # Anything that makes f.write() blow up mid-flight stands in for a full + # disk / killed container. + with self.assertRaises(Exception): + hm.write_config_atomically(path, object()) + + self.assertEqual(self.read(path), 'old content\n', + 'a failed write clobbered the previous config') + leftovers = [n for n in os.listdir(self.etc) if n.endswith('.tmp')] + self.assertEqual(leftovers, [], f'temp files left behind: {leftovers}') + + +if __name__ == '__main__': + print(f"testing haproxy_manager from: {MODULE_DIR}") + unittest.main(verbosity=2)