fix(config): take the rollback backup BEFORE writing the new config
generate_config() wrote /etc/haproxy/haproxy.cfg and only then called
reload_haproxy_safely(), which called create_backup(). The "backup" was
therefore a copy of the config that had just been written, so on a validation
failure restore_backup() restored the identical broken bytes: the advertised
rollback was a no-op and a fatal haproxy.cfg stayed on disk, where
start_haproxy() refuses to launch. Same shape as the June 2026 incident where
a missing template produced a fatal config and took an edge down.
Reproduced end to end before the fix (invalid config generated -> "Backups
created successfully" -> "Backups restored successfully" -> haproxy.cfg on
disk still invalid, `haproxy -c` rc=1).
Changes:
* create_backup() is now called by generate_config() BEFORE the first write,
which also covers blocked_ips.map (rewritten early in generate_config) and
coraza-spoe.cfg - both previously written before the backup and, for the
SPOE file, never backed up at all even though `haproxy -c` parses it.
* create_backup() refuses to promote a config HAProxy already rejects, so a
broken file on disk cannot overwrite a known-good backup ("rollback" must
not mean "restore a different broken config"). It returns (ok, status) so
the caller knows whether a rollback target exists.
* promote_current_config_to_backup() records the config as known-good only
after it has validated AND loaded, so a box whose first generation succeeded
has a rollback target immediately, and a config that never loaded is never
promoted.
* restore_backup() returns (restored, message) and distinguishes "no backup
available" from "restored". Every caller now surfaces the difference; a
failed rollback is logged CRITICAL and reported as ROLLBACK FAILED in the
API error message instead of silently looking like a successful recovery.
* reload_haproxy_safely(backup_status=...) no longer takes its own backup - it
runs after the write, where a backup is meaningless. Called without a status
it logs the contract violation rather than overwriting a good backup.
* validate_config_file() separates "config is invalid" from "validator could
not run" so a missing haproxy binary is not read as a bad config.
* Config writes are atomic (temp file + fsync + os.replace, mode preserved);
a truncated haproxy.cfg is as fatal as an invalid one. Removes the dead
temp_config_path variable whose comment claimed this already happened.
* Fast path: if the live config set is already byte-identical to the backup
(the normal case after a successful reload), skip the re-validation and the
copy, so this adds no `haproxy -c` latency to customer-facing API calls.
Tests: scripts/test-config-rollback.py - 17 self-contained stdlib-unittest
tests, no new dependencies (the repo has no Python test framework; the
existing scripts/test-*.sh are curl integration scripts). A stub `haproxy`
binary stands in for the validator. 12 of the 17 fail against the previous
code; every assertion was mutation-proven (9 mutations, each reddening only
the tests that cover it).
No template, QUIC or HTTP/3 changes.
This commit is contained in:
+330
-57
@@ -12,6 +12,7 @@ from datetime import datetime, timedelta
|
||||
import json
|
||||
import ipaddress
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
@@ -105,6 +106,12 @@ HAPROXY_CONFIG_PATH = '/etc/haproxy/haproxy.cfg'
|
||||
HAPROXY_BACKUP_PATH = '/etc/haproxy/haproxy.cfg.backup'
|
||||
BLOCKED_IPS_MAP_PATH = '/etc/haproxy/blocked_ips.map'
|
||||
BLOCKED_IPS_MAP_BACKUP_PATH = '/etc/haproxy/blocked_ips.map.backup'
|
||||
# Coraza SPOE engine file. `haproxy -c` parses this too (the frontend's
|
||||
# `filter spoe engine coraza config <path>` line points at it), so it is part
|
||||
# of the same restorable config set as haproxy.cfg — rolling back haproxy.cfg
|
||||
# while leaving a broken coraza-spoe.cfg behind still fails validation.
|
||||
CORAZA_SPOE_CONFIG_PATH = '/etc/haproxy/coraza-spoe.cfg'
|
||||
CORAZA_SPOE_BACKUP_PATH = '/etc/haproxy/coraza-spoe.cfg.backup'
|
||||
HAPROXY_SOCKET_PATH = '/var/run/haproxy.sock'
|
||||
SSL_CERTS_DIR = '/etc/haproxy/certs'
|
||||
# Stable per-host secret for QUIC Retry/address-validation tokens. Lives in the
|
||||
@@ -1795,6 +1802,21 @@ def generate_config():
|
||||
|
||||
config_parts = []
|
||||
|
||||
# Snapshot the last-known-good config BEFORE anything below touches a
|
||||
# file in /etc/haproxy. Everything this function writes (haproxy.cfg,
|
||||
# blocked_ips.map, coraza-spoe.cfg) is validated as one set by
|
||||
# `haproxy -c`, so the rollback point has to predate the first of them.
|
||||
# Taking it here (rather than inside reload_haproxy_safely(), which runs
|
||||
# after the writes) is what makes rollback real - see create_backup().
|
||||
backup_ok, backup_status = create_backup()
|
||||
if not backup_ok:
|
||||
# Could not even attempt a snapshot (I/O error). Writing a new
|
||||
# config now would leave us with no way back, so refuse.
|
||||
raise Exception(
|
||||
"Refusing to regenerate config: failed to back up the current "
|
||||
"configuration, so a failed change could not be rolled back"
|
||||
)
|
||||
|
||||
# Optional Coraza WAF integration. When HAPROXY_CORAZA_SPOE_BACKEND is
|
||||
# set on the haproxy-manager container, we render an extra TCP backend
|
||||
# pointing at a coraza-spoa sidecar AND inject a `filter spoe ...` line
|
||||
@@ -1984,25 +2006,21 @@ backend default-backend
|
||||
# how the file was authored.
|
||||
if not coraza_spoe_cfg.endswith('\n'):
|
||||
coraza_spoe_cfg += '\n'
|
||||
coraza_spoe_path = '/etc/haproxy/coraza-spoe.cfg'
|
||||
with open(coraza_spoe_path, 'w') as f:
|
||||
f.write(coraza_spoe_cfg)
|
||||
logger.info(f"Coraza SPOE engine config written to {coraza_spoe_path} "
|
||||
write_config_atomically(CORAZA_SPOE_CONFIG_PATH, coraza_spoe_cfg)
|
||||
logger.info(f"Coraza SPOE engine config written to "
|
||||
f"{CORAZA_SPOE_CONFIG_PATH} "
|
||||
f"(SPOA target: {coraza_spoe_backend})")
|
||||
|
||||
# Write complete configuration to tmp
|
||||
temp_config_path = "/etc/haproxy/haproxy.cfg"
|
||||
|
||||
config_content = '\n'.join(config_parts)
|
||||
logger.debug("Generated HAProxy configuration")
|
||||
|
||||
# Write complete configuration to tmp
|
||||
# Write new configuration to file
|
||||
with open(HAPROXY_CONFIG_PATH, 'w') as f:
|
||||
f.write(config_content)
|
||||
# Write new configuration to file (atomically - a truncated haproxy.cfg
|
||||
# is as fatal as an invalid one). The rollback point was taken above,
|
||||
# before this write.
|
||||
write_config_atomically(HAPROXY_CONFIG_PATH, config_content)
|
||||
|
||||
# Use safe reload with validation and rollback
|
||||
success, message = reload_haproxy_safely()
|
||||
success, message = reload_haproxy_safely(backup_status=backup_status)
|
||||
if success:
|
||||
logger.info("Configuration generated and HAProxy reloaded safely")
|
||||
log_operation('generate_config', True, 'Configuration generated and HAProxy reloaded safely')
|
||||
@@ -2019,61 +2037,304 @@ backend default-backend
|
||||
traceback.print_exc()
|
||||
raise
|
||||
|
||||
def create_backup():
|
||||
"""Create backup of current config and map files"""
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config backup / rollback
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rollback only works if the backup predates the write it is supposed to undo.
|
||||
# Until 2026-08 create_backup() ran from inside reload_haproxy_safely(), i.e.
|
||||
# AFTER generate_config() had already overwritten haproxy.cfg — so the "backup"
|
||||
# was a copy of the new (possibly broken) config and restore_backup() restored
|
||||
# the same broken bytes. The advertised rollback was a no-op and a fatal
|
||||
# haproxy.cfg persisted on disk, where start_haproxy() refuses to launch (the
|
||||
# June 2026 missing-template incident). create_backup() must now be called by
|
||||
# the writer, BEFORE the first byte is written.
|
||||
|
||||
# Statuses returned by create_backup() that mean a rollback target exists.
|
||||
_ROLLBACK_AVAILABLE_STATUSES = ('created', 'kept_previous')
|
||||
|
||||
|
||||
def _files_identical(path_a, path_b):
|
||||
"""Byte-compare two files.
|
||||
|
||||
Deliberately not filecmp.cmp(): it memoises on (size, mtime), and
|
||||
shutil.copy2() preserves mtime, so a stale cache entry could report a
|
||||
changed config as unchanged. These files are small; read them.
|
||||
"""
|
||||
try:
|
||||
if os.path.exists(HAPROXY_CONFIG_PATH):
|
||||
shutil.copy2(HAPROXY_CONFIG_PATH, HAPROXY_BACKUP_PATH)
|
||||
if os.path.exists(BLOCKED_IPS_MAP_PATH):
|
||||
shutil.copy2(BLOCKED_IPS_MAP_PATH, BLOCKED_IPS_MAP_BACKUP_PATH)
|
||||
logger.info("Backups created successfully")
|
||||
return True
|
||||
if os.path.getsize(path_a) != os.path.getsize(path_b):
|
||||
return False
|
||||
with open(path_a, 'rb') as fa, open(path_b, 'rb') as fb:
|
||||
while True:
|
||||
chunk_a = fa.read(65536)
|
||||
chunk_b = fb.read(65536)
|
||||
if chunk_a != chunk_b:
|
||||
return False
|
||||
if not chunk_a:
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _config_set_matches_backup():
|
||||
"""True if every live config file is byte-identical to its backup copy.
|
||||
|
||||
After a successful reload the live set has already been recorded as
|
||||
known-good (see promote_current_config_to_backup()), which is the common
|
||||
case at the start of the next generation. Recognising it lets create_backup()
|
||||
skip both the re-validation and the copy - worth doing because
|
||||
`haproxy -c` on an edge with hundreds of certificates is not free and
|
||||
generate_config() runs synchronously inside customer-facing API calls.
|
||||
"""
|
||||
for live_path, backup_path in _config_backup_pairs():
|
||||
if os.path.exists(live_path) != os.path.exists(backup_path):
|
||||
return False
|
||||
if (os.path.exists(live_path)
|
||||
and not _files_identical(live_path, backup_path)):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _config_backup_pairs():
|
||||
"""(live, backup) pairs forming one restorable config set.
|
||||
|
||||
Built at call time rather than at import so the module-level path constants
|
||||
stay patchable (tests, alternate deployments).
|
||||
"""
|
||||
return (
|
||||
(HAPROXY_CONFIG_PATH, HAPROXY_BACKUP_PATH),
|
||||
(BLOCKED_IPS_MAP_PATH, BLOCKED_IPS_MAP_BACKUP_PATH),
|
||||
(CORAZA_SPOE_CONFIG_PATH, CORAZA_SPOE_BACKUP_PATH),
|
||||
)
|
||||
|
||||
|
||||
def write_config_atomically(path, content):
|
||||
"""Write content to path via temp file + rename.
|
||||
|
||||
A half-written haproxy.cfg (disk full, container killed mid-write) is just
|
||||
as fatal as an invalid one and is invisible to the caller. os.replace() is
|
||||
atomic within a filesystem, so the file on disk is always either the whole
|
||||
old config or the whole new one — never a truncated hybrid. This also keeps
|
||||
the "existing config is already broken" case from being self-inflicted.
|
||||
"""
|
||||
directory = os.path.dirname(path) or '.'
|
||||
# Preserve the mode of the file we are replacing; mkstemp defaults to 0600
|
||||
# and HAProxy config files are conventionally 0644.
|
||||
try:
|
||||
mode = stat.S_IMODE(os.stat(path).st_mode)
|
||||
except OSError:
|
||||
mode = 0o644
|
||||
fd, tmp_path = tempfile.mkstemp(
|
||||
dir=directory, prefix=os.path.basename(path) + '.', suffix='.tmp'
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.chmod(tmp_path, mode)
|
||||
os.replace(tmp_path, path)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def create_backup(require_valid=True):
|
||||
"""Snapshot the CURRENT on-disk config set as the rollback point.
|
||||
|
||||
MUST be called BEFORE the new configuration is written — see the module
|
||||
comment above. Calling it afterwards silently disarms rollback.
|
||||
|
||||
require_valid=True (default) refuses to promote a config that HAProxy
|
||||
already rejects. Backing up a broken config would make "rollback" mean
|
||||
"restore a different broken config"; keeping the older, validated backup
|
||||
instead means a rollback always lands on something HAProxy will actually
|
||||
start with. Cost is one `haproxy -c` run per config generation.
|
||||
|
||||
Returns (ok, status):
|
||||
ok=False, status='error' - the copy itself failed; caller decides.
|
||||
status='created' - backup now holds the current config.
|
||||
status='kept_previous' - current config missing or invalid; the
|
||||
existing (older, good) backup was kept.
|
||||
status='unavailable' - nothing to roll back to at all (first
|
||||
run, or broken config and no prior
|
||||
backup). Rollback is NOT possible.
|
||||
"""
|
||||
try:
|
||||
snapshot_ok = True
|
||||
reason = None
|
||||
|
||||
if not os.path.exists(HAPROXY_CONFIG_PATH):
|
||||
snapshot_ok = False
|
||||
reason = 'no existing HAProxy config on disk (first run?)'
|
||||
elif _config_set_matches_backup():
|
||||
# The backup already IS the current config, recorded when it last
|
||||
# loaded successfully. Nothing to copy and nothing to re-validate.
|
||||
logger.debug("Config backup already matches the live config")
|
||||
return True, 'created'
|
||||
elif require_valid:
|
||||
status, msg = validate_config_file(HAPROXY_CONFIG_PATH)
|
||||
if status == 'invalid':
|
||||
snapshot_ok = False
|
||||
reason = f'current config on disk does not validate: {msg}'
|
||||
elif status == 'unavailable':
|
||||
# The validator itself could not run (no haproxy binary, etc).
|
||||
# That is NOT evidence the config is bad, and refusing to back
|
||||
# up would leave us with no rollback target at all, so fall
|
||||
# back to last-written semantics and say so loudly.
|
||||
logger.warning(
|
||||
f"Could not verify current config before backup ({msg}); "
|
||||
"backing it up unverified"
|
||||
)
|
||||
|
||||
if not snapshot_ok:
|
||||
if os.path.exists(HAPROXY_BACKUP_PATH):
|
||||
logger.warning(
|
||||
f"Not refreshing config backup: {reason}. Keeping the "
|
||||
f"existing backup at {HAPROXY_BACKUP_PATH} as the rollback "
|
||||
"target."
|
||||
)
|
||||
return True, 'kept_previous'
|
||||
logger.error(
|
||||
f"No config backup could be taken: {reason}, and no previous "
|
||||
f"backup exists at {HAPROXY_BACKUP_PATH}. ROLLBACK IS NOT "
|
||||
"AVAILABLE for this configuration change."
|
||||
)
|
||||
return True, 'unavailable'
|
||||
|
||||
for live_path, backup_path in _config_backup_pairs():
|
||||
if os.path.exists(live_path):
|
||||
shutil.copy2(live_path, backup_path)
|
||||
logger.info("Backup of last-known-good config created successfully")
|
||||
return True, 'created'
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create backup: {e}")
|
||||
return False
|
||||
return False, 'error'
|
||||
|
||||
def restore_backup():
|
||||
"""Restore from backup files"""
|
||||
def promote_current_config_to_backup():
|
||||
"""Record the live config as the known-good rollback target.
|
||||
|
||||
Called ONLY after the config has both validated and been loaded by HAProxy,
|
||||
so "backup" really means "the last configuration this box was running".
|
||||
Must never be called before a reload attempt: doing so would make the
|
||||
backup a copy of the config we may still have to roll back from - the same
|
||||
class of bug as backing up after the write.
|
||||
|
||||
Without this, a box whose very first generation succeeded has no rollback
|
||||
target at all until its second successful generation, and any corruption of
|
||||
haproxy.cfg in between leaves nothing to recover to.
|
||||
"""
|
||||
try:
|
||||
if os.path.exists(HAPROXY_BACKUP_PATH):
|
||||
shutil.copy2(HAPROXY_BACKUP_PATH, HAPROXY_CONFIG_PATH)
|
||||
if os.path.exists(BLOCKED_IPS_MAP_BACKUP_PATH):
|
||||
shutil.copy2(BLOCKED_IPS_MAP_BACKUP_PATH, BLOCKED_IPS_MAP_PATH)
|
||||
logger.info("Backups restored successfully")
|
||||
for live_path, backup_path in _config_backup_pairs():
|
||||
if os.path.exists(live_path):
|
||||
shutil.copy2(live_path, backup_path)
|
||||
logger.debug("Known-good config backup updated after successful reload")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to restore backup: {e}")
|
||||
# Non-fatal: the config is live and working, we just failed to record
|
||||
# it. Loud, because the next change now has a staler rollback target.
|
||||
logger.error(f"Failed to record known-good config backup: {e}")
|
||||
return False
|
||||
|
||||
def validate_haproxy_config():
|
||||
"""Validate HAProxy configuration file"""
|
||||
try:
|
||||
result = subprocess.run(['haproxy', '-c', '-f', HAPROXY_CONFIG_PATH],
|
||||
capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
logger.info("HAProxy configuration validation passed")
|
||||
return True, None
|
||||
else:
|
||||
error_msg = f"HAProxy configuration validation failed: {result.stderr}"
|
||||
logger.error(error_msg)
|
||||
return False, error_msg
|
||||
except Exception as e:
|
||||
error_msg = f"Error validating HAProxy config: {e}"
|
||||
logger.error(error_msg)
|
||||
return False, error_msg
|
||||
|
||||
def reload_haproxy_safely():
|
||||
"""Safely reload HAProxy with validation and rollback"""
|
||||
def restore_backup():
|
||||
"""Restore the backed-up config set over the live files.
|
||||
|
||||
Returns (restored, message). restored=False means NOTHING was rolled back
|
||||
and the live config is still whatever the failed change left on disk —
|
||||
callers MUST surface that difference, it is the difference between "we
|
||||
recovered" and "this edge is sitting on a config HAProxy will not load".
|
||||
"""
|
||||
if not os.path.exists(HAPROXY_BACKUP_PATH):
|
||||
msg = (f"No config backup at {HAPROXY_BACKUP_PATH} - cannot roll back; "
|
||||
f"{HAPROXY_CONFIG_PATH} still holds the failed configuration")
|
||||
logger.critical(msg)
|
||||
return False, msg
|
||||
try:
|
||||
# Create backup before changes
|
||||
if not create_backup():
|
||||
return False, "Failed to create backup"
|
||||
for live_path, backup_path in _config_backup_pairs():
|
||||
if os.path.exists(backup_path):
|
||||
shutil.copy2(backup_path, live_path)
|
||||
msg = f"Configuration restored from backup ({HAPROXY_BACKUP_PATH})"
|
||||
logger.info(msg)
|
||||
return True, msg
|
||||
except Exception as e:
|
||||
msg = (f"Failed to restore backup: {e} - {HAPROXY_CONFIG_PATH} may hold "
|
||||
"a broken configuration")
|
||||
logger.critical(msg)
|
||||
return False, msg
|
||||
|
||||
|
||||
def validate_config_file(config_path):
|
||||
"""Run `haproxy -c` against config_path.
|
||||
|
||||
Returns (status, message) with status one of:
|
||||
'valid' - HAProxy parsed the file successfully
|
||||
'invalid' - HAProxy rejected it (message carries stderr)
|
||||
'unavailable' - the validator could not be run at all (binary missing,
|
||||
timeout, ...). Deliberately distinct from 'invalid':
|
||||
it tells us nothing about the config.
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(['haproxy', '-c', '-f', config_path],
|
||||
capture_output=True, text=True)
|
||||
except Exception as e:
|
||||
return 'unavailable', f"Error validating HAProxy config: {e}"
|
||||
if result.returncode == 0:
|
||||
return 'valid', None
|
||||
return 'invalid', f"HAProxy configuration validation failed: {result.stderr}"
|
||||
|
||||
|
||||
def validate_haproxy_config():
|
||||
"""Validate the live HAProxy configuration file. Returns (is_valid, error)."""
|
||||
status, message = validate_config_file(HAPROXY_CONFIG_PATH)
|
||||
if status == 'valid':
|
||||
logger.info("HAProxy configuration validation passed")
|
||||
return True, None
|
||||
logger.error(message)
|
||||
return False, message
|
||||
|
||||
def reload_haproxy_safely(backup_status=None):
|
||||
"""Safely reload HAProxy with validation and rollback.
|
||||
|
||||
PRECONDITION: the caller must already have called create_backup() BEFORE
|
||||
writing the new config, and pass the status it returned. This function runs
|
||||
after the new config is on disk, so it cannot take a meaningful backup
|
||||
itself — doing so is exactly the bug this contract exists to prevent.
|
||||
|
||||
backup_status=None means the caller did not take a pre-write backup. We do
|
||||
NOT create one here (that would overwrite a genuinely good backup with the
|
||||
unverified new config); we log it and fall back to whatever backup already
|
||||
exists on disk.
|
||||
"""
|
||||
try:
|
||||
if backup_status is None:
|
||||
logger.error(
|
||||
"reload_haproxy_safely() called without a pre-write backup "
|
||||
"status - rollback will fall back to whatever backup already "
|
||||
"exists on disk. Callers must call create_backup() BEFORE "
|
||||
"writing the new configuration."
|
||||
)
|
||||
elif backup_status not in _ROLLBACK_AVAILABLE_STATUSES:
|
||||
logger.warning(
|
||||
f"Proceeding with reload without a rollback target "
|
||||
f"(backup status: {backup_status})"
|
||||
)
|
||||
|
||||
# Validate new configuration
|
||||
is_valid, error_msg = validate_haproxy_config()
|
||||
if not is_valid:
|
||||
# Restore backup on validation failure
|
||||
restore_backup()
|
||||
restored, restore_msg = restore_backup()
|
||||
if not restored:
|
||||
logger.critical(
|
||||
"Config validation failed AND rollback was not possible - "
|
||||
f"{HAPROXY_CONFIG_PATH} holds an invalid configuration that "
|
||||
"HAProxy will refuse to start with"
|
||||
)
|
||||
return False, (f"Config validation failed: {error_msg} | "
|
||||
f"ROLLBACK FAILED: {restore_msg}")
|
||||
return False, f"Config validation failed: {error_msg}"
|
||||
|
||||
# Attempt reload
|
||||
@@ -2094,20 +2355,28 @@ def reload_haproxy_safely():
|
||||
|
||||
if reload_result.returncode == 0:
|
||||
logger.info("HAProxy reloaded successfully")
|
||||
# Now - and only now - is this config known good.
|
||||
promote_current_config_to_backup()
|
||||
return True, "HAProxy reloaded successfully"
|
||||
else:
|
||||
# Reload failed, restore backup
|
||||
restore_backup()
|
||||
# Try to reload with backup config
|
||||
subprocess.run('echo "reload" | socat stdio /tmp/haproxy-cli',
|
||||
shell=True, capture_output=True)
|
||||
restored, restore_msg = restore_backup()
|
||||
if restored:
|
||||
# Try to reload with the restored (known-good) config
|
||||
subprocess.run(
|
||||
'echo "reload" | socat stdio /tmp/haproxy-cli',
|
||||
shell=True, capture_output=True)
|
||||
error_msg = f"HAProxy reload failed: {reload_result.stderr}"
|
||||
if not restored:
|
||||
error_msg += f" | ROLLBACK FAILED: {restore_msg}"
|
||||
logger.error(error_msg)
|
||||
return False, error_msg
|
||||
except Exception as e:
|
||||
# Critical error during reload, restore backup
|
||||
restore_backup()
|
||||
restored, restore_msg = restore_backup()
|
||||
error_msg = f"Critical error during reload: {e}"
|
||||
if not restored:
|
||||
error_msg += f" | ROLLBACK FAILED: {restore_msg}"
|
||||
logger.error(error_msg)
|
||||
return False, error_msg
|
||||
else:
|
||||
@@ -2118,11 +2387,15 @@ def reload_haproxy_safely():
|
||||
check=True, capture_output=True, text=True
|
||||
)
|
||||
logger.info("HAProxy started successfully")
|
||||
# Now - and only now - is this config known good.
|
||||
promote_current_config_to_backup()
|
||||
return True, "HAProxy started successfully"
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Start failed, restore backup
|
||||
restore_backup()
|
||||
restored, restore_msg = restore_backup()
|
||||
error_msg = f"Failed to start HAProxy: {e.stderr}"
|
||||
if not restored:
|
||||
error_msg += f" | ROLLBACK FAILED: {restore_msg}"
|
||||
logger.error(error_msg)
|
||||
return False, error_msg
|
||||
except Exception as e:
|
||||
|
||||
Executable
+467
@@ -0,0 +1,467 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression tests for HAProxy config backup / rollback ordering.
|
||||
|
||||
Why this file exists
|
||||
--------------------
|
||||
generate_config() used to write the new haproxy.cfg and only THEN call
|
||||
reload_haproxy_safely() -> create_backup(), so the "backup" was a copy of the
|
||||
config that had just been written. On a validation failure restore_backup()
|
||||
restored the identical broken bytes: the advertised rollback was a no-op and a
|
||||
fatal haproxy.cfg stayed on disk, where start_haproxy() refuses to launch.
|
||||
|
||||
These tests pin the ordering invariant (backup predates the write) and the
|
||||
observable end-to-end behaviour (after a failed validation the file on disk is
|
||||
the previous working config and HAProxy will start with it).
|
||||
|
||||
Running
|
||||
-------
|
||||
python3 scripts/test-config-rollback.py # tests the repo checkout
|
||||
HAPROXY_MANAGER_DIR=/some/other/tree \
|
||||
python3 scripts/test-config-rollback.py # tests another tree
|
||||
|
||||
The repo has no Python test framework (scripts/test-*.sh are curl-based
|
||||
integration scripts against a running API), so this is a self-contained
|
||||
stdlib-unittest script - no pytest, no venv, no new dependencies beyond the
|
||||
application's own requirements.txt (Flask/Jinja2/psutil), which are already
|
||||
present in the container image.
|
||||
|
||||
No HAProxy binary is required: a stub `haproxy` is put on PATH that mimics
|
||||
`haproxy -c -f <file>` by rejecting any config containing the token
|
||||
__BROKEN__, which is how the tests inject an invalid configuration.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import sqlite3
|
||||
import logging
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
BROKEN_TOKEN = '__BROKEN__'
|
||||
|
||||
MODULE_DIR = os.path.abspath(
|
||||
os.environ.get('HAPROXY_MANAGER_DIR',
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
|
||||
)
|
||||
|
||||
# haproxy_manager builds its Jinja2 environment from the relative path
|
||||
# Path('templates'), so it has to be imported with the module dir as cwd.
|
||||
os.chdir(MODULE_DIR)
|
||||
sys.path.insert(0, MODULE_DIR)
|
||||
|
||||
# The module opens /var/log/haproxy-manager.log at import time via
|
||||
# logging.FileHandler. Redirect that one call so the suite runs unprivileged.
|
||||
_LOG_DIR = tempfile.mkdtemp(prefix='haproxy-mgr-test-logs-')
|
||||
_real_file_handler = logging.FileHandler
|
||||
logging.FileHandler = (
|
||||
lambda fn, *a, **kw: _real_file_handler(
|
||||
os.path.join(_LOG_DIR, os.path.basename(fn)), *a, **kw)
|
||||
)
|
||||
try:
|
||||
import haproxy_manager as hm
|
||||
except ImportError as exc: # pragma: no cover - environment problem, not a failure
|
||||
sys.stderr.write(
|
||||
f"SKIP: cannot import haproxy_manager ({exc}).\n"
|
||||
"Install the application requirements first: pip install -r requirements.txt\n"
|
||||
)
|
||||
raise SystemExit(77)
|
||||
finally:
|
||||
logging.FileHandler = _real_file_handler
|
||||
|
||||
logging.getLogger('haproxy_manager').setLevel(logging.CRITICAL)
|
||||
|
||||
FAKE_HAPROXY = textwrap.dedent(f"""\
|
||||
#!/bin/sh
|
||||
# Test stub for the haproxy binary.
|
||||
# haproxy -c -f FILE -> exit 1 if FILE contains {BROKEN_TOKEN}, else 0
|
||||
# haproxy -W -S ... -f FILE (start) -> same validation, then exit 0
|
||||
cfg=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in -f) cfg="$2"; shift ;; esac
|
||||
shift
|
||||
done
|
||||
if [ -n "$cfg" ] && grep -q '{BROKEN_TOKEN}' "$cfg" 2>/dev/null; then
|
||||
echo "[ALERT] parsing [$cfg:1] : unknown keyword '{BROKEN_TOKEN}'" >&2
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
""")
|
||||
|
||||
|
||||
class RollbackTestCase(unittest.TestCase):
|
||||
"""Base fixture: an isolated fake /etc/haproxy plus a stub haproxy binary."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix='haproxy-rollback-test-')
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
|
||||
bindir = os.path.join(self.tmp, 'bin')
|
||||
os.makedirs(bindir)
|
||||
stub = os.path.join(bindir, 'haproxy')
|
||||
with open(stub, 'w') as fh:
|
||||
fh.write(FAKE_HAPROXY)
|
||||
os.chmod(stub, 0o755)
|
||||
self._old_path = os.environ['PATH']
|
||||
os.environ['PATH'] = bindir + os.pathsep + self._old_path
|
||||
self.addCleanup(lambda: os.environ.__setitem__('PATH', self._old_path))
|
||||
|
||||
self.etc = os.path.join(self.tmp, 'etc')
|
||||
os.makedirs(self.etc)
|
||||
|
||||
overrides = {
|
||||
'DB_FILE': os.path.join(self.etc, 'haproxy_config.db'),
|
||||
'HAPROXY_CONFIG_PATH': os.path.join(self.etc, 'haproxy.cfg'),
|
||||
'HAPROXY_BACKUP_PATH': os.path.join(self.etc, 'haproxy.cfg.backup'),
|
||||
'BLOCKED_IPS_MAP_PATH': os.path.join(self.etc, 'blocked_ips.map'),
|
||||
'BLOCKED_IPS_MAP_BACKUP_PATH': os.path.join(self.etc, 'blocked_ips.map.backup'),
|
||||
'CLUSTER_SECRET_PATH': os.path.join(self.etc, 'cluster-secret'),
|
||||
'SSL_CERTS_DIR': os.path.join(self.etc, 'certs'),
|
||||
'HAPROXY_SOCKET_PATH': os.path.join(self.etc, 'haproxy.sock'),
|
||||
# Added by the rollback fix; older trees do not have it.
|
||||
'CORAZA_SPOE_CONFIG_PATH': os.path.join(self.etc, 'coraza-spoe.cfg'),
|
||||
'CORAZA_SPOE_BACKUP_PATH': os.path.join(self.etc, 'coraza-spoe.cfg.backup'),
|
||||
}
|
||||
self._saved = {}
|
||||
for name, value in overrides.items():
|
||||
self._saved[name] = getattr(hm, name, None)
|
||||
setattr(hm, name, value)
|
||||
self.addCleanup(self._restore_globals)
|
||||
os.makedirs(hm.SSL_CERTS_DIR)
|
||||
|
||||
# log_operation() appends to a hardcoded /var/log path. Injecting `open`
|
||||
# into the module namespace shadows the builtin for that module only
|
||||
# (module globals are searched before builtins), so the real
|
||||
# log_operation code still runs.
|
||||
real_open = open
|
||||
log_dir = self.tmp
|
||||
|
||||
def _redirecting_open(path, *args, **kwargs):
|
||||
if isinstance(path, str) and path.startswith('/var/log/'):
|
||||
path = os.path.join(log_dir, os.path.basename(path))
|
||||
return real_open(path, *args, **kwargs)
|
||||
|
||||
hm.open = _redirecting_open
|
||||
self.addCleanup(lambda: hm.__dict__.pop('open', None))
|
||||
|
||||
hm.init_db()
|
||||
|
||||
def _restore_globals(self):
|
||||
for name, value in self._saved.items():
|
||||
if value is None:
|
||||
hm.__dict__.pop(name, None)
|
||||
else:
|
||||
setattr(hm, name, value)
|
||||
|
||||
# -- helpers ---------------------------------------------------------
|
||||
def add_domain(self, domain, backend_name, address='10.0.0.1'):
|
||||
with sqlite3.connect(hm.DB_FILE) as conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute('INSERT INTO domains (domain, ssl_enabled) VALUES (?, 0)',
|
||||
(domain,))
|
||||
domain_id = cur.lastrowid
|
||||
cur.execute('INSERT INTO backends (name, domain_id) VALUES (?, ?)',
|
||||
(backend_name, domain_id))
|
||||
backend_id = cur.lastrowid
|
||||
cur.execute(
|
||||
'INSERT INTO backend_servers '
|
||||
'(backend_id, server_name, server_address, server_port) '
|
||||
'VALUES (?, ?, ?, ?)',
|
||||
(backend_id, 'srv1', address, 8080))
|
||||
conn.commit()
|
||||
|
||||
def block_ip(self, ip):
|
||||
with sqlite3.connect(hm.DB_FILE) as conn:
|
||||
conn.execute('INSERT INTO blocked_ips (ip_address, reason) VALUES (?, ?)',
|
||||
(ip, 'test'))
|
||||
conn.commit()
|
||||
|
||||
def read(self, path):
|
||||
with open(path) as fh:
|
||||
return fh.read()
|
||||
|
||||
def config_is_loadable(self):
|
||||
"""True if HAProxy would accept the config currently on disk."""
|
||||
import subprocess
|
||||
return subprocess.run(
|
||||
['haproxy', '-c', '-f', hm.HAPROXY_CONFIG_PATH],
|
||||
capture_output=True).returncode == 0
|
||||
|
||||
def generate_good_config(self):
|
||||
self.add_domain('good.example.com', 'good_backend')
|
||||
hm.generate_config()
|
||||
self.assertTrue(self.config_is_loadable(),
|
||||
'fixture precondition: first generated config must be valid')
|
||||
return self.read(hm.HAPROXY_CONFIG_PATH)
|
||||
|
||||
def break_the_config(self):
|
||||
"""Queue a domain whose rendered backend the validator rejects."""
|
||||
self.add_domain('bad.example.com', BROKEN_TOKEN + '_backend', '10.0.0.2')
|
||||
|
||||
|
||||
class TestBackupOrdering(RollbackTestCase):
|
||||
|
||||
def test_backup_is_taken_before_the_new_config_is_written(self):
|
||||
"""The ordering invariant, asserted directly.
|
||||
|
||||
Whatever create_backup() sees on disk must be the OLD config; if the
|
||||
write happens first the backup is a copy of the new config and rollback
|
||||
is meaningless.
|
||||
"""
|
||||
good = self.generate_good_config()
|
||||
|
||||
seen = {}
|
||||
real_create_backup = hm.create_backup
|
||||
|
||||
def spy(*args, **kwargs):
|
||||
seen['config_on_disk'] = self.read(hm.HAPROXY_CONFIG_PATH)
|
||||
return real_create_backup(*args, **kwargs)
|
||||
|
||||
hm.create_backup = spy
|
||||
self.addCleanup(setattr, hm, 'create_backup', real_create_backup)
|
||||
|
||||
self.add_domain('second.example.com', 'second_backend', '10.0.0.3')
|
||||
hm.generate_config()
|
||||
|
||||
self.assertIn('config_on_disk', seen,
|
||||
'create_backup() was never called during generate_config()')
|
||||
self.assertEqual(
|
||||
seen['config_on_disk'], good,
|
||||
'create_backup() ran AFTER the new config was written - the backup '
|
||||
'is a copy of the new config, so rollback cannot undo anything')
|
||||
|
||||
def test_backup_tracks_the_last_known_good_config(self):
|
||||
"""After a change that validated AND loaded, the backup is that config.
|
||||
|
||||
The rollback target is "the last configuration HAProxy actually ran",
|
||||
not "the file that happened to be there last time".
|
||||
"""
|
||||
good = self.generate_good_config()
|
||||
self.add_domain('second.example.com', 'second_backend', '10.0.0.3')
|
||||
hm.generate_config()
|
||||
|
||||
live = self.read(hm.HAPROXY_CONFIG_PATH)
|
||||
self.assertNotEqual(live, good, 'fixture sanity: the new config should differ')
|
||||
self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), live,
|
||||
'the successful config was not recorded as known-good')
|
||||
|
||||
def test_backup_is_not_promoted_when_the_change_fails(self):
|
||||
"""A config that never loaded must not become the rollback target."""
|
||||
good = self.generate_good_config()
|
||||
self.break_the_config()
|
||||
with self.assertRaises(Exception):
|
||||
hm.generate_config()
|
||||
|
||||
self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good,
|
||||
'a config that failed validation was promoted to backup')
|
||||
|
||||
|
||||
class TestRollbackEndToEnd(RollbackTestCase):
|
||||
|
||||
def test_failed_validation_leaves_the_last_good_config_on_disk(self):
|
||||
good = self.generate_good_config()
|
||||
|
||||
self.break_the_config()
|
||||
with self.assertRaises(Exception):
|
||||
hm.generate_config()
|
||||
|
||||
on_disk = self.read(hm.HAPROXY_CONFIG_PATH)
|
||||
self.assertNotIn(BROKEN_TOKEN, on_disk,
|
||||
'the rejected config is still on disk - rollback was a no-op')
|
||||
self.assertEqual(on_disk, good,
|
||||
'on-disk config is not byte-identical to the last good one')
|
||||
|
||||
def test_haproxy_would_still_start_after_a_failed_change(self):
|
||||
"""The operational consequence: the edge can still come up."""
|
||||
self.generate_good_config()
|
||||
self.break_the_config()
|
||||
with self.assertRaises(Exception):
|
||||
hm.generate_config()
|
||||
|
||||
self.assertTrue(self.config_is_loadable(),
|
||||
'HAProxy would refuse to start with the config left on disk')
|
||||
with self.assertLogs('haproxy_manager', level='INFO') as captured:
|
||||
hm.start_haproxy()
|
||||
self.assertTrue(
|
||||
any('HAProxy started successfully' in line for line in captured.output),
|
||||
f'start_haproxy() did not succeed after rollback: {captured.output}')
|
||||
|
||||
def test_blocked_ips_map_is_rolled_back_too(self):
|
||||
"""generate_config() rewrites the map file before writing haproxy.cfg."""
|
||||
self.block_ip('192.0.2.10')
|
||||
self.generate_good_config()
|
||||
good_map = self.read(hm.BLOCKED_IPS_MAP_PATH)
|
||||
|
||||
self.block_ip('198.51.100.20')
|
||||
self.break_the_config()
|
||||
with self.assertRaises(Exception):
|
||||
hm.generate_config()
|
||||
|
||||
self.assertEqual(self.read(hm.BLOCKED_IPS_MAP_PATH), good_map,
|
||||
'blocked IPs map was not rolled back with the config')
|
||||
|
||||
def test_first_run_failure_reports_that_rollback_was_impossible(self):
|
||||
"""No prior config: there is nothing to restore, and that must be said.
|
||||
|
||||
A missing backup must never be reported as a successful restore, and it
|
||||
must never be turned into "restore an empty file".
|
||||
"""
|
||||
self.break_the_config()
|
||||
with self.assertRaises(Exception) as ctx:
|
||||
hm.generate_config()
|
||||
|
||||
self.assertIn('ROLLBACK FAILED', str(ctx.exception),
|
||||
'a failed change with no backup was not reported as such')
|
||||
self.assertFalse(os.path.exists(hm.HAPROXY_BACKUP_PATH),
|
||||
'a backup was fabricated from the broken config')
|
||||
# The broken config is deliberately left in place: start_haproxy() can
|
||||
# then detect it and try to regenerate. It must not be blanked.
|
||||
self.assertGreater(os.path.getsize(hm.HAPROXY_CONFIG_PATH), 0,
|
||||
'config file was emptied instead of left for diagnosis')
|
||||
|
||||
|
||||
class TestBackupPrimitives(RollbackTestCase):
|
||||
|
||||
def test_restore_backup_distinguishes_missing_backup_from_success(self):
|
||||
restored, message = hm.restore_backup()
|
||||
self.assertFalse(restored,
|
||||
'restore_backup() reported success with no backup present')
|
||||
self.assertIn('cannot roll back', message.lower())
|
||||
|
||||
good = self.generate_good_config()
|
||||
with open(hm.HAPROXY_CONFIG_PATH, 'w') as fh:
|
||||
fh.write('scribbled over\n')
|
||||
|
||||
restored, message = hm.restore_backup()
|
||||
self.assertTrue(restored, message)
|
||||
self.assertEqual(self.read(hm.HAPROXY_CONFIG_PATH), good)
|
||||
|
||||
def test_a_successful_generation_records_a_rollback_target(self):
|
||||
"""Even the first-ever generation must leave something to roll back to."""
|
||||
good = self.generate_good_config()
|
||||
self.assertTrue(
|
||||
os.path.exists(hm.HAPROXY_BACKUP_PATH),
|
||||
'after a successful reload there is still no known-good backup')
|
||||
self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good)
|
||||
|
||||
def test_a_broken_current_config_does_not_replace_a_good_backup(self):
|
||||
"""The known-good marker.
|
||||
|
||||
If the config already on disk is broken (previous failed write, manual
|
||||
edit), snapshotting it would make "rollback" mean "restore a different
|
||||
broken config". The older validated backup must survive.
|
||||
"""
|
||||
good = self.generate_good_config()
|
||||
self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good,
|
||||
'fixture: a good backup should exist by now')
|
||||
|
||||
with open(hm.HAPROXY_CONFIG_PATH, 'w') as fh:
|
||||
fh.write(f'garbage {BROKEN_TOKEN} config\n')
|
||||
|
||||
ok, status = hm.create_backup()
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual(status, 'kept_previous')
|
||||
self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good,
|
||||
'a broken config overwrote the known-good backup')
|
||||
|
||||
def test_reload_does_not_take_its_own_backup(self):
|
||||
"""reload_haproxy_safely() runs after the write, so it must not back up."""
|
||||
good = self.generate_good_config()
|
||||
with open(hm.HAPROXY_CONFIG_PATH, 'w') as fh:
|
||||
fh.write(f'broken {BROKEN_TOKEN}\n')
|
||||
|
||||
success, message = hm.reload_haproxy_safely(backup_status='created')
|
||||
|
||||
self.assertFalse(success)
|
||||
self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good,
|
||||
'reload_haproxy_safely() overwrote the good backup')
|
||||
self.assertEqual(self.read(hm.HAPROXY_CONFIG_PATH), good,
|
||||
'reload_haproxy_safely() did not roll the config back')
|
||||
|
||||
def test_unchanged_config_is_not_revalidated(self):
|
||||
"""Fast path: if the backup already is the live config, do no work.
|
||||
|
||||
generate_config() runs inside customer-facing API calls and
|
||||
`haproxy -c` is expensive on an edge with hundreds of certificates.
|
||||
"""
|
||||
self.generate_good_config()
|
||||
|
||||
calls = []
|
||||
real_validate = hm.validate_config_file
|
||||
hm.validate_config_file = lambda path: (calls.append(path),
|
||||
real_validate(path))[1]
|
||||
self.addCleanup(setattr, hm, 'validate_config_file', real_validate)
|
||||
|
||||
ok, status = hm.create_backup()
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual(status, 'created')
|
||||
self.assertEqual(calls, [],
|
||||
'the unchanged live config was re-validated needlessly')
|
||||
|
||||
def test_fast_path_does_not_hide_a_drifted_broken_config(self):
|
||||
"""If the live config drifted from the backup, the gate must still run."""
|
||||
good = self.generate_good_config()
|
||||
with open(hm.HAPROXY_CONFIG_PATH, 'w') as fh:
|
||||
fh.write(f'hand edited {BROKEN_TOKEN}\n')
|
||||
|
||||
ok, status = hm.create_backup()
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual(status, 'kept_previous',
|
||||
'a drifted broken config was silently accepted')
|
||||
self.assertEqual(self.read(hm.HAPROXY_BACKUP_PATH), good)
|
||||
|
||||
def test_backup_set_covers_every_file_generate_config_writes(self):
|
||||
pairs = dict(hm._config_backup_pairs())
|
||||
for path in (hm.HAPROXY_CONFIG_PATH, hm.BLOCKED_IPS_MAP_PATH,
|
||||
hm.CORAZA_SPOE_CONFIG_PATH):
|
||||
self.assertIn(path, pairs,
|
||||
f'{path} is written by generate_config() but is not '
|
||||
'part of the backed-up config set')
|
||||
|
||||
def test_coraza_spoe_config_round_trips(self):
|
||||
self.generate_good_config()
|
||||
with open(hm.CORAZA_SPOE_CONFIG_PATH, 'w') as fh:
|
||||
fh.write('spoe-good\n')
|
||||
hm.create_backup()
|
||||
with open(hm.CORAZA_SPOE_CONFIG_PATH, 'w') as fh:
|
||||
fh.write('spoe-broken\n')
|
||||
restored, message = hm.restore_backup()
|
||||
self.assertTrue(restored, message)
|
||||
self.assertEqual(self.read(hm.CORAZA_SPOE_CONFIG_PATH), 'spoe-good\n')
|
||||
|
||||
|
||||
class TestAtomicWrite(RollbackTestCase):
|
||||
|
||||
def test_write_is_atomic_and_preserves_mode(self):
|
||||
path = os.path.join(self.etc, 'atomic.cfg')
|
||||
with open(path, 'w') as fh:
|
||||
fh.write('old')
|
||||
os.chmod(path, 0o644)
|
||||
|
||||
hm.write_config_atomically(path, 'new content\n')
|
||||
|
||||
self.assertEqual(self.read(path), 'new content\n')
|
||||
self.assertEqual(oct(os.stat(path).st_mode & 0o777), oct(0o644))
|
||||
leftovers = [n for n in os.listdir(self.etc) if n.endswith('.tmp')]
|
||||
self.assertEqual(leftovers, [], f'temp files left behind: {leftovers}')
|
||||
|
||||
def test_failed_write_leaves_the_previous_file_intact(self):
|
||||
path = os.path.join(self.etc, 'atomic.cfg')
|
||||
with open(path, 'w') as fh:
|
||||
fh.write('old content\n')
|
||||
|
||||
# Anything that makes f.write() blow up mid-flight stands in for a full
|
||||
# disk / killed container.
|
||||
with self.assertRaises(Exception):
|
||||
hm.write_config_atomically(path, object())
|
||||
|
||||
self.assertEqual(self.read(path), 'old content\n',
|
||||
'a failed write clobbered the previous config')
|
||||
leftovers = [n for n in os.listdir(self.etc) if n.endswith('.tmp')]
|
||||
self.assertEqual(leftovers, [], f'temp files left behind: {leftovers}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(f"testing haproxy_manager from: {MODULE_DIR}")
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user