Compare commits

..
2 Commits
Author SHA1 Message Date
jknapp e5b73d8fa5 Keep the migration manifest probe under its output limit on large workspaces (#70)
Build App / compute-version (push) Successful in 5s
Secret Scan / scan (push) Successful in 5s
Build App / build-macos (push) Successful in 3m15s
Build App / build-linux (push) Successful in 5m45s
Build App / build-windows (push) Successful in 7m38s
Build App / create-tag (push) Successful in 12s
Build App / sync-to-github (push) Successful in 24s
2026-11-03 00:31:38 +00:00
jknapp 1fc1f44c35 Move existing gateways onto the pinned LiteLLM after an app update (#69)
Build App / compute-version (push) Successful in 6s
Secret Scan / scan (push) Successful in 5s
Build App / build-macos (push) Successful in 3m13s
Build App / build-linux (push) Successful in 5m52s
Build App / build-windows (push) Successful in 7m21s
Build App / create-tag (push) Successful in 17s
Build App / sync-to-github (push) Successful in 32s
2026-11-03 00:24:29 +00:00
10 changed files with 1108 additions and 129 deletions

No files matched your search

+18
View File
@@ -579,6 +579,24 @@ security update. Migration is the non-destructive way out; Reset is the destruct
the last commit. Migration diffs two filesystem manifests instead, filtered through dpkg
ownership and presence-in-the-new-base. Measured on a real project, that turns 8,677 raw path
differences into 2 genuinely user-authored ones.
- **`/workspace` is reported two levels deep, never file by file** (`SUMMARISED_ROOTS`). One
`find -xdev` pass is piped through `awk`, which turns each depth-2 directory into one `S` entry
sized as the sum of the files beneath it (the same number the per-file sum gave the disk gate),
or a plain `d` if it holds no file at any depth (so empty scaffolds stay empty to the filter). A
user's worktrees and `node_modules` made a flat walk 8.6 MB — past the exec cap, so every
staleness check failed on start and stop — and pruning `node_modules`/`.git` by name only
reached 6.3 MB; now the whole manifest is ~1 MB. Summaries lose nothing: below `/workspace/<x>`
every per-path filter is vacuous, and `tar` copies a directory recursively. Three rules:
- **A failed `awk` is an error, never an empty `/workspace`.** The pipeline prints
`###SUMMARISED-OK` only on success and `check_manifest_complete` refuses a manifest without
it — otherwise the swap would destroy what the walk failed to see.
- **A summary is exempt from presence-in-the-base**, because it hides which paths are inside;
`--skip-old-files` on restore is the per-file answer. That rests on the base shipping nothing
under `/workspace` — pinned by the ignored live test
`the_base_ships_nothing_under_a_summarised_root`, and logged as a warning if a base breaks it.
- **Only `/workspace`.** The other roots stay per file: dpkg ownership and presence-in-the-base
are exactly what make their delta honest (`/usr/local/lib/node_modules`, base content in
`/opt`), and they are under 1 MB together.
### Authentication
+3 -1
View File
@@ -327,7 +327,9 @@ async fn poll_loop(
// bridge or stopping the container doesn't wait out an in-flight poll.
let discovery = tokio::select! {
_ = cancel.changed() => break,
res = exec_oneshot_limited(&container_id, cmd, PROC_NET_OUTPUT_LIMIT) => res,
res = exec_oneshot_limited(&container_id, "claude", cmd, PROC_NET_OUTPUT_LIMIT) => {
res.map(|(out, _)| out)
}
};
match discovery {
+18 -10
View File
@@ -521,8 +521,9 @@ pub fn now_epoch_secs() -> u64 {
/// container. Falls back to 1000:1000 (the image's build-time ids) if the exec
/// fails, which is strictly better than the 0:0 a default tar header carries.
pub async fn container_user_ids(container_id: &str) -> (u64, u64) {
let out = exec_oneshot_limited(
let (out, _) = exec_oneshot_limited(
container_id,
"claude",
vec!["sh".to_string(), "-c".to_string(), "id -u; id -g".to_string()],
256,
)
@@ -543,9 +544,12 @@ pub async fn container_user_ids(container_id: &str) -> (u64, u64) {
/// caller sees a byte, and what it is reading is *container-controlled* — the
/// scheduler notifications reader `cat`s up to 50 files with no size cap, and
/// the auth bridge reads `/proc/net/tcp` every two seconds. Neither has an
/// upstream bound, so this is where the bound goes. Generous enough that no
/// legitimate reader (the largest is a package manifest of a full image) comes
/// close.
/// upstream bound, so this is where the bound goes. It is the default, not a
/// promise that every reader fits: the migration manifest of a running
/// container did pass it (8.6 MB from a busy `/workspace`), and now runs under
/// its own `migration::MANIFEST_OUTPUT_LIMIT` through [`exec_oneshot_limited`].
/// A reader that can legitimately grow with the user's files should do the
/// same rather than raise this.
pub const MAX_ONESHOT_OUTPUT: usize = 8 * 1024 * 1024;
/// The auth bridge's per-tick budget. It reads two procfs files whose rows are
@@ -586,16 +590,20 @@ pub async fn exec_oneshot(container_id: &str, cmd: Vec<String>) -> Result<String
exec_oneshot_env(container_id, cmd, Vec::new()).await
}
/// [`exec_oneshot`] with a caller-chosen output ceiling, for readers whose
/// input is fully container-controlled and whose legitimate output is small.
/// A one-shot exec as `user` with a caller-chosen output ceiling, returning the
/// combined output and the exit code.
///
/// For readers whose budget is not [`MAX_ONESHOT_OUTPUT`]: smaller where the
/// input is container-controlled and the legitimate output tiny (the auth
/// bridge's procfs poll, the uid probe), larger where it can genuinely pass
/// 8 MiB (the migration manifest, as root).
pub async fn exec_oneshot_limited(
container_id: &str,
user: &str,
cmd: Vec<String>,
limit: usize,
) -> Result<String, String> {
exec_oneshot_inner(container_id, "claude", cmd, Vec::new(), limit)
.await
.map(|(output, _)| output)
) -> Result<(String, i64), String> {
exec_oneshot_inner(container_id, user, cmd, Vec::new(), limit).await
}
/// Like `exec_oneshot`, but passes additional environment variables to the exec
+437 -75
View File
@@ -21,7 +21,7 @@
//! `docker inspect`.
use bollard::container::{
Config, CreateContainerOptions, ListContainersOptions, RemoveContainerOptions,
Config, CreateContainerOptions, ListContainersOptions,
StartContainerOptions, StopContainerOptions, UploadToContainerOptions,
};
use bollard::image::BuildImageOptions;
@@ -49,8 +49,176 @@ const GATEWAY_CONFIG_VOLUME: &str = "triple-c-gateway-config";
/// also above the floors set by the proxy auth-bypass, SSRF and salt-key CVEs —
/// see the long comment in `gateway-container/Dockerfile`, and keep the two in
/// lockstep.
const GATEWAY_REGISTRY_IMAGE: &str = "ghcr.io/berriai/litellm:v1.106.6";
const GATEWAY_LOCAL_IMAGE: &str = "triple-c-gateway:latest";
const GATEWAY_REGISTRY_REPO: &str = "ghcr.io/berriai/litellm";
const GATEWAY_LITELLM_VERSION: &str = "v1.106.6";
/// The digest [`GATEWAY_REGISTRY_REPO`]:[`GATEWAY_LITELLM_VERSION`] is pinned to — the same one the
/// Dockerfile's `FROM` uses. Pulls and lookups go by digest: a tag can be
/// re-pushed upstream, a digest cannot.
const GATEWAY_REGISTRY_DIGEST: &str =
"sha256:37e2dafb088a7723beb998590682957386dfc83ffea1848486f8e0e5cfac0b06";
fn registry_image_ref() -> String {
format!("{}@{}", GATEWAY_REGISTRY_REPO, GATEWAY_REGISTRY_DIGEST)
}
/// Local builds are tagged with the pin plus a hash of the embedded build
/// context (`triple-c-gateway:v1.106.6-1a2b3c4d`), never a floating tag. A
/// build made by an app version with a different pin *or* a different
/// Dockerfile/config then simply doesn't match, instead of quietly winning
/// over the current image the way the old `triple-c-gateway:latest` tag did.
/// Builds tagged `latest` by earlier app versions are no longer used.
const GATEWAY_LOCAL_REPO: &str = "triple-c-gateway";
/// The LiteLLM release this app is pinned to — the tag of
/// [`GATEWAY_REGISTRY_REPO`]:[`GATEWAY_LITELLM_VERSION`], e.g. `v1.106.6`.
fn pinned_litellm_version() -> &'static str {
GATEWAY_LITELLM_VERSION
}
fn local_image_ref() -> String {
static REF: OnceLock<String> = OnceLock::new();
REF.get_or_init(|| {
// config.yaml is hashed whole: a `#` line can sit inside a block
// scalar there, and it is rarely edited.
let context = sha256_hex(&format!(
"{}\0{}",
significant_lines(GATEWAY_DOCKERFILE),
GATEWAY_DEFAULT_CONFIG
));
format!(
"{}:{}-{}",
GATEWAY_LOCAL_REPO,
pinned_litellm_version(),
&context[..8]
)
})
.clone()
}
/// The Dockerfile minus full-line comments and blank lines, so rewording a
/// comment in a later release doesn't make every local build look outdated.
/// Parser directives (`# syntax=`, `# escape=`, `# check=`) look like comments
/// but change the build, so they are kept.
fn significant_lines(text: &str) -> String {
text.lines()
.map(str::trim_end)
.filter(|l| {
let t = l.trim_start();
if t.is_empty() {
return false;
}
match t.strip_prefix('#') {
None => true,
Some(rest) => rest
.split_once('=')
.map(|(key, _)| key.trim())
.is_some_and(|key| matches!(key, "syntax" | "escape" | "check")),
}
})
.collect::<Vec<_>>()
.join("\n")
}
/// The image the gateway container should run, and its content id. The id goes
/// into the container fingerprint so that a newly pulled or rebuilt image
/// recreates the container instead of leaving the old one running.
#[derive(Debug, Clone, PartialEq)]
struct ChosenImage {
reference: String,
id: String,
}
/// Local build first (same precedence as the STT container), then the pinned
/// upstream image. Both references name the pin, so either is current.
fn choose_gateway_image(local_id: Option<String>, registry_id: Option<String>) -> Option<ChosenImage> {
local_id
.map(|id| ChosenImage {
reference: local_image_ref(),
id,
})
.or_else(|| {
registry_id.map(|id| ChosenImage {
reference: registry_image_ref(),
id,
})
})
}
/// The existing gateway container, as far as `ensure` and status need it.
#[derive(Debug, Clone)]
struct GatewayContainer {
id: String,
state: String,
/// The config-fingerprint label it was created with.
fingerprint: String,
image_id: Option<String>,
}
/// Ids of the locally present images that carry the current pin.
#[derive(Debug, Clone, Default, PartialEq)]
struct PinnedImages {
local: Option<String>,
registry: Option<String>,
}
impl PinnedImages {
fn chosen(&self) -> Option<ChosenImage> {
choose_gateway_image(self.local.clone(), self.registry.clone())
}
}
/// A failed lookup of one image doesn't hide the other. An error is returned
/// only when it leaves the answer unknown — so "Docker is unreachable" is not
/// mistaken for "image missing".
async fn pinned_images() -> Result<PinnedImages, String> {
let (local_ref, registry_ref) = (local_image_ref(), registry_image_ref());
let (local, registry) = tokio::join!(
super::migration::image_id(&local_ref),
super::migration::image_id(&registry_ref)
);
match (local, registry) {
(Ok(local), Ok(registry)) => Ok(PinnedImages { local, registry }),
// The local build is preferred, so it alone decides when present.
(Ok(Some(local)), Err(e)) => {
log::warn!("{}", e);
Ok(PinnedImages {
local: Some(local),
registry: None,
})
}
// But a failed lookup of the preferred local build leaves the answer
// unknown even when the registry image is there: choosing the
// registry image would swap a working local-build gateway over.
(Err(e), _) | (_, Err(e)) => Err(e),
}
}
/// Whether the container should be moved onto `pinned` — the image half of
/// [`needs_recreate`]. A container whose image id is unknown counts as not on
/// it, in both places.
fn image_switch_pending(existing_image_id: Option<&str>, pinned: &ChosenImage) -> bool {
existing_image_id != Some(pinned.id.as_str())
}
/// Status's `image_outdated`: the container exists and isn't on a pinned
/// image. With a pinned image present this is exactly [`image_switch_pending`]
/// — what the next start or settings save acts on. With none (`Some(None)`) it
/// is outdated, but `ensure` keeps it as is until one is pulled or built.
/// `None` means the image check failed: unknown is reported as not outdated
/// rather than guessed.
fn container_image_outdated(
container_exists: bool,
container_image_id: Option<&str>,
chosen: Option<Option<&ChosenImage>>,
) -> bool {
match chosen {
_ if !container_exists => false,
Some(Some(pinned)) => image_switch_pending(container_image_id, pinned),
Some(None) => true,
None => false,
}
}
const GATEWAY_DOCKERFILE: &str = include_str!("../../../../gateway-container/Dockerfile");
const GATEWAY_DEFAULT_CONFIG: &str = include_str!("../../../../gateway-container/config.yaml");
@@ -201,6 +369,27 @@ async fn bridge_gateway_ip(docker: &Docker) -> Option<String> {
.find_map(|c| c.gateway.filter(|g| !g.trim().is_empty()))
}
/// Config and secret version only — the same format earlier versions wrote,
/// so updating the app doesn't by itself recreate every gateway. Image changes
/// are handled separately by [`needs_recreate`].
fn gateway_fingerprint(config_shape: &str, secret_version: &str) -> String {
sha256_hex(&format!("{}|{}", config_shape, secret_version))
}
/// The one rule for replacing an existing container: its config changed, or a
/// pinned image is present and the container runs something else. With no
/// pinned image (`pinned: None`) the container keeps its current image.
/// [`container_image_outdated`] reports the image half of this to the UI.
fn needs_recreate(
existing_fingerprint: &str,
fingerprint: &str,
existing_image_id: Option<&str>,
pinned: Option<&ChosenImage>,
) -> bool {
existing_fingerprint != fingerprint
|| pinned.is_some_and(|p| image_switch_pending(existing_image_id, p))
}
fn sha256_hex(input: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(input.as_bytes());
@@ -208,23 +397,32 @@ fn sha256_hex(input: &str) -> String {
}
pub async fn get_gateway_status(settings: &GatewaySettings) -> Result<GatewayStatus, String> {
let image_exists = super::image::image_exists(GATEWAY_REGISTRY_IMAGE)
.await
.unwrap_or(false)
|| super::image::image_exists(GATEWAY_LOCAL_IMAGE)
.await
.unwrap_or(false);
let (container_exists, running) = match find_gateway_container().await? {
Some((_, state, _)) => (true, state == "running"),
let (pinned, container) = tokio::join!(pinned_images(), find_gateway_container());
let chosen = match pinned {
Ok(p) => Some(p.chosen()),
Err(e) => {
log::warn!("Gateway image check failed: {}", e);
None
}
};
let image_exists = matches!(chosen, Some(Some(_)));
let container = container?;
let (container_exists, running) = match &container {
Some(c) => (true, c.state == "running"),
None => (false, false),
};
let image_outdated = container_image_outdated(
container_exists,
container.as_ref().and_then(|c| c.image_id.as_deref()),
chosen.as_ref().map(|c| c.as_ref()),
);
Ok(GatewayStatus {
container_exists,
running,
port: settings.port,
image_exists,
image_outdated,
model_count: settings.valid_models().len(),
has_api_key: secure::has_gateway_api_key(),
base_url: gateway_binding().await.base_url(settings.port),
@@ -235,7 +433,7 @@ pub async fn get_gateway_status(settings: &GatewaySettings) -> Result<GatewaySta
/// settings reconcile, which must not start anything the user never started.
pub async fn gateway_container_presence() -> Result<(bool, bool), String> {
Ok(match find_gateway_container().await? {
Some((_, state, _)) => (true, state == "running"),
Some(c) => (true, c.state == "running"),
None => (false, false),
})
}
@@ -252,8 +450,8 @@ fn is_gateway_container(names: Option<&Vec<String>>) -> bool {
names.is_some_and(|names| names.iter().any(|n| n == &expected))
}
/// `(id, state, config fingerprint label)` for the gateway container, if any.
async fn find_gateway_container() -> Result<Option<(String, String, String)>, String> {
/// The gateway container, if any.
async fn find_gateway_container() -> Result<Option<GatewayContainer>, String> {
let docker = get_docker()?;
let filters: HashMap<String, Vec<String>> = HashMap::from([(
@@ -284,7 +482,14 @@ async fn find_gateway_container() -> Result<Option<(String, String, String)>, St
.cloned()
.unwrap_or_default();
return Ok(Some((id, state, fingerprint)));
let image_id = container.image_id.clone().filter(|s| !s.is_empty());
return Ok(Some(GatewayContainer {
id,
state,
fingerprint,
image_id,
}));
}
Ok(None)
@@ -429,28 +634,11 @@ async fn upload_config(container_id: &str, config: &str) -> Result<(), String> {
async fn create_gateway_container(
settings: &GatewaySettings,
binding: &GatewayBinding,
image: &ChosenImage,
fingerprint: &str,
) -> Result<String, String> {
let docker = get_docker()?;
// Local build first, then the pinned upstream image — same precedence as
// the STT container.
let image = if super::image::image_exists(GATEWAY_LOCAL_IMAGE)
.await
.unwrap_or(false)
{
GATEWAY_LOCAL_IMAGE.to_string()
} else if super::image::image_exists(GATEWAY_REGISTRY_IMAGE)
.await
.unwrap_or(false)
{
GATEWAY_REGISTRY_IMAGE.to_string()
} else {
return Err(
"Gateway image not found. Please pull or build the image first.".to_string(),
);
};
let mut port_bindings = HashMap::new();
port_bindings.insert(
format!("{}/tcp", GATEWAY_INTERNAL_PORT),
@@ -491,7 +679,9 @@ async fn create_gateway_container(
);
let config = Config {
image: Some(image),
// The resolved id, not the tag: a build or pull retagging the image
// mid-create can't make the container disagree with its fingerprint.
image: Some(image.id.clone()),
// The upstream entrypoint (`docker/prod_entrypoint.sh`) execs
// `litellm "$@"`. Passed explicitly so the pulled upstream image and
// our locally built one behave identically.
@@ -522,6 +712,20 @@ async fn create_gateway_container(
Ok(response.id)
}
/// Stops a running container gracefully first, with Docker's default grace as
/// before (so in-flight LiteLLM requests can drain; the shorter
/// [`GATEWAY_STOP_GRACE_SECS`] is for app exit), then removes it, tolerating
/// one already gone.
async fn remove_gateway_container(docker: &Docker, id: &str, state: &str) -> Result<(), String> {
if state == "running" {
docker
.stop_container(id, None::<StopContainerOptions>)
.await
.map_err(|e| format!("Failed to stop gateway container: {}", e))?;
}
super::container::remove_container(id).await
}
/// Serialises every mutation of the single fixed-name gateway container.
///
/// `ensure_gateway_running` is check-then-act over one container name, so two
@@ -565,14 +769,66 @@ async fn ensure_gateway_running_locked(
// Rotation id, not a hash of either secret — see `storage::secure`.
let secret_version = secure::get_gateway_secret_version()?.unwrap_or_default();
let fingerprint = sha256_hex(&format!(
"{}|{}",
config_shape(settings, &binding),
secret_version
));
let (existing, pinned) = tokio::join!(find_gateway_container(), pinned_images());
let existing = existing?;
// A failed image lookup must not fail a gateway that's fine as it is: it
// takes the same path as "no pinned image" and keeps the current image.
let mut lookup_error = None;
let chosen = match pinned {
Ok(pinned) => pinned.chosen(),
Err(e) => {
log::warn!("Gateway image check failed: {}", e);
lookup_error = Some(e);
None
}
};
let on_pin = chosen.is_some();
let image = match chosen {
Some(image) => image,
// No pinned image yet (after an app update moved the pin, until the
// user pulls or builds). Never tear down a working gateway over that:
// keep it on the image it already runs — recreated with the new
// config if that changed — and let status report it as outdated.
None => match existing.as_ref().and_then(|c| c.image_id.clone()) {
Some(id) => {
log::warn!(
"Gateway image for LiteLLM {} not present; keeping the gateway on its current image {}",
pinned_litellm_version(),
id
);
ChosenImage {
reference: id.clone(),
id,
}
}
None => {
return Err(match lookup_error {
Some(e) => e,
None => format!(
"Gateway image for LiteLLM {} not found. Please pull or build the image first.",
pinned_litellm_version()
),
})
}
},
};
if let Some((id, state, existing_fingerprint)) = find_gateway_container().await? {
if existing_fingerprint == fingerprint {
let fingerprint = gateway_fingerprint(&config_shape(settings, &binding), &secret_version);
if let Some(GatewayContainer {
id,
state,
fingerprint: existing_fingerprint,
image_id: existing_image_id,
}) = existing
{
let pinned = on_pin.then_some(&image);
if !needs_recreate(
&existing_fingerprint,
&fingerprint,
existing_image_id.as_deref(),
pinned,
) {
if state == "running" {
return get_gateway_status(settings).await;
}
@@ -583,41 +839,21 @@ async fn ensure_gateway_running_locked(
return get_gateway_status(settings).await;
}
// Config or a secret changed — recreate so the new config is uploaded.
if state == "running" {
docker
.stop_container(&id, None::<StopContainerOptions>)
.await
.map_err(|e| format!("Failed to stop gateway container: {}", e))?;
}
docker
.remove_container(
&id,
Some(RemoveContainerOptions {
force: true,
..Default::default()
}),
)
.await
.map_err(|e| format!("Failed to remove gateway container: {}", e))?;
// Config, a secret or the image changed — recreate so the new config
// is uploaded and the pinned image is used.
remove_gateway_container(&docker, &id, &state).await?;
}
let id = create_gateway_container(settings, &binding, &fingerprint).await?;
let id = create_gateway_container(settings, &binding, &image, &fingerprint).await?;
// Upload before the first start: LiteLLM reads the config once at boot.
let rendered = render_config(settings, &api_key, &master_key);
if let Err(e) = upload_config(&id, &rendered).await {
// Don't leave a half-configured container behind for the next run to
// mistake for a good one.
let _ = docker
.remove_container(
&id,
Some(RemoveContainerOptions {
force: true,
..Default::default()
}),
)
.await;
if let Err(cleanup) = super::container::remove_container(&id).await {
log::warn!("{}", cleanup);
}
return Err(e);
}
@@ -627,12 +863,12 @@ async fn ensure_gateway_running_locked(
.map_err(|e| format!("Failed to start gateway container: {}", e))?;
log::info!(
"Model gateway started on {}:{} ({} model(s))",
"Model gateway started on {}:{} ({} model(s), {})",
binding.host_ip,
settings.port,
settings.valid_models().len()
settings.valid_models().len(),
image.reference
);
get_gateway_status(settings).await
}
@@ -646,7 +882,7 @@ pub async fn stop_gateway_container() -> Result<(), String> {
let _guard = gateway_lock().lock().await;
let docker = get_docker()?;
if let Some((id, state, _)) = find_gateway_container().await? {
if let Some(GatewayContainer { id, state, .. }) = find_gateway_container().await? {
if state == "running" {
docker
.stop_container(
@@ -690,7 +926,27 @@ pub async fn pull_gateway_image<F>(on_progress: F) -> Result<(), String>
where
F: Fn(String) + Send + 'static,
{
super::image::pull_image(GATEWAY_REGISTRY_IMAGE, on_progress).await
super::image::pull_image(&registry_image_ref(), on_progress).await?;
// A pull by digest leaves the image untagged, which `docker image prune`
// treats as dangling and deletes. Tag it locally; lookups still go by
// digest, so a re-pushed upstream tag changes nothing.
// Not fatal: the image is present and will be used either way.
if let Err(e) = get_docker()?
.tag_image(
&registry_image_ref(),
Some(bollard::image::TagImageOptions {
repo: GATEWAY_REGISTRY_REPO,
tag: GATEWAY_LITELLM_VERSION,
}),
)
.await
{
log::warn!(
"Pulled the gateway image but could not tag it (a prune may remove it): {}",
e
);
}
Ok(())
}
pub async fn build_gateway_image<F>(on_progress: F) -> Result<(), String>
@@ -702,8 +958,9 @@ where
let tar_bytes = create_gateway_build_context()
.map_err(|e| format!("Failed to create gateway build context: {}", e))?;
let tag = local_image_ref();
let options = BuildImageOptions {
t: GATEWAY_LOCAL_IMAGE,
t: tag.as_str(),
rm: true,
forcerm: true,
..Default::default()
@@ -785,6 +1042,111 @@ mod tests {
}
}
fn chosen(reference: &str, id: &str) -> ChosenImage {
ChosenImage {
reference: reference.to_string(),
id: id.to_string(),
}
}
#[test]
fn local_builds_are_tagged_with_the_pin_and_context() {
let v = pinned_litellm_version();
assert!(v.starts_with('v') && !v.contains(['/', '@', ':']), "bad pin tag {v:?}");
let tag = local_image_ref();
let suffix = tag
.strip_prefix(&format!("triple-c-gateway:{}-", v))
.expect("tag is repo:pin-hash");
assert_eq!(suffix.len(), 8);
assert!(suffix.chars().all(|c| c.is_ascii_hexdigit()));
}
#[test]
fn registry_lookups_go_by_digest() {
assert_eq!(
registry_image_ref(),
format!("ghcr.io/berriai/litellm@{}", GATEWAY_REGISTRY_DIGEST)
);
}
#[test]
fn dockerfile_from_matches_the_registry_pin() {
// Local builds are tagged with the pin; that is only truthful while the
// Dockerfile builds FROM the same tag.
let from = GATEWAY_DOCKERFILE
.lines()
.find(|l| l.starts_with("FROM "))
.expect("Dockerfile has a FROM line");
let expected = format!(
"FROM {}:{}@{}",
GATEWAY_REGISTRY_REPO, GATEWAY_LITELLM_VERSION, GATEWAY_REGISTRY_DIGEST
);
assert_eq!(from.trim(), expected, "gateway Dockerfile FROM is out of lockstep with the pin");
}
#[test]
fn local_build_wins_then_registry_then_nothing() {
assert_eq!(
choose_gateway_image(Some("sha256:local".into()), Some("sha256:reg".into())),
Some(chosen(&local_image_ref(), "sha256:local"))
);
assert_eq!(
choose_gateway_image(None, Some("sha256:reg".into())),
Some(chosen(&registry_image_ref(), "sha256:reg"))
);
assert_eq!(choose_gateway_image(None, None), None);
}
#[test]
fn outdated_matches_what_ensure_would_recreate() {
let pinned = chosen(&registry_image_ref(), "sha256:new");
assert!(!container_image_outdated(true, Some("sha256:new"), Some(Some(&pinned))));
assert!(container_image_outdated(true, Some("sha256:old"), Some(Some(&pinned))));
// No pinned image at all: whatever runs is not it.
assert!(container_image_outdated(true, Some("sha256:old"), Some(None)));
// No container, or the check failed: don't claim it.
assert!(!container_image_outdated(false, None, Some(Some(&pinned))));
assert!(!container_image_outdated(true, Some("sha256:old"), None));
// A container whose image id is unknown: status and ensure agree
// that it isn't on the pinned image.
assert!(container_image_outdated(true, None, Some(Some(&pinned))));
assert!(needs_recreate("fp", "fp", None, Some(&pinned)));
}
#[test]
fn comment_edits_dont_change_the_local_build_tag() {
let a = "# about\nFROM x\n\n # indented note\nRUN y\n";
let b = "# reworded\nFROM x\nRUN y\n# trailing\n";
assert_eq!(significant_lines(a), significant_lines(b));
assert_ne!(significant_lines(a), significant_lines("FROM x\nRUN z\n"));
// A parser directive changes the build.
assert_ne!(
significant_lines(b),
significant_lines(&format!("# syntax=docker/dockerfile:1.7\n{b}"))
);
}
#[test]
fn fingerprint_format_is_unchanged_from_earlier_versions() {
// What main wrote: sha256 of "shape|secret". Keeping it means the app
// update alone recreates nothing.
assert_eq!(gateway_fingerprint("shape", "1"), sha256_hex("shape|1"));
}
#[test]
fn recreate_on_config_change_or_a_present_new_pinned_image() {
let pinned = chosen(&registry_image_ref(), "sha256:new");
// Unchanged config, already on the pinned image: keep.
assert!(!needs_recreate("fp", "fp", Some("sha256:new"), Some(&pinned)));
// Pinned image present, container on another: recreate.
assert!(needs_recreate("fp", "fp", Some("sha256:old"), Some(&pinned)));
// No pinned image: keep the working gateway on its image.
assert!(!needs_recreate("fp", "fp", Some("sha256:old"), None));
// Config changed: always recreate.
assert!(needs_recreate("old", "fp", Some("sha256:new"), Some(&pinned)));
assert!(needs_recreate("old", "fp", Some("sha256:old"), None));
}
#[test]
fn valid_models_skips_incomplete_rows() {
assert_eq!(settings().valid_models().len(), 1);
+36 -13
View File
@@ -69,25 +69,32 @@ pub async fn get_local_image_digest(image_name: &str) -> Result<Option<String>,
Ok(None)
}
/// Split an image reference into the `fromImage` and `tag` the pull API takes.
/// A digest reference (`repo@sha256:…`, optionally `repo:tag@sha256:…`) is
/// pulled by digest: the API takes the digest in the `tag` field.
fn split_image_ref(image_name: &str) -> (&str, &str) {
if let Some((name, digest)) = image_name.split_once('@') {
let repo = match name.rfind(':') {
Some(pos) if !name[pos + 1..].contains('/') => &name[..pos],
_ => name,
};
return (repo, digest);
}
match image_name.rfind(':') {
// The colon is part of a port (e.g., host:port/repo), not a tag.
Some(pos) if image_name[pos + 1..].contains('/') => (image_name, "latest"),
Some(pos) => (&image_name[..pos], &image_name[pos + 1..]),
None => (image_name, "latest"),
}
}
pub async fn pull_image<F>(image_name: &str, on_progress: F) -> Result<(), String>
where
F: Fn(String) + Send + 'static,
{
let docker = get_docker()?;
// Parse image name into from_image and tag
let (from_image, tag) = if let Some(pos) = image_name.rfind(':') {
// Check that the colon is part of a tag, not a port
let after_colon = &image_name[pos + 1..];
if after_colon.contains('/') {
// The colon is part of a port (e.g., host:port/repo)
(image_name, "latest")
} else {
(&image_name[..pos], after_colon)
}
} else {
(image_name, "latest")
};
let (from_image, tag) = split_image_ref(image_name);
let options = CreateImageOptions {
from_image,
@@ -390,3 +397,19 @@ mod apt_refresh_tests {
assert!(BUILD_WORKFLOW.contains("push: ${{ gitea.event_name != 'pull_request' }}"));
}
}
#[cfg(test)]
mod pull_ref_tests {
use super::split_image_ref;
#[test]
fn split_image_ref_handles_tags_ports_and_digests() {
assert_eq!(split_image_ref("repo:v1"), ("repo", "v1"));
assert_eq!(split_image_ref("repo"), ("repo", "latest"));
assert_eq!(split_image_ref("host:5000/repo"), ("host:5000/repo", "latest"));
assert_eq!(split_image_ref("host:5000/repo:v1"), ("host:5000/repo", "v1"));
assert_eq!(split_image_ref("ghcr.io/a/b@sha256:abc"), ("ghcr.io/a/b", "sha256:abc"));
assert_eq!(split_image_ref("ghcr.io/a/b:v1@sha256:abc"), ("ghcr.io/a/b", "sha256:abc"));
assert_eq!(split_image_ref("host:5000/b@sha256:abc"), ("host:5000/b", "sha256:abc"));
}
}
+532 -13
View File
@@ -94,6 +94,68 @@ pub const MANIFEST_ROOTS: &[&str] = &[
"/var/www",
];
/// [`MANIFEST_ROOTS`] entries reported only to [`SUMMARY_DEPTH`], with every
/// directory at that depth that holds at least one non-directory (at any depth)
/// reported as one [`SUMMARY_KIND`] entry whose size is the sum of those
/// files' sizes — exactly what [`verbatim_payload_bytes`] would have summed
/// from a per-file listing. A directory holding no file is reported as a plain
/// `d`, so the empty-tree filter still drops it.
///
/// **Why `/workspace`, and why it has to be.** It is the one root whose size is
/// the user's rather than the image's: git worktrees, `node_modules`, a pnpm
/// store and vendored trees all sit in the writable layer beside the bind
/// mounts. A real container listed 94k entries — 8.6 MB of `find` output —
/// under `/workspace` alone, which overflowed
/// [`crate::docker::exec::MAX_ONESHOT_OUTPUT`] and failed the staleness check
/// on every start and stop. Pruning by name (`node_modules`, `.git`) was
/// measured on that container and only got it to 6.3 MB: most of it was a
/// vendored phpMyAdmin in each of eight worktrees. Depth is what bounds it.
///
/// **Why summarising loses nothing.** Every per-path filter in
/// [`compute_verbatim_paths`] is vacuous below `/workspace/<x>`: dpkg owns
/// nothing there, no [`COPY_EXCLUSIONS`] entry is there, bind mounts are
/// exactly `/workspace/<mount_name>` (a mount name cannot hold a `/`), and the
/// base ships an empty `/workspace`. So the verbatim set was already pruned to
/// first-level entries; a summary just stops listing what was going to be
/// pruned away. Migration's `tar` still archives the directory recursively.
///
/// **One `find` pass, aggregated by `awk`.** The listing and the sizes come
/// from the same walk, so they cannot disagree under concurrent writes, and
/// `-xdev` keeps it off the bind mounts (a mount point is listed, never
/// entered). Because the walk's presence then depends on `awk`, the pipeline
/// prints `###SUMMARISED-OK` only when `awk` succeeded, and
/// [`check_manifest_complete`] refuses a manifest without it — a missing
/// `/workspace` must never read as an empty one.
///
/// **Why the base-presence exemption is safe.** See the check in
/// [`compute_verbatim_paths`]; it rests on the base shipping nothing under
/// these roots, which `the_base_ships_nothing_under_a_summarised_root` pins and
/// which is logged loudly if it ever stops being true.
pub const SUMMARISED_ROOTS: &[&str] = &["/workspace"];
/// Depth below a [`SUMMARISED_ROOTS`] entry at which directories are
/// summarised instead of listed. 1 would be just as correct for migration (the
/// verbatim set prunes to first-level entries anyway); 2 keeps the immediate
/// contents of each top-level folder visible in the manifest for debugging, at
/// a few hundred lines on the container that overflowed.
pub const SUMMARY_DEPTH: usize = 2;
/// Marker the summarised walk prints after a successful pass, once per root.
const SUMMARISED_OK: &str = "SUMMARISED-OK";
/// [`ManifestEntry::kind`] of a summarised directory. Not a `find` `%y` letter.
pub const SUMMARY_KIND: char = 'S';
/// Output budget for one manifest exec against a running container.
///
/// Above [`crate::docker::exec::MAX_ONESHOT_OUTPUT`] on purpose. With
/// [`SUMMARISED_ROOTS`] a real container's manifest is a few MB (`/usr/local`
/// ~0.8 MB, `/var/lib` ~0.1 MB, the dpkg section a few hundred kB), but the
/// non-summarised roots are still walked file by file and a user can install
/// anything into `/opt`. Failing at 8 MiB cost the user every staleness check;
/// 32 MiB keeps that rare while still bounding a container-controlled read.
pub const MANIFEST_OUTPUT_LIMIT: usize = 32 * 1024 * 1024;
/// Roots holding **state a base-image swap destroys and no replay can put
/// back**. Reported by [`unpreserved_data`], never copied.
///
@@ -173,16 +235,28 @@ pub const PROBE_LABEL_MIGRATION: &str = "migration";
/// One entry from the filesystem walk.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ManifestEntry {
/// `find`'s `%y`: `f` regular, `d` directory, `l` symlink, …
/// `find`'s `%y`: `f` regular, `d` directory, `l` symlink, … — or
/// [`SUMMARY_KIND`] for a directory under a [`SUMMARISED_ROOTS`] entry that
/// was not listed file by file.
pub kind: char,
/// `find`'s `%s`; for a summary, the sum of `%s` over every non-directory
/// beneath it, from the same pass.
pub size: u64,
pub path: String,
}
impl ManifestEntry {
/// A directory reported as itself, with nothing implied about content.
/// A summary is deliberately *not* one: it stands for a subtree that holds
/// at least one file.
pub fn is_dir(&self) -> bool {
self.kind == 'd'
}
/// A directory whose contents were aggregated rather than listed.
pub fn is_summary(&self) -> bool {
self.kind == SUMMARY_KIND
}
}
/// Everything one probe run learned about an image or a running container.
@@ -202,6 +276,9 @@ pub struct Manifest {
pub etc_paths: BTreeSet<String>,
/// `package -> version` for every installed dpkg package.
pub dpkg_versions: BTreeMap<String, String>,
/// How many [`SUMMARISED_ROOTS`] walks reported finishing. See
/// [`check_manifest_complete`].
pub summarised_roots_complete: usize,
}
impl Manifest {
@@ -217,18 +294,26 @@ impl Manifest {
/// Sections are separated by sentinel lines so one exec answers every question;
/// on a 5.49 GB image the whole thing takes about three seconds. Every command
/// is failure-tolerant (`2>/dev/null`, no `set -e`) because a missing `npm` or
/// an unreadable directory must degrade one section, not the run.
/// an unreadable directory must degrade one section, not the run — with one
/// exception, the summarised walk, whose failure [`check_manifest_complete`]
/// turns into an error because degrading it would lose data.
pub fn manifest_script() -> String {
let feature_paths = FEATURE_PROBES
.iter()
.map(|(p, _)| shell_single_quote(p))
.collect::<Vec<_>>()
.join(" ");
let roots = MANIFEST_ROOTS
let plain_roots: Vec<&str> = MANIFEST_ROOTS
.iter()
.map(|p| shell_single_quote(p))
.copied()
.filter(|p| !SUMMARISED_ROOTS.contains(p))
.collect();
let plain = plain_walk(&plain_roots);
let summarised = SUMMARISED_ROOTS
.iter()
.map(|r| summarised_walk(r))
.collect::<Vec<_>>()
.join(" ");
.join("\necho '###PATHS'\n");
// The dpkg grep is anchored to the manifest roots so the section stays a
// few hundred kB instead of the ~40 MB a full ownership dump would be.
let dpkg_filter = MANIFEST_ROOTS
@@ -239,7 +324,9 @@ pub fn manifest_script() -> String {
format!(
r#"
echo '###PATHS'
find {roots} -xdev -printf '%y\t%s\t%p\n' 2>/dev/null
{plain}
echo '###PATHS'
{summarised}
echo '###DPKG'
cat /var/lib/dpkg/info/*.list 2>/dev/null | grep -E '^/({dpkg_filter})(/|$)'
echo '###APT'
@@ -260,6 +347,90 @@ exit 0
)
}
/// A per-file `find` over `roots`, or nothing at all when there are none.
///
/// The empty case is not cosmetic: `find -xdev …` with no starting point walks
/// `.`, which in a probe container is `/` — the whole filesystem, straight into
/// an output-capped manifest.
fn plain_walk(roots: &[&str]) -> String {
if roots.is_empty() {
return String::new();
}
let roots = roots
.iter()
.map(|p| shell_single_quote(p))
.collect::<Vec<_>>()
.join(" ");
format!("find {} -xdev -printf '%y\\t%s\\t%p\\n' 2>/dev/null", roots)
}
/// Aggregates a per-file `find` listing of one root (`kind<TAB>size<TAB>path`)
/// into [`SUMMARY_DEPTH`]-limited `###PATHS` lines. See [`SUMMARISED_ROOTS`].
///
/// Entries above the boundary are printed as they arrive; a directory *at* the
/// boundary is held back and printed at the end — as `S` with the summed size
/// of every non-directory beneath it, or as its own `d` line when it has none,
/// so an empty scaffold stays empty to the filter. A deeper entry whose
/// boundary ancestor was never seen is printed rather than dropped (it cannot
/// happen in a pre-order walk; erring towards over-listing is the safe side).
/// The path is everything after the second tab, so a tab in a name survives.
/// POSIX `awk` only: the image has `mawk`.
const SUMMARY_AWK: &str = r#"BEGIN { FS = "\t"; rl = length(root) }
{
kind = $1; size = $2
p = $0; sub(/^[^\t]*\t[^\t]*\t/, "", p)
if (p == root || substr(p, 1, rl + 1) != root "/") { print; next }
n = split(substr(p, rl + 2), c, "/")
if (n < depth) { print; next }
key = root
for (i = 1; i <= depth; i++) key = key "/" c[i]
if (n == depth) {
if (kind != "d") { print; next }
order[++nb] = key; dsize[key] = size; files[key] = 0; bytes[key] = 0
next
}
if (!(key in files)) { print; next }
if (kind != "d") { files[key]++; bytes[key] += size }
}
END {
for (i = 1; i <= nb; i++) {
k = order[i]
if (files[k] > 0) printf "S\t%.0f\t%s\n", bytes[k], k
else printf "d\t%s\t%s\n", dsize[k], k
}
}"#;
/// The `###PATHS` lines for one [`SUMMARISED_ROOTS`] entry, from one `find`
/// pass piped through [`SUMMARY_AWK`], then `###SUMMARISED-OK` only if `awk`
/// succeeded.
fn summarised_walk(root: &str) -> String {
format!(
"find {q} -xdev -printf '%y\\t%s\\t%p\\n' 2>/dev/null | awk -v root={q} -v depth={d} {prog} && echo '###{ok}'",
q = shell_single_quote(root),
d = SUMMARY_DEPTH,
prog = shell_single_quote(SUMMARY_AWK),
ok = SUMMARISED_OK,
)
}
/// Refuse a manifest whose summarised walk did not finish.
///
/// Every other section degrades on failure, and that is right for them — a
/// missing `npm` costs the npm delta. This one cannot: the summarised walk is
/// the *only* record of what lives under `/workspace`, so a silently absent one
/// reads as "nothing there" and the container swap would destroy it.
pub fn check_manifest_complete(m: &Manifest) -> Result<(), String> {
if m.summarised_roots_complete < SUMMARISED_ROOTS.len() {
return Err(format!(
"Could not list {} inside the container (the probe's `awk` step did not finish), \
so Triple-C cannot tell what an update would need to carry across.",
SUMMARISED_ROOTS.join(", ")
));
}
Ok(())
}
/// Parse the output of [`manifest_script`].
///
/// Unknown sections and malformed lines are skipped rather than failing: the
@@ -271,6 +442,9 @@ pub fn parse_manifest(raw: &str) -> Manifest {
for line in raw.lines() {
let line = line.strip_suffix('\r').unwrap_or(line);
if let Some(name) = line.strip_prefix("###") {
if name == SUMMARISED_OK {
m.summarised_roots_complete += 1;
}
section = match name {
"PATHS" | "DPKG" | "APT" | "NPM" | "FEATURES" | "ETC" | "PKGVER" | "END" => name,
_ => "",
@@ -416,7 +590,8 @@ pub fn prune_to_roots(paths: &BTreeSet<String>) -> Vec<String> {
/// * it lives under a [`COPY_ROOTS`] entry,
/// * it is not under a [`COPY_EXCLUSIONS`] entry or a bind-mount target,
/// * neither image's dpkg database owns it,
/// * the current base image does not already have it.
/// * the current base image does not already have it (a [`SUMMARY_KIND`]
/// entry is exempt from this one — see the comment at the check).
///
/// The result is then pruned to subtree roots. An empty result means the copy
/// step is skipped entirely.
@@ -446,7 +621,24 @@ pub fn compute_verbatim_paths(
if from.dpkg_owned.contains(p) || base.dpkg_owned.contains(p) {
continue;
}
if base_paths.contains(p) {
// Not for a summary: "the base already has this path" is a per-path
// judgment, and a summary hides which paths are inside it. Skipping one
// would drop every user file under any subtree the base happens to ship
// too. Copying it is safe — the restore's `--skip-old-files` keeps the
// base's copy of anything both have, which is the per-file answer.
// What it cannot do is notice base content the new base *removed*, so
// this rests on the base shipping nothing under a summarised root —
// pinned by `the_base_ships_nothing_under_a_summarised_root`, and said
// out loud here if a base ever breaks it.
if base_paths.contains(p) && entry.is_summary() {
log::warn!(
"The base image ships {}, which this manifest only summarises — copying the \
container's whole subtree and relying on --skip-old-files to keep the base's \
files. SUMMARISED_ROOTS assumes the base ships nothing there; revisit it.",
p
);
}
if base_paths.contains(p) && !entry.is_summary() {
continue;
}
candidates.insert(entry.path.clone());
@@ -1061,7 +1253,29 @@ pub async fn manifest_from_image(image: &str) -> Result<Manifest, String> {
}
));
}
Ok(parse_manifest(&out.stdout))
let m = parse_manifest(&out.stdout);
check_manifest_complete(&m)?;
Ok(m)
}
/// Turn a manifest exec's failure into something the user can act on.
///
/// The one case worth naming is the output ceiling: what reached the banner was
/// "OUTPUT_LIMIT: Command output exceeded 8388608 bytes and was abandoned",
/// true of a buffer and no help about a container. Same idea as
/// `file_commands::describe_listing_failure`.
pub fn describe_manifest_failure(error: String) -> String {
if error.starts_with(super::exec::OUTPUT_LIMIT_MARKER) {
return format!(
"This container has too many files for Triple-C to inspect for an update (its \
file listing passed {} MiB). The usual cause is a very large tree installed \
under /usr/local, /opt, /srv, /var/lib, /var/www or /etc, or a huge number of \
files directly in a top-level /workspace folder. Removing it, or moving it \
into a mounted project folder, will let the check run.",
MANIFEST_OUTPUT_LIMIT / (1024 * 1024)
);
}
error
}
/// Capture a [`Manifest`] from a *running* container.
@@ -1071,20 +1285,23 @@ pub async fn manifest_from_image(image: &str) -> Result<Manifest, String> {
/// installed since the last commit, and a verbatim set computed from a stale
/// manifest would silently fail to carry that work across.
pub async fn manifest_from_container(container_id: &str) -> Result<Manifest, String> {
let (out, code) = super::exec::exec_oneshot_as(
let (out, code) = super::exec::exec_oneshot_limited(
container_id,
"root",
vec!["/bin/sh".to_string(), "-c".to_string(), manifest_script()],
Vec::new(),
MANIFEST_OUTPUT_LIMIT,
)
.await?;
.await
.map_err(describe_manifest_failure)?;
if !out.contains("###END") {
return Err(format!(
"Probe of the running container did not complete (exit {})",
code
));
}
Ok(parse_manifest(&out))
let m = parse_manifest(&out);
check_manifest_complete(&m)?;
Ok(m)
}
/// Cached stopped-container manifests, keyed by container id, each paired with
@@ -2220,6 +2437,308 @@ mod tests {
assert_eq!(shell_single_quote("/opt/a'b"), r#"'/opt/a'\''b'"#);
}
// ── /workspace summarisation (the 8 MiB overflow) ───────────────────────
#[test]
fn a_summary_carries_its_own_size_and_is_not_a_directory() {
let raw = "###PATHS\n\
d\t4096\t/workspace/wt\n\
f\t10\t/workspace/wt/README.md\n\
S\t123456\t/workspace/wt/node_modules\n\
###SUMMARISED-OK\n\
###DPKG\n\
###END\n";
let m = parse_manifest(raw);
assert_eq!(m.paths.len(), 3);
let s = &m.paths[2];
assert!(s.is_summary());
assert!(!s.is_dir(), "a summary stands for content, not an empty directory");
assert_eq!(s.size, 123456);
assert!(check_manifest_complete(&m).is_ok());
}
#[test]
fn a_manifest_whose_summarised_walk_did_not_finish_is_refused() {
// If the aggregating `awk` is missing or dies, the /workspace walk is
// simply absent. Read as "nothing there", the migration would destroy
// the user's top-level /workspace folders — so it is an error instead.
let raw = "###PATHS\nd\t4096\t/opt\n###DPKG\n###END\n";
let m = parse_manifest(raw);
let err = check_manifest_complete(&m).unwrap_err();
assert!(err.contains("/workspace"), "{}", err);
}
#[test]
fn a_summarised_subtree_is_copied_whole_and_counted_in_the_payload() {
// `/workspace/node_modules` holds nothing at its own level but deeper
// summaries: it must not look like an empty tree and be dropped.
let from = manifest(
&[
('d', 4096, "/workspace/node_modules"),
('S', 50_000, "/workspace/node_modules/typescript"),
('S', 7_000, "/workspace/node_modules/.bin"),
('d', 4096, "/workspace/wt"),
('f', 10, "/workspace/wt/README.md"),
('S', 1_000, "/workspace/wt/src"),
],
&[],
&[],
);
let verbatim = compute_verbatim_paths(&from, &Manifest::default(), &[]);
assert_eq!(
verbatim,
vec!["/workspace/node_modules".to_string(), "/workspace/wt".to_string()]
);
assert_eq!(verbatim_payload_bytes(&from, &verbatim), 58_010);
}
#[test]
fn a_summary_the_base_also_has_is_still_copied() {
// The presence-in-base filter is a per-path judgment, and a summary
// hides which paths are inside it. Dropping it would lose the user's
// files under any subtree the base happens to ship; copying it is safe
// because the restore's `--skip-old-files` never clobbers the base.
// (Today's base ships an empty /workspace — the ignored live test
// `the_base_ships_nothing_under_a_summarised_root` pins that.)
let from = manifest(
&[
('d', 4096, "/workspace/tools"),
('S', 900, "/workspace/tools/lib"),
],
&[],
&[],
);
let base = manifest(
&[
('d', 4096, "/workspace/tools"),
('S', 100, "/workspace/tools/lib"),
],
&[],
&[],
);
assert_eq!(
compute_verbatim_paths(&from, &base, &[]),
vec!["/workspace/tools/lib".to_string()]
);
}
#[test]
fn the_workspace_is_walked_once_and_never_file_by_file() {
let s = manifest_script();
assert_eq!(
s.matches("find '/workspace'").count(),
1,
"/workspace must be walked in a single pass:\n{}",
s
);
assert!(!s.contains("'/workspace' '/var/lib'"), "flat walk is back:\n{}", s);
assert!(!s.contains("du "), "sizes come from the same pass, not du:\n{}", s);
assert!(s.contains("###SUMMARISED-OK"));
}
#[test]
fn a_walk_of_no_roots_is_no_walk_at_all() {
// `find -xdev …` with no starting point walks `.`, which is `/` in a
// probe container — the whole filesystem, into the manifest.
assert_eq!(plain_walk(&[]), "");
assert!(plain_walk(&["/opt"]).starts_with("find '/opt' -xdev"));
}
/// Runs the real walk against a scratch tree shaped like the container that
/// overflowed, then a section after it, through the real parser.
#[cfg(target_os = "linux")]
#[test]
fn the_summarised_walk_is_small_exact_and_ends_cleanly() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("workspace");
for wt in 0..4 {
let nm = root.join(format!("wt{}", wt)).join("node_modules");
for pkg in 0..50 {
let p = nm.join(format!("pkg{}", pkg)).join("lib");
std::fs::create_dir_all(&p).unwrap();
for f in 0..10 {
std::fs::write(p.join(format!("f{}.js", f)), b"0123456789").unwrap();
}
}
std::fs::write(root.join(format!("wt{}", wt)).join("README.md"), b"hi").unwrap();
}
// Second-level directories holding no file at any depth: one empty,
// one with only empty subdirectories. Neither is content.
std::fs::create_dir_all(root.join("wt0").join("empty")).unwrap();
std::fs::create_dir_all(root.join("scaffold").join("a").join("b").join("c")).unwrap();
std::fs::write(root.join("loose.txt"), b"x").unwrap();
let root_s = root.to_string_lossy().to_string();
let script = format!(
"echo '###PATHS'\n{}\necho '###APT'\necho socat\necho '###END'\n",
summarised_walk(&root_s)
);
let out = std::process::Command::new("/bin/sh")
.arg("-c")
.arg(&script)
.output()
.unwrap();
let text = String::from_utf8_lossy(&out.stdout).to_string();
let m = parse_manifest(&text);
// 2,000 files below; one line per first- and second-level entry above.
assert!(m.paths.len() < 20, "walk was not summarised: {}", text);
// Exactly the old per-file sum: 50 packages × 10 files × 10 bytes.
// Directory inode sizes are not content and must not be counted.
let nm0 = format!("{}/wt0/node_modules", root_s);
let entry = m.paths.iter().find(|e| e.path == nm0).expect("summary present");
assert!(entry.is_summary());
assert_eq!(entry.size, 5_000, "{:?}", entry);
for empty in [format!("{}/wt0/empty", root_s), format!("{}/scaffold/a", root_s)] {
let e = m.paths.iter().find(|e| e.path == empty).expect("listed");
assert!(e.is_dir(), "a tree with no files is not content: {:?}", e);
}
assert!(m.paths.iter().any(|e| e.path == format!("{}/loose.txt", root_s)));
// The walk terminates its section: the next one still parses.
assert_eq!(m.apt_manual, strs(&["socat"]));
assert_eq!(m.summarised_roots_complete, 1, "{}", text);
// And the migration sees the same thing it would have per file.
let from = Manifest {
paths: m
.paths
.iter()
.map(|e| ManifestEntry {
path: e.path.replacen(&root_s, "/workspace", 1),
..e.clone()
})
.collect(),
..Default::default()
};
let verbatim = compute_verbatim_paths(&from, &Manifest::default(), &[]);
assert!(!verbatim.contains(&"/workspace/scaffold".to_string()), "{:?}", verbatim);
assert_eq!(
verbatim_payload_bytes(&from, &["/workspace/wt0".to_string()]),
5_000 + 2
);
}
/// Pins the assumption that makes summaries safe to exempt from the
/// presence-in-base check: the base ships nothing under a summarised root.
/// If a future Dockerfile puts content in /workspace this fails, and
/// `compute_verbatim_paths` needs revisiting before it ships.
///
/// ```text
/// cargo test -- --ignored --nocapture base_ships_nothing
/// ```
#[tokio::test]
#[ignore = "needs a Docker daemon and the base image"]
async fn the_base_ships_nothing_under_a_summarised_root() {
let image = std::env::var("TRIPLE_C_TEST_IMAGE")
.unwrap_or_else(|_| "ghcr.io/shadowdao/triple-c-sandbox:latest".to_string());
let m = manifest_from_image(&image).await.expect("base probe");
for root in SUMMARISED_ROOTS {
let inside: Vec<_> = m
.paths
.iter()
.filter(|e| is_under(&e.path, root) && e.path != *root)
.map(|e| e.path.as_str())
.collect();
assert!(inside.is_empty(), "the base ships {:?} under {}", inside, root);
}
}
/// The overflow end to end, against a scratch container off the real base:
/// 80k files in a writable-layer `/workspace` tree (past 8 MiB unsummarised)
/// beside a bind-mounted project folder.
///
/// ```text
/// cargo test -- --ignored --nocapture large_workspace
/// ```
#[cfg(unix)]
#[tokio::test]
#[ignore = "needs a Docker daemon; creates and removes a throwaway container"]
async fn a_large_workspace_is_probed_without_overflowing_and_still_copied() {
fn docker_cli(args: &[&str]) -> String {
let out = std::process::Command::new("docker")
.args(args)
.output()
.expect("docker CLI");
assert!(
out.status.success(),
"docker {:?} failed: {}",
args,
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8_lossy(&out.stdout).trim().to_string()
}
let image = std::env::var("TRIPLE_C_TEST_IMAGE")
.unwrap_or_else(|_| "ghcr.io/shadowdao/triple-c-sandbox:latest".to_string());
let host = tempfile::tempdir().unwrap();
for i in 0..50 {
std::fs::write(host.path().join(format!("host-{}.txt", i)), b"h").unwrap();
}
let mount = format!("{}:/workspace/proj", host.path().display());
let id = docker_cli(&[
"run", "-d", "--label", "triple-c.managed=true",
"-v", &mount,
"--entrypoint", "/bin/sh",
&image, "-c", "sleep 300",
]);
// 80k files with long names, three levels down: ~9.2 MB as a flat walk.
let populate = "for w in 0 1 2 3 4 5 6 7; do for p in $(seq 1 100); do \
d=/workspace/worktree-number-$w/node_modules/package-with-a-long-name-$p/lib; \
mkdir -p $d; cd $d && seq -f 'module-file-with-a-long-name-%g.js' 1 100 | xargs touch; \
done; done; echo x > /workspace/worktree-number-0/README.md; \
head -c 4096 /dev/zero > /workspace/worktree-number-0/node_modules/real.js";
docker_cli(&["exec", "-u", "root", &id, "sh", "-c", populate]);
let result = manifest_from_container(&id).await;
let _ = std::process::Command::new("docker").args(["rm", "-f", &id]).output();
let m = result.expect("the probe must not overflow");
let ws: Vec<_> = m.paths.iter().filter(|e| is_under(&e.path, "/workspace")).collect();
assert!(ws.len() < 100, "/workspace was walked file by file: {} entries", ws.len());
assert!(
!ws.iter().any(|e| is_under(&e.path, "/workspace/proj") && e.path != "/workspace/proj"),
"the walk crossed into the bind mount"
);
let nm = m
.paths
.iter()
.find(|e| e.path == "/workspace/worktree-number-0/node_modules")
.expect("summary entry");
// Every file below is empty but `real.js`: the per-file sum, exactly,
// with no directory inode sizes in it — computed by the image's `mawk`.
assert!(nm.is_summary() && nm.size == 4096, "{:?}", nm);
assert_eq!(m.summarised_roots_complete, SUMMARISED_ROOTS.len());
let excl = bind_mount_exclusions(&[pp("proj")]);
let verbatim = compute_verbatim_paths(&m, &Manifest::default(), &excl);
let ws_verbatim: Vec<_> =
verbatim.iter().filter(|p| is_under(p, "/workspace")).cloned().collect();
assert_eq!(
ws_verbatim,
(0..8).map(|w| format!("/workspace/worktree-number-{}", w)).collect::<Vec<_>>()
);
}
#[test]
fn an_overflowing_probe_says_what_to_do_rather_than_quoting_a_buffer() {
let raw = format!(
"{}: Command output exceeded 33554432 bytes and was abandoned",
crate::docker::exec::OUTPUT_LIMIT_MARKER
);
let msg = describe_manifest_failure(raw);
assert!(!msg.contains(crate::docker::exec::OUTPUT_LIMIT_MARKER), "{}", msg);
assert!(!msg.contains("33554432"), "{}", msg);
// Every root still walked file by file, so the advice is never wrong
// about where to look.
for root in MANIFEST_ROOTS.iter().chain(["/etc"].iter()) {
if !SUMMARISED_ROOTS.contains(root) {
assert!(msg.contains(root), "message omits {}: {}", root, msg);
}
}
// Anything else passes through untouched.
assert_eq!(describe_manifest_failure("boom".into()), "boom");
}
// ── Stale rollback pins (A5) ─────────────────────────────────────────────
@@ -89,6 +89,12 @@ pub struct GatewayStatus {
pub running: bool,
pub port: u16,
pub image_exists: bool,
/// The gateway container exists but runs something other than the pinned
/// image (e.g. after an app update moved the pin). If a pinned image is
/// present (`image_exists`), Stop and Start recreates it on that image;
/// otherwise the container keeps its current image until one is pulled or
/// built.
pub image_outdated: bool,
/// Number of fully-specified models in the current settings.
pub model_count: usize,
/// Whether a provider API key is present in the keychain.
@@ -39,6 +39,7 @@ const status = (over: Partial<GatewayStatus> = {}): GatewayStatus => ({
running: true,
port: 4000,
image_exists: true,
image_outdated: false,
model_count: 0,
has_api_key: false,
base_url: "http://host.docker.internal:4000",
@@ -102,4 +103,35 @@ describe("GatewaySettings", () => {
);
await waitFor(() => expect(getGatewayStatus).toHaveBeenCalledTimes(2));
});
it("can stop a gateway running an outdated image, and says what to do", async () => {
appSettings = settingsWithGateway(true);
getGatewayStatus.mockResolvedValue(status({ image_outdated: true, image_exists: false }));
render(<GatewaySettings />);
const stop = await screen.findByRole("button", { name: "Stop" });
expect(screen.getByText(/image update available, Pull or Build/i)).toBeInTheDocument();
fireEvent.click(stop);
await waitFor(() => expect(stopGateway).toHaveBeenCalledTimes(1));
});
it("says Stop and Start applies a new image once it is present", async () => {
appSettings = settingsWithGateway(true);
getGatewayStatus.mockResolvedValue(status({ image_outdated: true }));
render(<GatewaySettings />);
expect(await screen.findByText(/image ready, Stop and Start/i)).toBeInTheDocument();
});
it("can restart a stopped gateway on its old image when no new image is present", async () => {
appSettings = settingsWithGateway(true);
getGatewayStatus.mockResolvedValue(
status({ running: false, image_exists: false, image_outdated: true }),
);
render(<GatewaySettings />);
expect(await screen.findByRole("button", { name: "Start" })).toBeInTheDocument();
// Starting it keeps the old LiteLLM, so say so.
expect(screen.getByText(/Stopped — gateway image update available/)).toBeInTheDocument();
});
});
+24 -17
View File
@@ -178,24 +178,29 @@ export default function GatewaySettings() {
setTimeout(() => setCopied(null), 2000);
};
const tone: StatusTone = !status?.image_exists
? "off"
: status.running
? healthy === false
? "busy"
: "running"
: status.container_exists
? "stopped"
: "off";
// A running container is shown (and stoppable) whatever its image.
const tone: StatusTone = status?.running
? healthy === false
? "busy"
: "running"
: status?.container_exists
? "stopped"
: "off";
const statusLabel = !status?.image_exists
? "No image"
: status.running
? healthy === false
? "Starting…"
const statusLabel = status?.running
? healthy === false
? "Starting…"
: status.image_outdated
? status.image_exists
? `Running on port ${status.port} — new gateway image ready, Stop and Start to apply`
: `Running on port ${status.port} — gateway image update available, Pull or Build to get it`
: `Running on port ${status.port}`
: status.container_exists
? "Stopped"
: status?.container_exists
? status.image_outdated && !status.image_exists
? "Stopped — gateway image update available, Pull or Build to get it"
: "Stopped"
: !status?.image_exists
? "No image"
: "Image ready";
// Rendered in whichever branch is live — only one of them ever mounts.
@@ -254,7 +259,9 @@ export default function GatewaySettings() {
{/* ── Container ─────────────────────────────────────────────── */}
<div className="flex items-center gap-3 flex-wrap">
<StatusIndicator tone={tone} label={statusLabel} className="text-xs" />
{status?.image_exists && (
{/* A stopped container can restart on the image it has, so Start
needs either that container or a pinned image. */}
{(status?.image_exists || status?.container_exists) && (
<Button
variant={status.running ? "danger" : "primary"}
disabled={loading}
+2
View File
@@ -486,6 +486,8 @@ export interface GatewayStatus {
running: boolean;
port: number;
image_exists: boolean;
/** The container runs something other than the pinned image. With `image_exists`, Stop + Start recreates it on the pinned image; without, it keeps its image until one is pulled or built. */
image_outdated: boolean;
model_count: number;
/** Presence only — the provider API key never leaves the keychain. */
has_api_key: boolean;