Move existing gateways onto the pinned LiteLLM after an app update

An app update that bumps the LiteLLM pin left existing gateways on the old
version: the container fingerprint covered only config and secrets, so the
old container kept matching, and a local `triple-c-gateway:latest` from an
earlier app version always won over the new pinned image.

- Resolve the gateway image once, and put its image id in the container
  fingerprint so a newly pulled or rebuilt image recreates the container.
- Label local builds with the pinned LiteLLM version and ignore a local
  build whose label doesn't match (older or unlabelled builds), falling back
  to the pinned upstream image; status reports no usable image so the UI
  offers a pull/build.
- Test that the gateway Dockerfile's FROM stays in lockstep with the pin,
  since the build label relies on it.

Every gateway is recreated once on its next start after this ships.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/src-tauri/src/docker/gateway.rs

@@ -66,6 +66,89 @@ const GATEWAY_INTERNAL_PORT: u16 = 4000;
 
 const CONFIG_FINGERPRINT_LABEL: &str = "triple-c.gateway.config-fingerprint";
 
+/// Stamped on local builds so a build of an *older* pin can be told apart from
+/// a current one. `triple-c-gateway:latest` is a floating tag: without this, a
+/// build made by a previous app version would keep winning over the new pin
+/// forever, because the local build takes precedence.
+const LITELLM_VERSION_LABEL: &str = "triple-c.gateway.litellm-version";
+
+/// The LiteLLM release this app is pinned to — the tag of
+/// [`GATEWAY_REGISTRY_IMAGE`], e.g. `v1.106.6`.
+fn pinned_litellm_version() -> &'static str {
+    GATEWAY_REGISTRY_IMAGE
+        .rsplit_once(':')
+        .map(|(_, tag)| tag)
+        .unwrap_or_default()
+}
+
+/// What we need to know about one locally present gateway image.
+#[derive(Debug, Clone, PartialEq)]
+struct ImageFacts {
+    id: String,
+    litellm_version: Option<String>,
+}
+
+/// The image the gateway container should run, and its content id. The id goes
+/// into the container fingerprint so that a newly pulled or rebuilt image
+/// recreates the container instead of leaving the old one running.
+#[derive(Debug, Clone, PartialEq)]
+struct ChosenImage {
+    reference: &'static str,
+    id: String,
+}
+
+/// Local build first (same precedence as the STT container), but only a build
+/// of the currently pinned LiteLLM — an older or unlabelled build is ignored so
+/// the pinned upstream image is used, or the user is asked to pull/build.
+fn choose_gateway_image(
+    local: Option<ImageFacts>,
+    registry: Option<ImageFacts>,
+) -> Option<ChosenImage> {
+    if let Some(local) = local {
+        if local.litellm_version.as_deref() == Some(pinned_litellm_version()) {
+            return Some(ChosenImage {
+                reference: GATEWAY_LOCAL_IMAGE,
+                id: local.id,
+            });
+        }
+        log::info!(
+            "Ignoring local gateway build {} (LiteLLM {}, app pins {})",
+            GATEWAY_LOCAL_IMAGE,
+            local.litellm_version.as_deref().unwrap_or("unknown"),
+            pinned_litellm_version()
+        );
+    }
+    registry.map(|r| ChosenImage {
+        reference: GATEWAY_REGISTRY_IMAGE,
+        id: r.id,
+    })
+}
+
+/// `Ok(None)` when the image is not present locally; other Docker errors are
+/// returned so "Docker is unreachable" is not mistaken for "image missing".
+async fn inspect_gateway_image(reference: &str) -> Result<Option<ImageFacts>, String> {
+    let docker = get_docker()?;
+    match docker.inspect_image(reference).await {
+        Ok(img) => Ok(Some(ImageFacts {
+            id: img.id.unwrap_or_default(),
+            litellm_version: img
+                .config
+                .and_then(|c| c.labels)
+                .and_then(|l| l.get(LITELLM_VERSION_LABEL).cloned()),
+        })),
+        Err(bollard::errors::Error::DockerResponseServerError {
+            status_code: 404, ..
+        }) => Ok(None),
+        Err(e) => Err(format!("Failed to inspect image {}: {}", reference, e)),
+    }
+}
+
+async fn resolve_gateway_image() -> Result<Option<ChosenImage>, String> {
+    let local = inspect_gateway_image(GATEWAY_LOCAL_IMAGE).await?;
+    let registry = inspect_gateway_image(GATEWAY_REGISTRY_IMAGE).await?;
+    Ok(choose_gateway_image(local, registry))
+}
+
 /// The default bridge gateway address on a stock native-Linux engine. Only a
 /// fallback: the real value is read from the `bridge` network's IPAM config.
 const DEFAULT_BRIDGE_GATEWAY: &str = "172.17.0.1";
@@ -201,6 +284,10 @@ async fn bridge_gateway_ip(docker: &Docker) -> Option<String> {
         .find_map(|c| c.gateway.filter(|g| !g.trim().is_empty()))
 }
 
+fn gateway_fingerprint(config_shape: &str, secret_version: &str, image_id: &str) -> String {
+    sha256_hex(&format!("{}|{}|{}", config_shape, secret_version, image_id))
+}
+
 fn sha256_hex(input: &str) -> String {
     let mut hasher = Sha256::new();
     hasher.update(input.as_bytes());
@@ -208,12 +295,15 @@ fn sha256_hex(input: &str) -> String {
 }
 
 pub async fn get_gateway_status(settings: &GatewaySettings) -> Result<GatewayStatus, String> {
-    let image_exists = super::image::image_exists(GATEWAY_REGISTRY_IMAGE)
-        .await
-        .unwrap_or(false)
-        || super::image::image_exists(GATEWAY_LOCAL_IMAGE)
-            .await
-            .unwrap_or(false);
+    // Only a *usable* image counts: an outdated local build reports false so
+    // the UI offers a pull/build of the pinned version.
+    let image_exists = match resolve_gateway_image().await {
+        Ok(chosen) => chosen.is_some(),
+        Err(e) => {
+            log::warn!("Gateway image check failed: {}", e);
+            false
+        }
+    };
 
     let (container_exists, running) = match find_gateway_container().await? {
         Some((_, state, _)) => (true, state == "running"),
@@ -429,28 +519,11 @@ async fn upload_config(container_id: &str, config: &str) -> Result<(), String> {
 async fn create_gateway_container(
     settings: &GatewaySettings,
     binding: &GatewayBinding,
+    image: &ChosenImage,
     fingerprint: &str,
 ) -> Result<String, String> {
     let docker = get_docker()?;
 
-    // Local build first, then the pinned upstream image — same precedence as
-    // the STT container.
-    let image = if super::image::image_exists(GATEWAY_LOCAL_IMAGE)
-        .await
-        .unwrap_or(false)
-    {
-        GATEWAY_LOCAL_IMAGE.to_string()
-    } else if super::image::image_exists(GATEWAY_REGISTRY_IMAGE)
-        .await
-        .unwrap_or(false)
-    {
-        GATEWAY_REGISTRY_IMAGE.to_string()
-    } else {
-        return Err(
-            "Gateway image not found. Please pull or build the image first.".to_string(),
-        );
-    };
-
     let mut port_bindings = HashMap::new();
     port_bindings.insert(
         format!("{}/tcp", GATEWAY_INTERNAL_PORT),
@@ -491,7 +564,7 @@ async fn create_gateway_container(
     );
 
     let config = Config {
-        image: Some(image),
+        image: Some(image.reference.to_string()),
         // The upstream entrypoint (`docker/prod_entrypoint.sh`) execs
         // `litellm "$@"`. Passed explicitly so the pulled upstream image and
         // our locally built one behave identically.
@@ -565,11 +638,21 @@ async fn ensure_gateway_running_locked(
 
     // Rotation id, not a hash of either secret — see `storage::secure`.
     let secret_version = secure::get_gateway_secret_version()?.unwrap_or_default();
-    let fingerprint = sha256_hex(&format!(
-        "{}|{}",
-        config_shape(settings, &binding),
-        secret_version
-    ));
+    let image = resolve_gateway_image().await?.ok_or_else(|| {
+        format!(
+            "Gateway image for LiteLLM {} not found. Please pull or build the image first.",
+            pinned_litellm_version()
+        )
+    })?;
+
+    // The image id is part of the fingerprint: after an app update moves the
+    // pin (and the new image is pulled or built), the old container no longer
+    // matches and is recreated on the new image.
+    let fingerprint = gateway_fingerprint(
+        &config_shape(settings, &binding),
+        &secret_version,
+        &image.id,
+    );
 
     if let Some((id, state, existing_fingerprint)) = find_gateway_container().await? {
         if existing_fingerprint == fingerprint {
@@ -602,7 +685,7 @@ async fn ensure_gateway_running_locked(
             .map_err(|e| format!("Failed to remove gateway container: {}", e))?;
     }
 
-    let id = create_gateway_container(settings, &binding, &fingerprint).await?;
+    let id = create_gateway_container(settings, &binding, &image, &fingerprint).await?;
 
     // Upload before the first start: LiteLLM reads the config once at boot.
     let rendered = render_config(settings, &api_key, &master_key);
@@ -704,6 +787,9 @@ where
 
     let options = BuildImageOptions {
         t: GATEWAY_LOCAL_IMAGE,
+        // The Dockerfile's FROM is held to the same pin by a test, so the
+        // label is truthful. See `LITELLM_VERSION_LABEL`.
+        labels: HashMap::from([(LITELLM_VERSION_LABEL, pinned_litellm_version())]),
         rm: true,
         forcerm: true,
         ..Default::default()
@@ -785,6 +871,89 @@ mod tests {
         }
     }
 
+    fn facts(id: &str, version: Option<&str>) -> ImageFacts {
+        ImageFacts {
+            id: id.to_string(),
+            litellm_version: version.map(str::to_string),
+        }
+    }
+
+    #[test]
+    fn pinned_version_is_the_registry_tag() {
+        let v = pinned_litellm_version();
+        assert!(
+            v.starts_with('v') && !v.contains(['/', '@', ':']),
+            "bad pin tag {v:?}"
+        );
+    }
+
+    #[test]
+    fn dockerfile_from_matches_the_registry_pin() {
+        // The local-build label claims the pinned version; that is only true
+        // while the Dockerfile builds FROM the same tag.
+        let from = GATEWAY_DOCKERFILE
+            .lines()
+            .find(|l| l.starts_with("FROM "))
+            .expect("Dockerfile has a FROM line");
+        assert!(
+            from.starts_with(&format!("FROM {}@sha256:", GATEWAY_REGISTRY_IMAGE)),
+            "gateway Dockerfile FROM ({}) is out of lockstep with {}",
+            from,
+            GATEWAY_REGISTRY_IMAGE
+        );
+    }
+
+    #[test]
+    fn current_local_build_wins() {
+        let chosen = choose_gateway_image(
+            Some(facts("sha256:local", Some(pinned_litellm_version()))),
+            Some(facts("sha256:registry", None)),
+        );
+        assert_eq!(
+            chosen,
+            Some(ChosenImage {
+                reference: GATEWAY_LOCAL_IMAGE,
+                id: "sha256:local".into()
+            })
+        );
+    }
+
+    #[test]
+    fn outdated_or_unlabelled_local_build_falls_back_to_registry() {
+        for version in [Some("v1.82.6"), None] {
+            let chosen = choose_gateway_image(
+                Some(facts("sha256:local", version)),
+                Some(facts("sha256:registry", None)),
+            );
+            assert_eq!(
+                chosen,
+                Some(ChosenImage {
+                    reference: GATEWAY_REGISTRY_IMAGE,
+                    id: "sha256:registry".into()
+                }),
+                "local build labelled {:?}",
+                version
+            );
+        }
+    }
+
+    #[test]
+    fn outdated_local_build_alone_is_not_usable() {
+        assert_eq!(
+            choose_gateway_image(Some(facts("sha256:old", Some("v1.82.6"))), None),
+            None
+        );
+        assert_eq!(choose_gateway_image(None, None), None);
+    }
+
+    #[test]
+    fn new_image_changes_the_fingerprint() {
+        let a = gateway_fingerprint("shape", "1", "sha256:old");
+        let b = gateway_fingerprint("shape", "1", "sha256:new");
+        assert_ne!(a, b);
+        assert_eq!(a, gateway_fingerprint("shape", "1", "sha256:old"));
+    }
+
     #[test]
     fn valid_models_skips_incomplete_rows() {
         assert_eq!(settings().valid_models().len(), 1);