Compare commits

...
6 Commits
Author SHA1 Message Date
jknapp 1659788b70 Merge pull request 'AWS host phase 6: EFS, copy-at-connect, setup scripts, lifetime cap, ports, gateway sidecar, cost estimate' (#83) from feat/aws-phase6 into main
Secret Scan / scan (push) Successful in 22s
Build Container / build-container (push) Successful in 4m51s
2026-11-16 20:36:36 +00:00
shadowdaoandClaude Opus 5.5 0a325cb7f2 fix(aws): Host settings merge saves against the latest config and cache the environment's lifetime cap
Build App (Preview) / compute-version (pull_request) Successful in 38s
Secret Scan / scan (push) Successful in 17s
Secret Scan / scan (pull_request) Successful in 11s
Build App (Preview) / create-release (pull_request) Successful in 7s
Build App (Preview) / test (pull_request) Successful in 8m14s
Build App (Preview) / build-macos (pull_request) Successful in 6m4s
Build App (Preview) / build-windows (pull_request) Successful in 10m43s
Build Container / build-container (pull_request) Successful in 19m45s
Build App (Preview) / build-linux (pull_request) Successful in 16m22s
Build App (Preview) / prune-previews (pull_request) Successful in 9s
A save right after the setup script's blur (a toggle) spread the render-time
config and reverted the script; saves now start from the stored config plus
every patch saved since. aws_environment_max_hours is cached per connection,
region and environment for 10 minutes, so re-mounting the section does not
ask AWS again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 12:10:37 -08:00
shadowdaoandClaude Opus 5.5 ea37f8434f fix(aws): retry the copy-at-connect secret read while the secret is pending deletion
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 12:08:46 -08:00
shadowdaoandClaude Opus 5.5 544409c554 fix(aws): no Pricing client counts as a failed fetch, so the 5-minute back-off applies
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 12:08:46 -08:00
shadowdaoandClaude Opus 5.5 be96b5f68f fix(aws): the copy-at-connect warning gets its own toast topic
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 12:08:46 -08:00
shadowdaoandClaude Opus 5.5 ff5ae9862b fix(aws): a waiting clone retry no longer holds the retry lock, so a later connect cannot strand it
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 12:07:14 -08:00
13 changed files with 369 additions and 40 deletions

No files matched your search

+8 -2
View File
@@ -374,8 +374,14 @@ impl RatesCache {
};
let fetched = match tokio::time::timeout(FETCH_TIMEOUT, fetch).await {
Ok(Some(r)) => r,
// No client could be built: the stale cache answers.
Ok(None) => return stale,
// No client could be built: a failure like any other (building
// one reads credentials, which can be slow), and the stale cache
// answers.
Ok(None) => {
log::info!("No Fargate rates for {region}: no Pricing client could be built");
self.note_failure(region, now, true);
return stale;
}
Err(_) => Err(format!("timed out after {}s", FETCH_TIMEOUT.as_secs())),
};
match fetched {
+21
View File
@@ -395,3 +395,24 @@ async fn building_the_client_counts_against_the_same_timeout() {
.await;
assert_eq!(fresh, Some(RATES));
}
#[tokio::test]
async fn no_client_counts_as_a_failure_and_is_not_asked_again_for_a_while() {
let (_d, c) = cache();
let none = || async { None::<FakeApi> };
assert_eq!(c.rates_with::<FakeApi, FakeApi>(none(), "us-west-2", at(T0)).await, None);
// Within the back-off the client is not even built.
let r = c
.rates_with::<FakeApi, FakeApi>(
async { panic!("no client is built within the back-off") },
"us-west-2",
at(T0 + 60),
)
.await;
assert_eq!(r, None);
// After it, a working client is asked and clears the failure.
let api = FakeApi::fixture();
let later = T0 + RETRY_AFTER_FAILURE.as_secs();
assert_eq!(c.rates(Some(&api), "us-west-2", at(later)).await, Some(RATES));
assert_eq!(api.calls(), 1);
}
+25 -18
View File
@@ -1048,24 +1048,31 @@ pub async fn prepare_workspace<W: WorkspaceApi>(
{
Some(arn) => arn,
None => {
let old = api.project_secret_value(&secret_name).await?;
if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) {
log::info!(
"Project {}: its secret still held credentials; replacing it so no version keeps them",
project.id
);
replace_project_secret(
api,
&secret_name,
&env.kms_key_arn,
&json,
&tags,
RECREATE_TIMING,
)
.await?
} else {
put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?
}
// The secret existed at the create but may be marked for
// deletion by now (another start replacing it): wait that out
// like the writes do. Once it is gone the read says `None`
// and the put below makes it again.
let old = retry_while_pending_deletion(&secret_name, RECREATE_TIMING, || {
api.project_secret_value(&secret_name)
})
.await?;
if old.as_deref().is_some_and(|o| holds_credentials(o, &payload, custom)) {
log::info!(
"Project {}: its secret still held credentials; replacing it so no version keeps them",
project.id
);
replace_project_secret(
api,
&secret_name,
&env.kms_key_arn,
&json,
&tags,
RECREATE_TIMING,
)
.await?
} else {
put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?
}
}
},
None => put_secret_retrying(api, &secret_name, &env.kms_key_arn, &json, &tags).await?,
+38 -1
View File
@@ -800,6 +800,8 @@ struct Fake {
secret_reads: Mutex<usize>,
/// Puts that fail as "scheduled for deletion" before one succeeds.
pending_puts: Mutex<usize>,
/// Reads that fail as "marked for deletion" before one succeeds.
pending_reads: Mutex<usize>,
/// Calls `create_project_secret` got.
creates: Mutex<usize>,
}
@@ -847,8 +849,15 @@ impl WorkspaceApi for Fake {
c.push((fs_id.into(), root.into(), tags.to_vec()));
Ok(format!("fsap-new{}", c.len()))
}
async fn project_secret_value(&self, _name: &str) -> Result<Option<String>, AwsError> {
async fn project_secret_value(&self, name: &str) -> Result<Option<String>, AwsError> {
*self.secret_reads.lock().unwrap() += 1;
{
let mut pending = self.pending_reads.lock().unwrap();
if *pending > 0 {
*pending -= 1;
return Err(pending_deletion_error(name));
}
}
Ok(self.existing_secret.lock().unwrap().clone())
}
async fn delete_project_secret(&self, name: &str) -> Result<(), AwsError> {
@@ -1751,6 +1760,34 @@ async fn the_start_after_a_timed_out_replace_retries_the_create() {
assert!(!e.contains("Restore"), "{e}");
}
#[tokio::test(start_paused = true)]
async fn a_copy_at_connect_read_of_a_secret_being_deleted_is_retried() {
// CreateSecret said it exists, then the read meets a deletion AWS is
// still finishing (another start replacing it): retried like the writes.
let env = pairs(&[("TZ", "UTC")]);
let fake = Fake::onboarded();
*fake.existing_secret.lock().unwrap() = Some(
r#"{"TRIPLEC_AGENT_TOKEN":"t","TRIPLEC_CREDENTIALS":"connect","TZ":"UTC","TRIPLEC_REPOS":"[]"}"#.into(),
);
*fake.pending_reads.lock().unwrap() = 3;
prepare_workspace(&fake, &who(), &env_info(), &copy_at_connect_project(), &env, &[], &[], IMAGE, None)
.await
.unwrap();
assert_eq!(*fake.secret_reads.lock().unwrap(), 4);
assert_eq!(fake.secrets.lock().unwrap().len(), 1);
// Still being deleted past the limit: try again in a minute.
let fake = Fake::onboarded();
*fake.existing_secret.lock().unwrap() = Some(r#"{"TZ":"UTC"}"#.into());
*fake.pending_reads.lock().unwrap() = 1000;
let e = prepare_workspace(&fake, &who(), &env_info(), &copy_at_connect_project(), &env, &[], &[], IMAGE, None)
.await
.unwrap_err()
.to_string();
assert!(e.contains("Start the project again in a minute"), "{e}");
assert!(fake.secrets.lock().unwrap().is_empty());
}
#[test]
fn a_copy_at_connect_payload_drops_the_custom_variables_fingerprint() {
let env = pairs(&[("TRIPLE_C_CUSTOM_ENV", "63ce86"), ("TZ", "UTC")]);
+21 -9
View File
@@ -1583,16 +1583,19 @@ fn emit_workspace_warning(app_handle: &tauri::AppHandle, project_id: &str, name:
}
/// Copy-at-connect's warning (Q2): the credentials could not be copied into
/// a connected workspace. Same `workspace-warning` event as the follower's.
/// a connected workspace. Same `workspace-warning` event as the follower's,
/// with its own `topic` so neither replaces the other's toast.
fn emit_connect_warning(app_handle: &tauri::AppHandle, project_id: &str, name: &str, detail: &str) {
let _ = app_handle.emit(
"workspace-warning",
serde_json::json!({
"project_id": project_id,
"message": format!("“{name}”: the credentials could not be copied into the workspace"),
"detail": detail,
}),
);
let _ = app_handle.emit("workspace-warning", connect_warning_payload(project_id, name, detail));
}
fn connect_warning_payload(project_id: &str, name: &str, detail: &str) -> serde_json::Value {
serde_json::json!({
"project_id": project_id,
"message": format!("“{name}”: the credentials could not be copied into the workspace"),
"detail": detail,
"topic": "connect",
})
}
/// Port mappings that could not be forwarded to a connected AWS workspace
@@ -2199,6 +2202,15 @@ pub async fn reconcile_aws_projects(
#[cfg(test)]
mod tests {
#[test]
fn the_connect_warning_has_its_own_toast_topic() {
let v = super::connect_warning_payload("p1", "demo", "upload failed");
assert_eq!(v["topic"], "connect");
assert_eq!(v["project_id"], "p1");
assert_eq!(v["detail"], "upload failed");
assert!(v["message"].as_str().unwrap().contains("demo"));
}
use super::*;
// ── Pending-cleanup aging ────────────────────────────────────────────
@@ -2440,6 +2440,55 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
assert_eq!(fs::read_to_string(c.p("update-calls")).unwrap(), "ran\n", "the CA stamp skips a reinstall");
}
#[test]
fn a_connect_that_takes_over_from_a_waiting_retry_does_not_lose_done() {
// Connect 1 arrives while a clone that started without credentials is
// still running: its retry waits for `done`. The clone finishes and
// connect 2 arrives before the waiter's next look: connect 2 removes
// `done` and retries at once. The waiter used to hold the retry lock
// while it waited, so connect 2's retry queued behind it and the waiter
// never saw `done` again (stalled for the whole retry_wait).
let tmp = tempfile::tempdir().unwrap();
let c = Connect::new(tmp.path());
// A long poll, so connect 2 lands in the gap deterministically.
let script = fs::read_to_string(&c.script).unwrap().replace(" sleep 2\n", " sleep 6\n");
fs::write(&c.script, script).unwrap();
let remote = bare_remote(tmp.path(), "remote");
fs::write(c.p("run/repos.json"), format!(r#"[{{"url":"{}","branch":null,"folder":"bad"}}]"#, remote.display())).unwrap();
fs::write(c.p("clone/repos"), "bad\tcloning\nbad\tfailed\n").unwrap();
fs::write(c.p("clone/no-credentials"), "").unwrap();
let env = format!("GIT_TOKEN={}\n", b64("ghp_fake-fixture"));
let out = c.connect(&[], &env);
assert!(out.status.success(), "{}", text(&out));
std::thread::sleep(std::time::Duration::from_millis(500));
// The first clone ends; connect 2 comes before the waiter looks again.
fs::write(c.p("clone/done"), "").unwrap();
let started = std::time::Instant::now();
let out = c.connect(&[], &env);
assert!(out.status.success(), "{}", text(&out));
// Connect 2's retry runs straight away and writes `done`.
while !c.p("clone/done").exists() {
assert!(
started.elapsed() < std::time::Duration::from_secs(5),
"`done` was lost: {}",
fs::read_to_string(c.p("log")).unwrap()
);
std::thread::sleep(std::time::Duration::from_millis(100));
}
// The waiter wakes, sees another retry took over, and clones nothing.
std::thread::sleep(std::time::Duration::from_secs(7));
assert!(c.p("clone/done").exists());
assert_eq!(
fs::read_to_string(c.p("clone/repos")).unwrap(),
"bad\tcloning\nbad\tfailed\nbad\tcloning\nbad\tcloned\n",
"{}",
fs::read_to_string(c.p("log")).unwrap()
);
assert!(!c.p("clone/no-credentials").exists());
}
#[test]
fn a_failed_ca_move_keeps_the_ca_installed_before() {
let tmp = tempfile::tempdir().unwrap();
@@ -8,6 +8,7 @@ import HostSection from "./HostSection";
import { awsProject, localProject } from "../../../../test/awsFixtures";
import type { AppSettings, AuthBridgeStatus, Project } from "../../../../lib/types";
import { useAppState } from "../../../../store/appState";
import { clearEnvironmentMaxHoursCache } from "../../../../lib/environmentMaxHours";
const OFF_BRIDGE: AuthBridgeStatus = { enabled: false, active_ports: [], conflicts: [] };
const removeAwsResources = vi.fn();
@@ -27,7 +28,10 @@ vi.mock("@tauri-apps/plugin-dialog", () => ({ open: vi.fn(async () => null) }));
const save = vi.fn().mockResolvedValue(true);
beforeEach(() => vi.clearAllMocks());
beforeEach(() => {
vi.clearAllMocks();
clearEnvironmentMaxHoursCache();
});
function toggle(name: string) {
return screen.getByRole("switch", { name });
@@ -376,6 +380,32 @@ describe("HostSection", () => {
expect(save.mock.calls[1][0].aws.setup_script).toBeNull();
});
it("a toggle right after the setup script's blur keeps the script", async () => {
// The store has not caught up with the first save when the second comes.
const p = awsProject();
renderHost(p);
const box = screen.getByLabelText("Setup script");
fireEvent.change(box, { target: { value: "make deps\n" } });
fireEvent.blur(box);
fireEvent.click(toggle("Spot"));
await waitFor(() => expect(save).toHaveBeenCalledTimes(2));
expect(save.mock.calls[0][0].aws.setup_script).toBe("make deps\n");
expect(save.mock.calls[1][0].aws).toMatchObject({ setup_script: "make deps\n", spot: true });
});
it("starts again from the stored config after a refused save", async () => {
save.mockResolvedValueOnce(false);
renderHost();
const box = screen.getByLabelText("Setup script");
fireEvent.change(box, { target: { value: "make deps\n" } });
fireEvent.blur(box);
await waitFor(() => expect(save).toHaveBeenCalledTimes(1));
await act(async () => {});
fireEvent.click(toggle("Spot"));
await waitFor(() => expect(save).toHaveBeenCalledTimes(2));
expect(save.mock.calls[1][0].aws).toMatchObject({ setup_script: null, spot: true });
});
it("refuses a setup script over 32 KiB with the backend's message", () => {
renderHost();
const box = screen.getByLabelText("Setup script");
@@ -413,6 +443,7 @@ describe("HostSection", () => {
cleanup();
for (const none of [0, null]) {
clearEnvironmentMaxHoursCache();
awsEnvironmentMaxHours.mockResolvedValueOnce(none);
renderHost();
await waitFor(() => expect(awsEnvironmentMaxHours).toHaveBeenCalled());
@@ -422,6 +453,23 @@ describe("HostSection", () => {
}
});
it("reads the lifetime cap once per environment across re-mounts", async () => {
awsEnvironmentMaxHours.mockResolvedValue(24);
renderHost();
await screen.findByText(/stops workspaces after 24 h/);
cleanup();
renderHost();
// Shown from the cache, without asking AWS again.
await screen.findByText(/stops workspaces after 24 h/);
expect(awsEnvironmentMaxHours).toHaveBeenCalledTimes(1);
cleanup();
// Another environment is its own read.
renderHost(awsProject({ aws_placement: { region: "eu-west-1", environment: "prod" } }));
await waitFor(() => expect(awsEnvironmentMaxHours).toHaveBeenCalledTimes(2));
awsEnvironmentMaxHours.mockReset();
});
it("does not let the credentials setting change while the workspace runs", () => {
renderHost(awsProject(), true);
expect(screen.getByRole("radio", { name: "Copied when you connect" })).toBeDisabled();
@@ -1,6 +1,7 @@
import { useEffect, useState } from "react";
import { useEffect, useRef, useState } from "react";
import type { AwsWorkspaceConfig, HostConnection, Project, RepoSource } from "../../../../lib/types";
import * as commands from "../../../../lib/tauri-commands";
import { environmentMaxHours } from "../../../../lib/environmentMaxHours";
import { useAppState } from "../../../../store/appState";
import {
COPY_AT_CONNECT_GATEWAY_NOTE,
@@ -80,13 +81,13 @@ export default function HostSection({ project, save, disabled }: Props) {
const environment = project.aws_placement?.environment;
useEffect(() => {
setMaxHours(null);
if (!project.host_id || !environment) return;
if (!project.host_id || !region || !environment) return;
let cancelled = false;
// Best effort: an unreadable record just shows no cap line.
Promise.resolve()
.then(() => commands.awsEnvironmentMaxHours(project.id))
// Best effort: an unreadable record just shows no cap line. Cached per
// environment, so re-opening this section doesn't ask AWS again.
environmentMaxHours(project.id, project.host_id, region, environment)
.then((h) => {
if (!cancelled) setMaxHours(h ?? null);
if (!cancelled) setMaxHours(h);
})
.catch(() => {});
return () => {
@@ -94,7 +95,35 @@ export default function HostSection({ project, save, disabled }: Props) {
};
}, [project.id, project.host_id, region, environment]);
const saveAws = (patch: Partial<AwsWorkspaceConfig>) => save({ aws: { ...aws, ...patch } });
// The config the next save starts from: the stored one plus every patch
// saved since. A blur's save followed at once by a toggle's (before the
// store has the first back) must not spread the render-time `aws` and
// revert the first. Re-based on the prop when it changes with no save in
// flight.
const latest = useRef({ base: project.aws, value: aws, inFlight: 0 });
if (latest.current.base !== project.aws && latest.current.inFlight === 0) {
latest.current = { base: project.aws, value: aws, inFlight: 0 };
}
const saveAws = async (patch: Partial<AwsWorkspaceConfig>) => {
const l = latest.current;
const next = { ...l.value, ...patch };
l.value = next;
l.inFlight += 1;
// A refused save is not the config any more: start again from the store.
const refused = () => {
if (l.value === next) l.value = l.base ?? defaultAwsWorkspace();
};
try {
const ok = await save({ aws: next });
if (!ok) refused();
return ok;
} catch (e) {
refused();
throw e;
} finally {
l.inFlight -= 1;
}
};
const hasResources = !!(
aws.task_arn ||
@@ -55,4 +55,21 @@ describe("useWorkspaceWarnings", () => {
expect(toasts).toHaveLength(2);
expect(toasts[1].detail).toContain("already in use");
});
it("keeps a copy-at-connect warning beside the start's own warning", async () => {
renderHook(() => useWorkspaceWarnings());
await vi.waitFor(() => expect(handler).not.toBeNull());
handler!({ payload: { project_id: "p1", message: "started", detail: "clone failed" } });
handler!({
payload: {
project_id: "p1",
message: "“demo”: the credentials could not be copied into the workspace",
detail: "upload failed",
topic: "connect",
},
});
const { toasts } = useAppState.getState();
expect(toasts).toHaveLength(2);
expect(toasts.map((t) => t.detail)).toEqual(["clone failed", "upload failed"]);
});
});
+1 -1
View File
@@ -10,7 +10,7 @@ export interface WorkspaceWarningEvent {
message: string;
detail: string;
/** Set when the warning is about something else than the start's own
* follow (`"ports"`): keeps it from replacing that toast. */
* follow (`"ports"`, `"connect"`): keeps it from replacing that toast. */
topic?: string;
}
+48
View File
@@ -0,0 +1,48 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
const awsEnvironmentMaxHours = vi.fn(async (_id: string) => 24 as number | null);
vi.mock("./tauri-commands", () => ({
awsEnvironmentMaxHours: (id: string) => awsEnvironmentMaxHours(id),
}));
import {
MAX_HOURS_TTL_MS,
clearEnvironmentMaxHoursCache,
environmentMaxHours,
} from "./environmentMaxHours";
describe("environmentMaxHours", () => {
beforeEach(() => {
awsEnvironmentMaxHours.mockClear();
clearEnvironmentMaxHoursCache();
});
it("asks once per connection, region and environment until the TTL passes", async () => {
expect(await environmentMaxHours("p1", "c1", "eu-west-1", "dev", 0)).toBe(24);
// Another project in the same environment shares the read.
expect(await environmentMaxHours("p2", "c1", "eu-west-1", "dev", 1000)).toBe(24);
expect(awsEnvironmentMaxHours).toHaveBeenCalledTimes(1);
await environmentMaxHours("p1", "c2", "eu-west-1", "dev", 1000);
await environmentMaxHours("p1", "c1", "us-east-1", "dev", 1000);
await environmentMaxHours("p1", "c1", "eu-west-1", "prod", 1000);
expect(awsEnvironmentMaxHours).toHaveBeenCalledTimes(4);
await environmentMaxHours("p1", "c1", "eu-west-1", "dev", MAX_HOURS_TTL_MS);
expect(awsEnvironmentMaxHours).toHaveBeenCalledTimes(5);
});
it("shares a read in flight and keeps an unreadable record (null)", async () => {
awsEnvironmentMaxHours.mockResolvedValueOnce(null);
const a = environmentMaxHours("p1", "c1", "eu-west-1", "dev", 0);
const b = environmentMaxHours("p1", "c1", "eu-west-1", "dev", 0);
expect(await a).toBeNull();
expect(await b).toBeNull();
expect(awsEnvironmentMaxHours).toHaveBeenCalledTimes(1);
});
it("does not keep a rejected read", async () => {
awsEnvironmentMaxHours.mockRejectedValueOnce(new Error("Project p1 not found"));
await expect(environmentMaxHours("p1", "c1", "eu-west-1", "dev", 0)).rejects.toThrow();
expect(await environmentMaxHours("p1", "c1", "eu-west-1", "dev", 1)).toBe(24);
expect(awsEnvironmentMaxHours).toHaveBeenCalledTimes(2);
});
});
+45
View File
@@ -0,0 +1,45 @@
import * as commands from "./tauri-commands";
/** How long an environment's lifetime cap is reused before AWS is asked again. */
export const MAX_HOURS_TTL_MS = 10 * 60 * 1000;
interface Entry {
at: number;
value: Promise<number | null>;
}
/** Keyed by connection, region and environment: the record is the
* environment's, so every project placed there shares one read. */
const cache = new Map<string, Entry>();
/**
* `aws_environment_max_hours`, cached per (connection, region, environment)
* for [`MAX_HOURS_TTL_MS`], so re-mounting Host settings does not read the
* environment record from AWS again. A read in flight is shared; a rejected
* one is not kept (the next mount asks again).
*/
export function environmentMaxHours(
projectId: string,
connectionId: string,
region: string,
environment: string,
now: number = Date.now(),
): Promise<number | null> {
const key = JSON.stringify([connectionId, region, environment]);
const hit = cache.get(key);
if (hit && now - hit.at < MAX_HOURS_TTL_MS) return hit.value;
const value = Promise.resolve()
.then(() => commands.awsEnvironmentMaxHours(projectId))
.then((h) => h ?? null);
const entry = { at: now, value };
cache.set(key, entry);
value.catch(() => {
if (cache.get(key) === entry) cache.delete(key);
});
return value;
}
/** For tests. */
export function clearEnvironmentMaxHoursCache() {
cache.clear();
}
+11 -1
View File
@@ -155,18 +155,28 @@ start_retry() {
# the clone), and a later connect must not wait for it.
exec 9>&-
trap '' HUP
exec 8>"$shm/clone-retry.lock" && flock 8
if [ "$mode" = after-done ]; then
# Waits *without* the retry lock: a later connect that sees
# `done` first removes it and queues a `now` retry behind that
# lock, so a waiter holding it would never see `done` again.
# Any retry removes `no-credentials`; once it is gone someone
# else is cloning with credentials and this waiter has no job.
waited=0
while [ ! -e "$clone_status/done" ]; do
[ -e "$clone_status/no-credentials" ] || exit 0
[ "$waited" -lt "$retry_wait" ] || exit 0
sleep 2
waited=$((waited + 2))
done
exec 8>"$shm/clone-retry.lock" && flock 8
# Checked again under the lock: a `now` retry may have taken
# over (removed `done`) or already run (removed `no-credentials`).
[ -e "$clone_status/done" ] && [ -e "$clone_status/no-credentials" ] || exit 0
mapfile -t folders < <(failed_folders)
[ "${#folders[@]}" -gt 0 ] || exit 0
rm -f "$clone_status/done"
else
exec 8>"$shm/clone-retry.lock" && flock 8
mapfile -t folders < <(failed_folders)
fi
# From here the clone is running again; `done` however it ends.