fix(aws): a waiting clone retry no longer holds the retry lock, so a later connect cannot strand it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
shadowdaoandClaude Opus 5.5 committed 2026-11-16 12:07:14 -08:00
1 parent 9e1e72d2a4
commit ff5ae9862b
2 files changed
+60 -1

No files matched your search

@@ -2440,6 +2440,55 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
assert_eq!(fs::read_to_string(c.p("update-calls")).unwrap(), "ran\n", "the CA stamp skips a reinstall");
}
#[test]
fn a_connect_that_takes_over_from_a_waiting_retry_does_not_lose_done() {
// Connect 1 arrives while a clone that started without credentials is
// still running: its retry waits for `done`. The clone finishes and
// connect 2 arrives before the waiter's next look: connect 2 removes
// `done` and retries at once. The waiter used to hold the retry lock
// while it waited, so connect 2's retry queued behind it and the waiter
// never saw `done` again (stalled for the whole retry_wait).
let tmp = tempfile::tempdir().unwrap();
let c = Connect::new(tmp.path());
// A long poll, so connect 2 lands in the gap deterministically.
let script = fs::read_to_string(&c.script).unwrap().replace(" sleep 2\n", " sleep 6\n");
fs::write(&c.script, script).unwrap();
let remote = bare_remote(tmp.path(), "remote");
fs::write(c.p("run/repos.json"), format!(r#"[{{"url":"{}","branch":null,"folder":"bad"}}]"#, remote.display())).unwrap();
fs::write(c.p("clone/repos"), "bad\tcloning\nbad\tfailed\n").unwrap();
fs::write(c.p("clone/no-credentials"), "").unwrap();
let env = format!("GIT_TOKEN={}\n", b64("ghp_fake-fixture"));
let out = c.connect(&[], &env);
assert!(out.status.success(), "{}", text(&out));
std::thread::sleep(std::time::Duration::from_millis(500));
// The first clone ends; connect 2 comes before the waiter looks again.
fs::write(c.p("clone/done"), "").unwrap();
let started = std::time::Instant::now();
let out = c.connect(&[], &env);
assert!(out.status.success(), "{}", text(&out));
// Connect 2's retry runs straight away and writes `done`.
while !c.p("clone/done").exists() {
assert!(
started.elapsed() < std::time::Duration::from_secs(5),
"`done` was lost: {}",
fs::read_to_string(c.p("log")).unwrap()
);
std::thread::sleep(std::time::Duration::from_millis(100));
}
// The waiter wakes, sees another retry took over, and clones nothing.
std::thread::sleep(std::time::Duration::from_secs(7));
assert!(c.p("clone/done").exists());
assert_eq!(
fs::read_to_string(c.p("clone/repos")).unwrap(),
"bad\tcloning\nbad\tfailed\nbad\tcloning\nbad\tcloned\n",
"{}",
fs::read_to_string(c.p("log")).unwrap()
);
assert!(!c.p("clone/no-credentials").exists());
}
#[test]
fn a_failed_ca_move_keeps_the_ca_installed_before() {
let tmp = tempfile::tempdir().unwrap();
+11 -1
View File
@@ -155,18 +155,28 @@ start_retry() {
# the clone), and a later connect must not wait for it.
exec 9>&-
trap '' HUP
exec 8>"$shm/clone-retry.lock" && flock 8
if [ "$mode" = after-done ]; then
# Waits *without* the retry lock: a later connect that sees
# `done` first removes it and queues a `now` retry behind that
# lock, so a waiter holding it would never see `done` again.
# Any retry removes `no-credentials`; once it is gone someone
# else is cloning with credentials and this waiter has no job.
waited=0
while [ ! -e "$clone_status/done" ]; do
[ -e "$clone_status/no-credentials" ] || exit 0
[ "$waited" -lt "$retry_wait" ] || exit 0
sleep 2
waited=$((waited + 2))
done
exec 8>"$shm/clone-retry.lock" && flock 8
# Checked again under the lock: a `now` retry may have taken
# over (removed `done`) or already run (removed `no-credentials`).
[ -e "$clone_status/done" ] && [ -e "$clone_status/no-credentials" ] || exit 0
mapfile -t folders < <(failed_folders)
[ "${#folders[@]}" -gt 0 ] || exit 0
rm -f "$clone_status/done"
else
exec 8>"$shm/clone-retry.lock" && flock 8
mapfile -t folders < <(failed_folders)
fi
# From here the clone is running again; `done` however it ends.