fix(aws): a waiting clone retry no longer holds the retry lock, so a later connect cannot strand it
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9e1e72d2a4
commit
ff5ae9862b
2 files changed
+60
-1
No files matched your search
@@ -2440,6 +2440,55 @@ fn apply_connect_installs_the_ca_merges_ssh_writes_git_credentials_and_retries_o
|
||||
assert_eq!(fs::read_to_string(c.p("update-calls")).unwrap(), "ran\n", "the CA stamp skips a reinstall");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_connect_that_takes_over_from_a_waiting_retry_does_not_lose_done() {
|
||||
// Connect 1 arrives while a clone that started without credentials is
|
||||
// still running: its retry waits for `done`. The clone finishes and
|
||||
// connect 2 arrives before the waiter's next look: connect 2 removes
|
||||
// `done` and retries at once. The waiter used to hold the retry lock
|
||||
// while it waited, so connect 2's retry queued behind it and the waiter
|
||||
// never saw `done` again (stalled for the whole retry_wait).
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let c = Connect::new(tmp.path());
|
||||
// A long poll, so connect 2 lands in the gap deterministically.
|
||||
let script = fs::read_to_string(&c.script).unwrap().replace(" sleep 2\n", " sleep 6\n");
|
||||
fs::write(&c.script, script).unwrap();
|
||||
let remote = bare_remote(tmp.path(), "remote");
|
||||
fs::write(c.p("run/repos.json"), format!(r#"[{{"url":"{}","branch":null,"folder":"bad"}}]"#, remote.display())).unwrap();
|
||||
fs::write(c.p("clone/repos"), "bad\tcloning\nbad\tfailed\n").unwrap();
|
||||
fs::write(c.p("clone/no-credentials"), "").unwrap();
|
||||
let env = format!("GIT_TOKEN={}\n", b64("ghp_fake-fixture"));
|
||||
|
||||
let out = c.connect(&[], &env);
|
||||
assert!(out.status.success(), "{}", text(&out));
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
// The first clone ends; connect 2 comes before the waiter looks again.
|
||||
fs::write(c.p("clone/done"), "").unwrap();
|
||||
let started = std::time::Instant::now();
|
||||
let out = c.connect(&[], &env);
|
||||
assert!(out.status.success(), "{}", text(&out));
|
||||
|
||||
// Connect 2's retry runs straight away and writes `done`.
|
||||
while !c.p("clone/done").exists() {
|
||||
assert!(
|
||||
started.elapsed() < std::time::Duration::from_secs(5),
|
||||
"`done` was lost: {}",
|
||||
fs::read_to_string(c.p("log")).unwrap()
|
||||
);
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
// The waiter wakes, sees another retry took over, and clones nothing.
|
||||
std::thread::sleep(std::time::Duration::from_secs(7));
|
||||
assert!(c.p("clone/done").exists());
|
||||
assert_eq!(
|
||||
fs::read_to_string(c.p("clone/repos")).unwrap(),
|
||||
"bad\tcloning\nbad\tfailed\nbad\tcloning\nbad\tcloned\n",
|
||||
"{}",
|
||||
fs::read_to_string(c.p("log")).unwrap()
|
||||
);
|
||||
assert!(!c.p("clone/no-credentials").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_ca_move_keeps_the_ca_installed_before() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -155,18 +155,28 @@ start_retry() {
|
||||
# the clone), and a later connect must not wait for it.
|
||||
exec 9>&-
|
||||
trap '' HUP
|
||||
exec 8>"$shm/clone-retry.lock" && flock 8
|
||||
if [ "$mode" = after-done ]; then
|
||||
# Waits *without* the retry lock: a later connect that sees
|
||||
# `done` first removes it and queues a `now` retry behind that
|
||||
# lock, so a waiter holding it would never see `done` again.
|
||||
# Any retry removes `no-credentials`; once it is gone someone
|
||||
# else is cloning with credentials and this waiter has no job.
|
||||
waited=0
|
||||
while [ ! -e "$clone_status/done" ]; do
|
||||
[ -e "$clone_status/no-credentials" ] || exit 0
|
||||
[ "$waited" -lt "$retry_wait" ] || exit 0
|
||||
sleep 2
|
||||
waited=$((waited + 2))
|
||||
done
|
||||
exec 8>"$shm/clone-retry.lock" && flock 8
|
||||
# Checked again under the lock: a `now` retry may have taken
|
||||
# over (removed `done`) or already run (removed `no-credentials`).
|
||||
[ -e "$clone_status/done" ] && [ -e "$clone_status/no-credentials" ] || exit 0
|
||||
mapfile -t folders < <(failed_folders)
|
||||
[ "${#folders[@]}" -gt 0 ] || exit 0
|
||||
rm -f "$clone_status/done"
|
||||
else
|
||||
exec 8>"$shm/clone-retry.lock" && flock 8
|
||||
mapfile -t folders < <(failed_folders)
|
||||
fi
|
||||
# From here the clone is running again; `done` however it ends.
|
||||
|
||||
Reference in new issue
Block a user