fix(e2e): Worker not available error in RDP tests (#6386)

* test(e2e): Add more frequent ssh checks

(cherry picked from commit a35dd28fd142f08d11cdf1967a916405170d9a97)

* test(e2e): Add step to wait for worker to init

(cherry picked from commit 57b649301f3d13e560bfd74f26e39bdeb24f72bd)

enos/enos-scenario-e2e-aws-rdp-base.hcl

@@ -290,6 +290,7 @@ scenario "e2e_aws_rdp_base" {
       step.create_boundary_cluster,
       step.create_rdp_domain_controller,
       step.create_rdp_member_server,
+      step.create_windows_worker,
       step.create_bucket
     ]
 

enos/modules/aws_rdp_domain_controller/main.tf

@@ -482,7 +482,7 @@ data "aws_instance" "instance_password" {
 resource "enos_local_exec" "wait_for_ssh" {
   depends_on = [time_sleep.wait_for_reboot]
   count      = var.server_version != "2016" ? 1 : 0
-  inline     = ["timeout 600s bash -c 'until ssh -i ${abspath(local_sensitive_file.private_key.filename)} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.domain_controller.public_ip} \"echo ready\"; do sleep 10; done'"]
+  inline     = ["timeout 600s bash -c 'until ssh -i ${abspath(local_sensitive_file.private_key.filename)} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 Administrator@${aws_instance.domain_controller.public_ip} \"echo ready\"; do sleep 10; done'"]
 }
 
 locals {

enos/modules/aws_rdp_member_server/main.tf

@@ -287,7 +287,7 @@ data "aws_instance" "instance_password" {
 resource "enos_local_exec" "wait_for_ssh" {
   count      = var.server_version != "2016" ? 1 : 0
   depends_on = [time_sleep.wait_5_minutes]
-  inline     = ["timeout 600s bash -c 'until ssh -i ${local.private_key} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.member_server.public_ip} \"echo ready\"; do sleep 10; done'"]
+  inline     = ["timeout 600s bash -c 'until ssh -i ${local.private_key} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 Administrator@${aws_instance.member_server.public_ip} \"echo ready\"; do sleep 10; done'"]
 }
 
 # Retrieve the domain hostname of the member server, which will be used in

enos/modules/aws_rdp_member_server_with_worker/main.tf

@@ -290,7 +290,7 @@ resource "enos_local_exec" "wait_for_ssh" {
   depends_on = [
     aws_instance.worker,
   ]
-  inline = ["timeout 600s bash -c 'until ssh -i ${local.private_key} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.worker.public_ip} \"echo ready\"; do sleep 10; done'"]
+  inline = ["timeout 600s bash -c 'until ssh -i ${local.private_key} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 Administrator@${aws_instance.worker.public_ip} \"echo ready\"; do sleep 10; done'"]
 }
 
 resource "enos_local_exec" "make_dir" {
@@ -372,20 +372,25 @@ resource "enos_local_exec" "run_powershell_script" {
   inline = ["ssh -i ${local.private_key} -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.worker.public_ip} ${local.test_dir}/${basename(local_file.powershell_script.filename)}"]
 }
 
-resource "time_sleep" "wait_2_minutes" {
+resource "time_sleep" "wait_for_instance_reboot_in_script" {
   depends_on      = [enos_local_exec.run_powershell_script]
   create_duration = "2m"
 }
 
-data "aws_instance" "instance_password" {
-  depends_on        = [time_sleep.wait_2_minutes]
-  instance_id       = aws_instance.worker.id
-  get_password_data = true
-}
-
 # used for debug
 resource "local_file" "powershell_script_output" {
   depends_on = [enos_local_exec.run_powershell_script]
   content    = enos_local_exec.run_powershell_script.stdout
   filename   = "${path.root}/.terraform/tmp/setup_worker.out"
 }
+
+resource "enos_local_exec" "wait_for_worker" {
+  depends_on = [time_sleep.wait_for_instance_reboot_in_script]
+  inline     = ["timeout 600s bash -c 'until ssh -i ${local.private_key} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.worker.public_ip} \"if (-not (Test-Path C:/Test/worker.out)) { exit 1 }\"; do sleep 10; done'"]
+}
+
+data "aws_instance" "instance_password" {
+  depends_on        = [enos_local_exec.wait_for_worker]
+  instance_id       = aws_instance.worker.id
+  get_password_data = true
+}

enos/modules/aws_windows_client/main.tf

@@ -310,7 +310,7 @@ resource "local_sensitive_file" "private_key" {
 # can just SSH using the private key
 resource "enos_local_exec" "wait_for_ssh" {
   depends_on = [aws_instance.client]
-  inline     = ["timeout 600s bash -c 'until ssh -i ${abspath(local_sensitive_file.private_key.filename)} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.client.public_ip} \"echo ready\"; do sleep 10; done'"]
+  inline     = ["timeout 600s bash -c 'until ssh -i ${abspath(local_sensitive_file.private_key.filename)} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=5 Administrator@${aws_instance.client.public_ip} \"echo ready\"; do sleep 10; done'"]
 }
 
 resource "enos_local_exec" "get_go_version" {