chore(test) fix flaky tests (#6315)

* internal/cmd/commands/server/listener_reload_test.go remove t.Parallel() comment

* internal/cmd/commands/server/worker_initial_upstreams_reload_test.go remove t.Parallel

* internal/cmd/commands/server/worker_tags_reload_test.go wait by polling worker instead of sleep

* internal/daemon/worker/routing_info.go make deep copy in LastRoutingInfoSuccess to avoid race condition

* internal/tests/cluster/parallel/orphaned_connections_test.go raise grace period time to allow test to wait for longer

* improve logging in sequential tests

* move ports around and use t.Cleanup to send shutdown signal to test server

* fix one typo

* do final assert in t.Cleanup instead of defer

* undo t.Cleanup change

* add another signal to force shutdown running servers

* add exit early and increase server start timeout

* tighten workerRPC settings on session tests

* tighten more workerRPC timing

* make gen

* add comment explaining deep copying lastRoutingInfoSuccess

internal/tests/cluster/parallel/multi_controller_worker_test.go

@@ -53,7 +53,7 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) {
 		WorkerAuthKms:     c1.Config().WorkerAuthKms,
 		InitialUpstreams:  append(c1.ClusterAddrs(), c2.ClusterAddrs()...),
 		Logger:            logger.Named("w1"),
-		WorkerRPCInterval: time.Second,
+		WorkerRPCInterval: 500 * time.Millisecond,
 	})
 
 	wg.Add(2)
@@ -99,7 +99,7 @@ func TestMultiControllerMultiWorkerConnections(t *testing.T) {
 		WorkerAuthKms:     c1.Config().WorkerAuthKms,
 		InitialUpstreams:  c1.ClusterAddrs(),
 		Logger:            logger.Named("w3"),
-		WorkerRPCInterval: time.Second,
+		WorkerRPCInterval: 500 * time.Millisecond,
 	})
 
 	wg.Add(2)
@@ -157,7 +157,7 @@ func TestWorkerAppendInitialUpstreams(t *testing.T) {
 		InitialUpstreams: initialUpstreams,
 		Logger:           logger.Named("w1"),
 		SuccessfulControllerRPCGracePeriodDuration: 1 * time.Second,
-		WorkerRPCInterval:                          time.Second,
+		WorkerRPCInterval:                          500 * time.Millisecond,
 	})
 
 	// Wait for worker to send routing info