Tests: improve failover resliency under load

Since this is a race/thread pool thing, we should wait and see if the messges come through rather than assuming any specific time period. This gives more lead time (still capped) while completing the test as fast as possible.

StackExchange.Redis.Tests/Failover.cs

@@ -239,7 +239,7 @@ public async Task SubscriptionsSurviveMasterSwitchAsync()
                 Log("SubA ping: " + subA.Ping());
                 Log("SubB ping: " + subB.Ping());
                 // If redis is under load due to this suite, it may take a moment to send across.
-                await Task.Delay(250).ForAwait();
+                await UntilCondition(5000, () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
 
                 Assert.Equal(2, Interlocked.Read(ref aCount));
                 Assert.Equal(2, Interlocked.Read(ref bCount));
@@ -256,7 +256,7 @@ public async Task SubscriptionsSurviveMasterSwitchAsync()
                         a.GetServer(TestConfig.Current.FailoverSlaveServerAndPort).MakeMaster(ReplicationChangeOptions.All, sw);
                         Log(sw.ToString());
                     }
-                    await Task.Delay(5000).ForAwait();
+                    await UntilCondition(3000, () => b.GetServer(TestConfig.Current.FailoverMasterServerAndPort).IsSlave).ForAwait();
                     subA.Ping();
                     subB.Ping();
                     Log("Pausing...");
@@ -275,7 +275,7 @@ public async Task SubscriptionsSurviveMasterSwitchAsync()
                     Log("B outstanding: " + b.GetCounters().TotalOutstanding);
                     subA.Ping();
                     subB.Ping();
-                    await Task.Delay(2000).ForAwait();
+                    await Task.Delay(1000).ForAwait();
                     epA = subA.SubscribedEndpoint(channel);
                     epB = subB.SubscribedEndpoint(channel);
                     Log("A: " + EndPointCollection.ToString(epA));
@@ -285,6 +285,7 @@ public async Task SubscriptionsSurviveMasterSwitchAsync()
                     subA.Ping();
                     subB.Ping();
                     Log("Checking...");
+                    await UntilCondition(5000, () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait();
 
                     Assert.Equal(2, Interlocked.Read(ref aCount));
                     Assert.Equal(2, Interlocked.Read(ref bCount));

StackExchange.Redis.Tests/TestBase.cs

@@ -325,5 +325,15 @@ protected static TimeSpan RunConcurrent(Action work, int threads, int timeout =
 
             return watch.Elapsed;
         }
+
+        protected async Task UntilCondition(int maxMilliseconds, Func<bool> predicate, int perLoop = 100)
+        {
+            var spent = 0;
+            while (spent < maxMilliseconds && !predicate())
+            {
+                await Task.Delay(perLoop).ForAwait();
+                spent += perLoop;
+            }
+        }
     }
 }