Automatic retry during connect (configurable)

StackExchange.Redis.Tests/ConnectFailTimeout.cs

@@ -6,6 +6,7 @@ namespace StackExchange.Redis.Tests
     [TestFixture]
     public class ConnectFailTimeout : TestBase
     {
+#if DEBUG
         [TestCase]
         public void NoticesConnectFail()
         {
@@ -34,6 +35,6 @@ public void NoticesConnectFail()
                 System.Console.WriteLine(time);
             }
         }
-
+#endif
     }
 }

StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj

@@ -99,6 +99,7 @@
     <Compile Include="TaskTests.cs" />
     <Compile Include="TestBase.cs" />
     <Compile Include="Transactions.cs" />
+    <Compile Include="VPNTest.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />

StackExchange.Redis.Tests/VPNTest.cs

+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StackExchange.Redis.Tests
+{
+    [TestFixture]
+    public class VPNTest : TestBase
+    {
+
+        [Test]
+        [MaxTime(100000)]
+        [TestCase("or-devredis01.ds.stackexchange.com:6379")]
+        public void Execute(string config)
+        {
+            for (int i = 0; i < 50; i++)
+            {
+                var log = new StringWriter();
+                try
+                {
+                    var options = ConfigurationOptions.Parse(config);
+                    options.SyncTimeout = 3000;
+                    options.ConnectRetry = 5;
+                    using (var conn = ConnectionMultiplexer.Connect(options, log))
+                    {
+                        var ttl = conn.GetDatabase().Ping();
+                        Console.WriteLine(ttl);
+                    }
+                }
+                catch
+                {
+                    Console.WriteLine(log);
+                    Assert.Fail();
+                }
+                Console.WriteLine();
+                Console.WriteLine("===");
+                Console.WriteLine();
+            }
+        }
+    }
+}

StackExchange.Redis/StackExchange/Redis/ConfigurationOptions.cs

@@ -72,7 +72,7 @@ internal static void Unknown(string key)
                         Version = "version", ConnectTimeout = "connectTimeout", Password = "password",
                         TieBreaker = "tiebreaker", WriteBuffer = "writeBuffer", Ssl = "ssl", SslHost = "sslHost",
                         ConfigChannel = "configChannel", AbortOnConnectFail = "abortConnect", ResolveDns = "resolveDns",
-                        ChannelPrefix = "channelPrefix", Proxy = "proxy";
+                        ChannelPrefix = "channelPrefix", Proxy = "proxy", ConnectRetry = "connectRetry";
             private static readonly Dictionary<string, string> normalizedOptions = new[]
             {
                 AllowAdmin, SyncTimeout,
@@ -80,7 +80,7 @@ internal static void Unknown(string key)
                 Version, ConnectTimeout, Password,
                 TieBreaker, WriteBuffer, Ssl, SslHost,
                 ConfigChannel, AbortOnConnectFail, ResolveDns,
-                ChannelPrefix, Proxy
+                ChannelPrefix, Proxy, ConnectRetry
             }.ToDictionary(x => x, StringComparer.InvariantCultureIgnoreCase);
 
             public static string TryNormalize(string value)
@@ -105,7 +105,7 @@ public static string TryNormalize(string value)
 
         private Version defaultVersion;
 
-        private int? keepAlive, syncTimeout, connectTimeout, writeBuffer;
+        private int? keepAlive, syncTimeout, connectTimeout, writeBuffer, connectRetry;
 
         private Proxy? proxy;
 
@@ -153,6 +153,11 @@ public static string TryNormalize(string value)
         /// </summary>
         public string ClientName { get { return clientName; } set { clientName = value; } }
 
+        /// <summary>
+        /// The number of times to repeat the initial connect cycle if no servers respond promptly
+        /// </summary>
+        public int ConnectRetry { get { return connectRetry ?? 3; } set { connectRetry = value; } }
+
         /// <summary>
         /// The command-map associated with this configuration
         /// </summary>
@@ -302,6 +307,7 @@ public ConfigurationOptions Clone()
                 CertificateSelectionCallback = CertificateSelectionCallback,
                 ChannelPrefix = ChannelPrefix.Clone(),
                 SocketManager = SocketManager,
+                connectRetry = connectRetry
             };
             foreach (var item in endpoints)
                 options.endpoints.Add(item);
@@ -343,6 +349,7 @@ public override string ToString()
             Append(sb, OptionKeys.AbortOnConnectFail, abortOnConnectFail);
             Append(sb, OptionKeys.ResolveDns, resolveDns);
             Append(sb, OptionKeys.ChannelPrefix, (string)ChannelPrefix);
+            Append(sb, OptionKeys.ConnectRetry, connectRetry);
             Append(sb, OptionKeys.Proxy, proxy);
             if(commandMap != null) commandMap.AppendDeltas(sb);
             return sb.ToString();
@@ -433,7 +440,7 @@ static bool IsOption(string option, string prefix)
         void Clear()
         {
             clientName = serviceName = password = tieBreaker = sslHost = configChannel = null;
-            keepAlive = syncTimeout = connectTimeout = writeBuffer = null;
+            keepAlive = syncTimeout = connectTimeout = writeBuffer = connectRetry = null;
             allowAdmin = abortOnConnectFail = resolveDns = ssl = null;
             defaultVersion = null;
             endpoints.Clear();
@@ -500,6 +507,9 @@ private void DoParse(string configuration, bool ignoreUnknown)
                             case OptionKeys.ConnectTimeout:
                                 ConnectTimeout = OptionKeys.ParseInt32(key, value);
                                 break;
+                            case OptionKeys.ConnectRetry:
+                                ConnectRetry = OptionKeys.ParseInt32(key, value);
+                                break;
                             case OptionKeys.Version:
                                 DefaultVersion = OptionKeys.ParseVersion(key, value);
                                 break;

StackExchange.Redis/StackExchange/Redis/PhysicalBridge.cs

@@ -301,6 +301,17 @@ internal void OnConnected(PhysicalConnection connection)
             }
         }
 
+
+        internal void ResetNonConnected()
+        {
+            var tmp = physical;
+            if (tmp != null && state != (int)State.ConnectedEstablished)
+            {
+                tmp.RecordConnectionFailed(ConnectionFailureType.UnableToConnect);
+            }
+            GetConnection();
+        }
+
         internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailureType failureType, Exception innerException)
         {
             if (reportNextFailure)

StackExchange.Redis/StackExchange/Redis/ServerEndPoint.cs

@@ -46,6 +46,14 @@ internal sealed partial class ServerEndPoint : IDisposable
 
         private Version version;
 
+
+        internal void ResetNonConnected()
+        {
+            var tmp = interactive;
+            if (tmp != null) tmp.ResetNonConnected();
+            tmp = subscription;
+            if (tmp != null) tmp.ResetNonConnected();
+        }
         public ServerEndPoint(ConnectionMultiplexer multiplexer, EndPoint endpoint)
         {
             this.multiplexer = multiplexer;

StackExchange.Redis/StackExchange/Redis/SocketManager.cs

@@ -138,7 +138,8 @@ internal SocketToken BeginConnect(EndPoint endpoint, ISocketCallback callback)
                 }
                 throw;
             }
-            return new SocketToken(socket);
+            var token = new SocketToken(socket);
+            return token;
         }
         internal void SetFastLoopbackOption(Socket socket)
         {
@@ -228,6 +229,19 @@ private void EndConnectImpl(IAsyncResult ar)
                         break;
                 }
             }
+            catch(ObjectDisposedException)
+            {
+                ConnectionMultiplexer.TraceWithoutContext("(socket shutdown)");
+                if (tuple != null)
+                {
+                    try
+                    { tuple.Item2.Error(); }
+                    catch (Exception inner)
+                    {
+                        ConnectionMultiplexer.TraceWithoutContext(inner.Message);
+                    }
+                }
+            }
             catch(Exception outer)
             {
                 ConnectionMultiplexer.TraceWithoutContext(outer.Message);