Surface error information

StackExchange.Redis.Tests/ConnectionFailedErrors.cs

+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using NUnit.Framework;
+using System.Threading;
+
+namespace StackExchange.Redis.Tests
+{
+    [TestFixture]
+    public class ConnectionFailedErrors : TestBase
+    {
+        [Test]
+        [TestCase(true)]
+        [TestCase(false)]
+        public void SSLCertificateValidationError(bool isCertValidationSucceeded)
+        {
+            string name, password;
+            GetAzureCredentials(out name, out password);
+            var options = new ConfigurationOptions();
+            options.EndPoints.Add(name + ".redis.cache.windows.net");
+            options.Ssl = true;
+            options.Password = password;
+            options.CertificateValidation += (sender, cert, chain, errors) => { return isCertValidationSucceeded; };
+            options.AbortOnConnectFail = false;
+
+            using (var connection = ConnectionMultiplexer.Connect(options))
+            {
+                connection.ConnectionFailed += (object sender, ConnectionFailedEventArgs e) =>
+                {
+                    Assert.That(e.FailureType.ToString(), Is.EqualTo(ConnectionFailureType.AuthenticationFailure.ToString()));
+                };
+                if (!isCertValidationSucceeded)
+                {
+                    //validate that in this case it throws an certificatevalidation exception
+                    var ex = Assert.Throws<RedisConnectionException>(() => connection.GetDatabase().Ping());
+                  
+                    ((AggregateException)ex.InnerException).Handle(e =>
+                    {
+                       var rde = (RedisConnectionException)e;
+                       Assert.That(rde.FailureType.ToString(), Is.EqualTo(ConnectionFailureType.AuthenticationFailure.ToString()));
+                       Assert.That(rde.InnerException.Message, Is.EqualTo("The remote certificate is invalid according to the validation procedure."));
+                       return e is RedisConnectionException;
+                    });
+                    
+                }
+                else
+                {
+                    Assert.DoesNotThrow(() => connection.GetDatabase().Ping());
+                }
+
+                //wait for a second for connectionfailed event to fire
+                Thread.Sleep(1000);
+            }
+
+
+        }
+
+        [Test]
+        public void AuthenticationFailureError()
+        {
+            string name, password;
+            GetAzureCredentials(out name, out password);
+            var options = new ConfigurationOptions();
+            options.EndPoints.Add(name + ".redis.cache.windows.net");
+            options.Ssl = true;
+            options.Password = "";
+            options.AbortOnConnectFail = false;
+            using (var muxer = ConnectionMultiplexer.Connect(options))
+            {
+                muxer.ConnectionFailed += (object sender, ConnectionFailedEventArgs e) =>
+                {
+                    Assert.That(e.FailureType.ToString(), Is.EqualTo(ConnectionFailureType.AuthenticationFailure.ToString()));
+                };
+
+                var ex = Assert.Throws<RedisConnectionException>(() => muxer.GetDatabase().Ping());
+
+                ((AggregateException)ex.InnerException).Handle(e =>
+                {
+                    var rde = (RedisConnectionException)e;
+                    Assert.That(rde.FailureType.ToString(), Is.EqualTo(ConnectionFailureType.AuthenticationFailure.ToString()));
+                    return e is RedisConnectionException;
+                });
+                //wait for a second  for connectionfailed event to fire
+                Thread.Sleep(1000);
+            }
+        }
+
+        [Test]
+        public void SocketFailureError()
+        {
+            var options = new ConfigurationOptions();
+            options.EndPoints.Add(".redis.cache.windows.net");
+            options.Ssl = true;
+            options.Password = "";
+            options.AbortOnConnectFail = false;
+            using (var muxer = ConnectionMultiplexer.Connect(options))
+            {
+                var ex = Assert.Throws<RedisConnectionException>(() => muxer.GetDatabase().Ping());
+                 ((AggregateException)ex.InnerException).Handle(e =>
+                {
+                    var rde = (RedisConnectionException)e;
+                    Assert.That(rde.FailureType.ToString(), Is.EqualTo(ConnectionFailureType.SocketFailure.ToString()));
+                    return e is RedisConnectionException;
+                });
+            }
+        }
+
+        [Test]
+        public void CheckFailureRecovered()
+        {
+            try
+            {
+                using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
+                {
+                    var conn = muxer.GetDatabase();
+                    var server = muxer.GetServer(muxer.GetEndPoints()[0]);
+
+                    muxer.AllowConnect = false;
+                    SocketManager.ConnectCompletionType = CompletionType.Async;
+
+                    server.SimulateConnectionFailure();
+
+                    Assert.AreEqual(ConnectionFailureType.SocketFailure, ((RedisConnectionException)muxer.GetServerSnapShot()[0].LastException).FailureType);
+
+                    // should reconnect within 1 keepalive interval
+                    muxer.AllowConnect = true;
+                    Thread.Sleep(2000);
+
+                    Assert.Null(muxer.GetServerSnapShot()[0].LastException);
+                }
+            }
+            finally
+            {
+                SocketManager.ConnectCompletionType = CompletionType.Any;
+                ClearAmbientFailures();
+            }
+        }
+    }
+}

StackExchange.Redis.Tests/TestBase.cs

@@ -310,5 +310,15 @@ protected static TimeSpan RunConcurrent(Action work, int threads, int timeout =
             return watch.Elapsed;
         }
 
+        
+        protected virtual void GetAzureCredentials(out string name, out string password)
+        {
+            var lines = File.ReadAllLines(@"d:\dev\azure.txt");
+            if (lines == null || lines.Length != 2)
+                Assert.Inconclusive("azure credentials missing");
+            name = lines[0];
+            password = lines[1];
+        }
+
     }
 }

StackExchange.Redis/StackExchange/Redis/ConnectionMultiplexer.cs

@@ -295,7 +295,7 @@ internal void MakeMaster(ServerEndPoint server, ReplicationChangeOptions options
 
             if (server == null) throw new ArgumentNullException(nameof(server));
             var srv = new RedisServer(this, server, null);
-            if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, RedisCommand.SLAVEOF, null, server);
+            if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, RedisCommand.SLAVEOF, null, server, GetServerSnapShot());
 
             if (log == null) log = TextWriter.Null;
             CommandMap.AssertAvailable(RedisCommand.SLAVEOF);
@@ -1657,6 +1657,12 @@ internal void UpdateClusterRange(ClusterConfiguration configuration)
 
         private readonly ServerSelectionStrategy serverSelectionStrategy;
 
+        internal ServerEndPoint[] GetServerSnapShot()
+        {
+            var tmp = serverSnapshot;
+            return tmp;
+        }
+
         internal ServerEndPoint SelectServer(Message message)
         {
             if (message == null) return null;
@@ -1866,7 +1872,7 @@ internal Task<T> ExecuteAsyncImpl<T>(Message message, ResultProcessor<T> process
                 var source = ResultBox<T>.Get(tcs);
                 if (!TryPushMessageToBridge(message, processor, source, ref server))
                 {
-                    ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server));
+                    ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server, GetServerSnapShot()));
                 }
                 return tcs.Task;
             }
@@ -1907,7 +1913,7 @@ internal T ExecuteSyncImpl<T>(Message message, ResultProcessor<T> processor, Ser
                 {
                     if (!TryPushMessageToBridge(message, processor, source, ref server))
                     {
-                        throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server);
+                        throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server, GetServerSnapShot());
                     }
 
                     if (Monitor.Wait(source, timeoutMilliseconds))

StackExchange.Redis/StackExchange/Redis/ExceptionFactory.cs

 using System;
-
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+
 namespace StackExchange.Redis
 {
     internal static class ExceptionFactory
     {
-        const string DataCommandKey = "redis-command",
-            DataServerKey = "redis-server";
+        const string DataCommandKey = "redis-command",
+            DataServerKey = "redis-server",
+            DataServerEndpoint = "server-endpoint",
+            DataConnectionState = "connection-state",
+            DataLastFailure = "last-failure",
+            DataLastInnerException = "last-innerexception";
+
 
         internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
         {
@@ -67,13 +75,58 @@ internal static Exception MultiSlot(bool includeDetail, Message message)
             return ex;
         }
 
-        internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
-        {
-            string s = GetLabel(includeDetail, command, message);
-            var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, "No connection is available to service this operation: " + s);
-            if (includeDetail) AddDetail(ex, message, server, s);
-            return ex;
-        }
+        internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server, ServerEndPoint[] serverSnapshot)
+        {
+            string s = GetLabel(includeDetail, command, message);
+
+            if (server != null)
+            {
+                //if we already have the serverEndpoint for connection failure use that
+                //otherwise it would output state of all the endpoints
+                serverSnapshot = new ServerEndPoint[] { server };
+            }
+            List<Exception> data;
+            string exceptionmessage = "No connection is available to service this operation: " + s + GetServerSnapShotLabel(serverSnapshot, out data);
+            var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage,new AggregateException(data));
+            if (includeDetail)
+            {
+                AddDetail(ex, message, server, s);
+            }
+            return ex;
+        }
+
+        internal static string GetServerSnapShotLabel(ServerEndPoint[] serverSnapshot, out List<Exception> data)
+        {
+            List<Exception> exceptions = new List<Exception>();
+            StringBuilder connectionStateSummary = new StringBuilder();
+            Action<string, string> add = (k, v) =>
+             {
+                 connectionStateSummary.Append("; ");
+                 connectionStateSummary.Append(k);
+                 connectionStateSummary.Append(":");
+                 connectionStateSummary.Append(v);
+             };
+            if (serverSnapshot != null)
+            {
+                string serverSnapshotName;
+                for (int i = 0; i < serverSnapshot.Length; i++)
+                {
+                    serverSnapshotName = serverSnapshot[i].EndPoint.ToString();
+                    add(DataServerEndpoint, serverSnapshot[i].EndPoint.ToString());
+                    add(DataConnectionState, serverSnapshot[i].ConnectionState.ToString());
+                    
+                    if (serverSnapshot[i].LastException != null && serverSnapshot[i].LastException is RedisConnectionException)
+                    {
+                        var lastException = ((RedisConnectionException)serverSnapshot[i].LastException);
+                        exceptions.Add(lastException);
+                        add(DataLastFailure, lastException.FailureType.ToString());
+                        add(DataLastInnerException, lastException.InnerException?.Message);
+                    }
+                }
+            }
+            data = exceptions;
+            return connectionStateSummary.ToString();
+        }
 
         internal static Exception NotSupported(bool includeDetail, RedisCommand command)
         {
@@ -105,6 +158,7 @@ private static void AddDetail(Exception exception, Message message, ServerEndPoi
 
                 if (server != null) exception.Data.Add(DataServerKey, Format.ToString(server.EndPoint));
             }
+            
         }
 
         static string GetLabel(bool includeDetail, RedisCommand command, Message message)

StackExchange.Redis/StackExchange/Redis/PhysicalBridge.cs

@@ -16,7 +16,7 @@ enum WriteResult
         CompetingWriter,
         NoConnection,
     }
-
+    
     sealed partial class PhysicalBridge : IDisposable
     {
         internal readonly string Name;
@@ -63,11 +63,14 @@ public enum State : byte
             ConnectedEstablishing,
             ConnectedEstablished,
             Disconnected
-        }
+        }
+
+        public Exception LastException { get; private set; }
 
         public ConnectionType ConnectionType { get; }
 
         public bool IsConnected => state == (int)State.ConnectedEstablished;
+        
 
         public ConnectionMultiplexer Multiplexer { get; }
 
@@ -264,7 +267,7 @@ internal void KeepAlive()
                 Multiplexer.Trace("Enqueue: " + msg);
                 if (!TryEnqueue(msg, ServerEndPoint.IsSlave))
                 {
-                    OnInternalError(ExceptionFactory.NoConnectionAvailable(Multiplexer.IncludeDetailInExceptions, msg.Command, msg, ServerEndPoint));
+                    OnInternalError(ExceptionFactory.NoConnectionAvailable(Multiplexer.IncludeDetailInExceptions, msg.Command, msg, ServerEndPoint, Multiplexer.GetServerSnapShot()));
                 }
             }
         }
@@ -302,6 +305,7 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur
         {
             if (reportNextFailure)
             {
+                LastException = innerException;
                 reportNextFailure = false; // until it is restored
                 var endpoint = ServerEndPoint.EndPoint;
                 Multiplexer.OnConnectionFailed(endpoint, ConnectionType, failureType, innerException, reconfigureNextFailure);
@@ -350,6 +354,7 @@ internal void OnFullyEstablished(PhysicalConnection connection)
             if (physical == connection && !isDisposed && ChangeState(State.ConnectedEstablishing, State.ConnectedEstablished))
             {
                 reportNextFailure = reconfigureNextFailure = true;
+                LastException = null;
                 Interlocked.Exchange(ref failConnectCount, 0);
                 ServerEndPoint.OnFullyEstablished(connection);
                 Multiplexer.RequestWrite(this, true);

StackExchange.Redis/StackExchange/Redis/PhysicalConnection.cs

@@ -781,9 +781,9 @@ SocketMode ISocketCallback.Connected(Stream stream, TextWriter log)
                     {
                         ssl.AuthenticateAsClient(host);
                     }
-                    catch (AuthenticationException)
+                    catch (AuthenticationException authexception)
                     {
-                        RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure);
+                        RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure, authexception);
                         Multiplexer.Trace("Encryption failure");
                         return SocketMode.Abort;
                     }

StackExchange.Redis/StackExchange/Redis/RedisBatch.cs

@@ -30,13 +30,13 @@ public void Execute()
                 if (server == null)
                 {
                     FailNoServer(snapshot);
-                    throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
+                    throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server,multiplexer.GetServerSnapShot());
                 }
                 var bridge = server.GetBridge(message.Command);
                 if (bridge == null)
                 {
                     FailNoServer(snapshot);
-                    throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
+                    throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapShot());
                 }
 
                 // identity a list

StackExchange.Redis/StackExchange/Redis/RedisServer.cs

@@ -542,7 +542,7 @@ internal override Task<T> ExecuteAsync<T>(Message message, ResultProcessor<T> pr
 
                 // no need to deny exec-sync here; will be complete before they see if
                 var tcs = TaskSource.Create<T>(asyncState);
-                ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server));
+                ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapShot()));
                 return tcs.Task;
             }
             return base.ExecuteAsync<T>(message, processor, server);
@@ -555,7 +555,7 @@ internal override T ExecuteSync<T>(Message message, ResultProcessor<T> processor
             if (!server.IsConnected)
             {
                 if (message == null || message.IsFireAndForget) return default(T);
-                throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
+                throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapShot());
             }
             return base.ExecuteSync<T>(message, processor, server);
         }

StackExchange.Redis/StackExchange/Redis/ServerEndPoint.cs

@@ -91,6 +91,37 @@ public bool IsConnected
             }
         }
 
+        internal Exception LastException
+        {
+            get
+            {
+                var tmp1 = interactive;
+                var tmp2 = subscription;
+
+                //check if subscription endpoint has a better lastexception
+                if (tmp2 != null && tmp2.LastException != null)
+                {
+                    if (!tmp2.LastException.Data["Redis-FailureType"].ToString().Equals(ConnectionFailureType.UnableToConnect.ToString()))
+                    {
+                        return tmp2.LastException;
+                    }
+                }
+                return tmp1?.LastException;
+            }
+        }
+
+        internal PhysicalBridge.State ConnectionState
+        {
+            get
+            {
+                var tmp = interactive;
+                return tmp.ConnectionState;
+            }
+        }
+
+
+
+
         public bool IsSlave { get { return isSlave; } set { SetConfig(ref isSlave, value); } }
 
         public long OperationCount
@@ -524,7 +555,7 @@ internal Task<T> QueueDirectAsync<T>(Message message, ResultProcessor<T> process
             if (bridge == null) bridge = GetBridge(message.Command);
             if (!bridge.TryEnqueue(message, isSlave))
             {
-                ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, this));
+                ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, this, multiplexer.GetServerSnapShot()));
             }
             return tcs.Task;
         }