Merge pull request #369 from deepakverma/certificatevalidation

SE.Redis to surface error from serverendpoint as an innerexception 

StackExchange.Redis.Tests/ConnectionFailedErrors.cs

+using NUnit.Framework;
+using System.Threading;
+
+namespace StackExchange.Redis.Tests
+{
+    [TestFixture]
+    public class ConnectionFailedErrors : TestBase
+    {
+        [Test]
+        [TestCase(true)]
+        [TestCase(false)]
+        public void SSLCertificateValidationError(bool isCertValidationSucceeded)
+        {
+            string name, password;
+            GetAzureCredentials(out name, out password);
+            var options = new ConfigurationOptions();
+            options.EndPoints.Add(name + ".redis.cache.windows.net");
+            options.Ssl = true;
+            options.Password = password;
+            options.CertificateValidation += (sender, cert, chain, errors) => { return isCertValidationSucceeded; };
+            options.AbortOnConnectFail = false;
+
+            using (var connection = ConnectionMultiplexer.Connect(options))
+            {
+                connection.ConnectionFailed += (object sender, ConnectionFailedEventArgs e) =>
+                {
+                    Assert.That(e.FailureType, Is.EqualTo(ConnectionFailureType.AuthenticationFailure));
+                };
+                if (!isCertValidationSucceeded)
+                {
+                    //validate that in this case it throws an certificatevalidation exception
+                    var ex = Assert.Throws<RedisConnectionException>(() => connection.GetDatabase().Ping());
+                    var rde = (RedisConnectionException)ex.InnerException;
+                    Assert.That(rde.FailureType, Is.EqualTo(ConnectionFailureType.AuthenticationFailure));
+                    Assert.That(rde.InnerException.Message, Is.EqualTo("The remote certificate is invalid according to the validation procedure."));
+                }
+                else
+                {
+                    Assert.DoesNotThrow(() => connection.GetDatabase().Ping());
+                }
+
+                //wait for a second for connectionfailed event to fire
+                Thread.Sleep(1000);
+            }
+
+
+        }
+
+        [Test]
+        public void AuthenticationFailureError()
+        {
+            string name, password;
+            GetAzureCredentials(out name, out password);
+            var options = new ConfigurationOptions();
+            options.EndPoints.Add(name + ".redis.cache.windows.net");
+            options.Ssl = true;
+            options.Password = "";
+            options.AbortOnConnectFail = false;
+            using (var muxer = ConnectionMultiplexer.Connect(options))
+            {
+                muxer.ConnectionFailed += (object sender, ConnectionFailedEventArgs e) =>
+                {
+                    Assert.That(e.FailureType, Is.EqualTo(ConnectionFailureType.AuthenticationFailure));
+                };
+                var ex = Assert.Throws<RedisConnectionException>(() => muxer.GetDatabase().Ping());
+                var rde = (RedisConnectionException)ex.InnerException;
+                Assert.That(rde.FailureType, Is.EqualTo(ConnectionFailureType.AuthenticationFailure));
+                Assert.That(rde.InnerException.Message, Is.EqualTo("Error: NOAUTH Authentication required. Verify if the Redis password provided is correct."));
+                //wait for a second  for connectionfailed event to fire
+                Thread.Sleep(1000);
+            }
+        }
+
+        [Test]
+        public void SocketFailureError()
+        {
+            var options = new ConfigurationOptions();
+            options.EndPoints.Add(".redis.cache.windows.net");
+            options.Ssl = true;
+            options.Password = "";
+            options.AbortOnConnectFail = false;
+            using (var muxer = ConnectionMultiplexer.Connect(options))
+            {
+                var ex = Assert.Throws<RedisConnectionException>(() => muxer.GetDatabase().Ping());
+                var rde = (RedisConnectionException)ex.InnerException;
+                Assert.That(rde.FailureType, Is.EqualTo(ConnectionFailureType.SocketFailure));
+            }
+        }
+
+        [Test]
+        public void CheckFailureRecovered()
+        {
+            try
+            {
+                using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
+                {
+                    var conn = muxer.GetDatabase();
+                    var server = muxer.GetServer(muxer.GetEndPoints()[0]);
+
+                    muxer.AllowConnect = false;
+                    SocketManager.ConnectCompletionType = CompletionType.Async;
+
+                    server.SimulateConnectionFailure();
+
+                    Assert.AreEqual(ConnectionFailureType.SocketFailure, ((RedisConnectionException)muxer.GetServerSnapshot()[0].LastException).FailureType);
+
+                    // should reconnect within 1 keepalive interval
+                    muxer.AllowConnect = true;
+                    Thread.Sleep(2000);
+
+                    Assert.Null(muxer.GetServerSnapshot()[0].LastException);
+                }
+            }
+            finally
+            {
+                SocketManager.ConnectCompletionType = CompletionType.Any;
+                ClearAmbientFailures();
+            }
+        }
+    }
+}

StackExchange.Redis.Tests/ExceptionFactoryTests.cs

+using System;
+using NUnit.Framework;
+
+namespace StackExchange.Redis.Tests
+{
+    [TestFixture]
+    public class ExceptionFactoryTests : TestBase
+    {
+        [Test]
+        public void NullLastException()
+        {
+            using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
+            {
+                var conn = muxer.GetDatabase();
+                Assert.Null(muxer.GetServerSnapshot()[0].LastException);
+                var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null, null, muxer.GetServerSnapshot());
+                Assert.Null(ex.InnerException);
+            }
+
+        }
+
+        [Test]
+        public void NullSnapshot()
+        {
+            var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null, null, null);
+            Assert.Null(ex.InnerException);
+        }
+
+        [Test]
+        public void MultipleEndpointsThrowAggregateException()
+        {
+            try
+            {
+                using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
+                {
+                    var conn = muxer.GetDatabase();
+                    muxer.AllowConnect = false;
+                    SocketManager.ConnectCompletionType = CompletionType.Async;
+
+                    foreach (var endpoint in muxer.GetEndPoints())
+                    {
+                        muxer.GetServer(endpoint).SimulateConnectionFailure();
+                    }
+
+                    var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null, null, muxer.GetServerSnapshot());
+                    Assert.IsInstanceOf<RedisConnectionException>(ex);
+                    Assert.IsInstanceOf<AggregateException>(ex.InnerException);
+                    var aggException = (AggregateException)ex.InnerException;
+                    Assert.That(aggException.InnerExceptions.Count, Is.EqualTo(2));
+                    for (int i = 0; i < aggException.InnerExceptions.Count; i++)
+                    {
+                        Assert.That(((RedisConnectionException)aggException.InnerExceptions[i]).FailureType, Is.EqualTo(ConnectionFailureType.SocketFailure));
+                    }
+                }
+            }
+            finally
+            {
+                SocketManager.ConnectCompletionType = CompletionType.Any;
+                ClearAmbientFailures();
+            }
+        }
+
+        [Test]
+        public void NullInnerExceptionForMultipleEndpointsWithNoLastException()
+        {
+            try
+            {
+                using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
+                {
+                    var conn = muxer.GetDatabase();
+                    muxer.AllowConnect = false;
+                    SocketManager.ConnectCompletionType = CompletionType.Async;
+                    var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null, null, muxer.GetServerSnapshot());
+                    Assert.IsInstanceOf<RedisConnectionException>(ex);
+                    Assert.Null(ex.InnerException);
+                 }
+            }
+            finally
+            {
+                SocketManager.ConnectCompletionType = CompletionType.Any;
+                ClearAmbientFailures();
+            }
+        }
+
+        [Test]
+        public void ServerTakesPrecendenceOverSnapshot()
+        {
+             try
+            {
+                using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
+                {
+                    var conn = muxer.GetDatabase();
+                    muxer.AllowConnect = false;
+                    SocketManager.ConnectCompletionType = CompletionType.Async;
+
+                    muxer.GetServer(muxer.GetEndPoints()[0]).SimulateConnectionFailure();
+
+                    var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null,muxer.GetServerSnapshot()[0], muxer.GetServerSnapshot());
+                    Assert.IsInstanceOf<RedisConnectionException>(ex);
+                    Assert.IsInstanceOf<Exception>(ex.InnerException);
+                    Assert.That(muxer.GetServerSnapshot()[0].LastException, Is.EqualTo(ex.InnerException));
+                }
+            }
+            finally
+            {
+                SocketManager.ConnectCompletionType = CompletionType.Any;
+                ClearAmbientFailures();
+            }
+
+        }
+
+    }
+}

StackExchange.Redis.Tests/TestBase.cs

@@ -310,5 +310,15 @@ protected static TimeSpan RunConcurrent(Action work, int threads, int timeout =
             return watch.Elapsed;
         }
 
+        
+        protected virtual void GetAzureCredentials(out string name, out string password)
+        {
+            var lines = File.ReadAllLines(@"d:\dev\azure.txt");
+            if (lines == null || lines.Length != 2)
+                Assert.Inconclusive("azure credentials missing");
+            name = lines[0];
+            password = lines[1];
+        }
+
     }
 }

StackExchange.Redis/StackExchange/Redis/ConnectionFailureType.cs

@@ -18,7 +18,7 @@ public enum ConnectionFailureType
         /// </summary>
         SocketFailure,
         /// <summary>
-        /// The connection did not authenticate correctly
+        /// Either SSL Stream or Redis authentication failed
         /// </summary>
         AuthenticationFailure,
         /// <summary>

StackExchange.Redis/StackExchange/Redis/ConnectionMultiplexer.cs

@@ -295,7 +295,7 @@ internal void MakeMaster(ServerEndPoint server, ReplicationChangeOptions options
 
             if (server == null) throw new ArgumentNullException(nameof(server));
             var srv = new RedisServer(this, server, null);
-            if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, RedisCommand.SLAVEOF, null, server);
+            if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, RedisCommand.SLAVEOF, null, server, GetServerSnapshot());
 
             if (log == null) log = TextWriter.Null;
             CommandMap.AssertAvailable(RedisCommand.SLAVEOF);
@@ -1657,6 +1657,12 @@ internal void UpdateClusterRange(ClusterConfiguration configuration)
 
         private readonly ServerSelectionStrategy serverSelectionStrategy;
 
+        internal ServerEndPoint[] GetServerSnapshot()
+        {
+            var tmp = serverSnapshot;
+            return tmp;
+        }
+
         internal ServerEndPoint SelectServer(Message message)
         {
             if (message == null) return null;
@@ -1866,7 +1872,7 @@ internal Task<T> ExecuteAsyncImpl<T>(Message message, ResultProcessor<T> process
                 var source = ResultBox<T>.Get(tcs);
                 if (!TryPushMessageToBridge(message, processor, source, ref server))
                 {
-                    ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server));
+                    ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server, GetServerSnapshot()));
                 }
                 return tcs.Task;
             }
@@ -1907,7 +1913,7 @@ internal T ExecuteSyncImpl<T>(Message message, ResultProcessor<T> processor, Ser
                 {
                     if (!TryPushMessageToBridge(message, processor, source, ref server))
                     {
-                        throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server);
+                        throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server, GetServerSnapshot());
                     }
 
                     if (Monitor.Wait(source, timeoutMilliseconds))

StackExchange.Redis/StackExchange/Redis/ExceptionFactory.cs

 using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
 
 namespace StackExchange.Redis
 {
     internal static class ExceptionFactory
     {
         const string DataCommandKey = "redis-command",
-            DataServerKey = "redis-server";
+            DataServerKey = "redis-server",
+            DataServerEndpoint = "server-endpoint",
+            DataConnectionState = "connection-state",
+            DataLastFailure = "last-failure",
+            DataLastInnerException = "last-innerexception";
+
 
         internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
         {
@@ -67,14 +75,50 @@ internal static Exception MultiSlot(bool includeDetail, Message message)
             return ex;
         }
 
-        internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
+        internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server, ServerEndPoint[] serverSnapshot)
         {
             string s = GetLabel(includeDetail, command, message);
-            var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, "No connection is available to service this operation: " + s);
-            if (includeDetail) AddDetail(ex, message, server, s);
+
+            if (server != null)
+            {
+                //if we already have the serverEndpoint for connection failure use that
+                //otherwise it would output state of all the endpoints
+                serverSnapshot = new ServerEndPoint[] { server };
+            }
+            string exceptionmessage = "No connection is available to service this operation: " + s ;
+            var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage, GetServerSnapshotInnerExceptions(serverSnapshot));
+            if (includeDetail)
+            {
+                AddDetail(ex, message, server, s);
+            }
             return ex;
         }
 
+        internal static Exception GetServerSnapshotInnerExceptions(ServerEndPoint[] serverSnapshot)
+        {
+            List<Exception> innerExceptions = new List<Exception>();
+            if (serverSnapshot != null)
+            {
+                for (int i = 0; i < serverSnapshot.Length; i++)
+                {
+                    if (serverSnapshot[i].LastException != null)
+                    {
+                        var lastException = serverSnapshot[i].LastException;
+                        innerExceptions.Add(lastException);
+                    }
+                }
+            }
+            if (innerExceptions.Count == 1)
+            {
+                return innerExceptions[0];
+            }
+            else if(innerExceptions.Count > 1)
+            {
+                return new AggregateException(innerExceptions);
+            }
+            return null;
+        }
+
         internal static Exception NotSupported(bool includeDetail, RedisCommand command)
         {
             string s = GetLabel(includeDetail, command, null);

StackExchange.Redis/StackExchange/Redis/PhysicalBridge.cs

@@ -65,6 +65,8 @@ public enum State : byte
             Disconnected
         }
 
+        public Exception LastException { get; private set; }
+
         public ConnectionType ConnectionType { get; }
 
         public bool IsConnected => state == (int)State.ConnectedEstablished;
@@ -264,7 +266,7 @@ internal void KeepAlive()
                 Multiplexer.Trace("Enqueue: " + msg);
                 if (!TryEnqueue(msg, ServerEndPoint.IsSlave))
                 {
-                    OnInternalError(ExceptionFactory.NoConnectionAvailable(Multiplexer.IncludeDetailInExceptions, msg.Command, msg, ServerEndPoint));
+                    OnInternalError(ExceptionFactory.NoConnectionAvailable(Multiplexer.IncludeDetailInExceptions, msg.Command, msg, ServerEndPoint, Multiplexer.GetServerSnapshot()));
                 }
             }
         }
@@ -302,6 +304,7 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur
         {
             if (reportNextFailure)
             {
+                LastException = innerException;
                 reportNextFailure = false; // until it is restored
                 var endpoint = ServerEndPoint.EndPoint;
                 Multiplexer.OnConnectionFailed(endpoint, ConnectionType, failureType, innerException, reconfigureNextFailure);
@@ -350,6 +353,7 @@ internal void OnFullyEstablished(PhysicalConnection connection)
             if (physical == connection && !isDisposed && ChangeState(State.ConnectedEstablishing, State.ConnectedEstablished))
             {
                 reportNextFailure = reconfigureNextFailure = true;
+                LastException = null;
                 Interlocked.Exchange(ref failConnectCount, 0);
                 ServerEndPoint.OnFullyEstablished(connection);
                 Multiplexer.RequestWrite(this, true);
@@ -381,6 +385,7 @@ internal void OnHeartbeat(bool ifConnectedOnly)
                         int connectTimeMilliseconds = unchecked(Environment.TickCount - VolatileWrapper.Read(ref connectStartTicks));
                         if (connectTimeMilliseconds >= Multiplexer.RawConfig.ConnectTimeout)
                         {
+                            LastException = ExceptionFactory.UnableToConnect("ConnectTimeout");
                             Trace("Aborting connect");
                             // abort and reconnect
                             var snapshot = physical;

StackExchange.Redis/StackExchange/Redis/PhysicalConnection.cs

@@ -781,9 +781,9 @@ SocketMode ISocketCallback.Connected(Stream stream, TextWriter log)
                     {
                         ssl.AuthenticateAsClient(host);
                     }
-                    catch (AuthenticationException)
+                    catch (AuthenticationException authexception)
                     {
-                        RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure);
+                        RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure, authexception);
                         Multiplexer.Trace("Encryption failure");
                         return SocketMode.Abort;
                     }

StackExchange.Redis/StackExchange/Redis/RedisBatch.cs

@@ -30,13 +30,13 @@ public void Execute()
                 if (server == null)
                 {
                     FailNoServer(snapshot);
-                    throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
+                    throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server,multiplexer.GetServerSnapshot());
                 }
                 var bridge = server.GetBridge(message.Command);
                 if (bridge == null)
                 {
                     FailNoServer(snapshot);
-                    throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
+                    throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot());
                 }
 
                 // identity a list

StackExchange.Redis/StackExchange/Redis/RedisServer.cs

@@ -542,7 +542,7 @@ internal override Task<T> ExecuteAsync<T>(Message message, ResultProcessor<T> pr
 
                 // no need to deny exec-sync here; will be complete before they see if
                 var tcs = TaskSource.Create<T>(asyncState);
-                ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server));
+                ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot()));
                 return tcs.Task;
             }
             return base.ExecuteAsync<T>(message, processor, server);
@@ -555,7 +555,7 @@ internal override T ExecuteSync<T>(Message message, ResultProcessor<T> processor
             if (!server.IsConnected)
             {
                 if (message == null || message.IsFireAndForget) return default(T);
-                throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
+                throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot());
             }
             return base.ExecuteSync<T>(message, processor, server);
         }

StackExchange.Redis/StackExchange/Redis/ResultProcessor.cs

@@ -1165,7 +1165,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, R
                 {
                     if (result.IsEqual(authFail) || result.IsEqual(authRequired))
                     {
-                        connection.RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure);
+                        connection.RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure, new Exception(result.ToString() + " Verify if the Redis password provided is correct."));
                     }
                     else if (result.AssertStarts(loading))
                     {

StackExchange.Redis/StackExchange/Redis/ServerEndPoint.cs

@@ -91,6 +91,34 @@ public bool IsConnected
             }
         }
 
+        internal Exception LastException
+        {
+            get
+            {
+                var tmp1 = interactive;
+                var tmp2 = subscription;
+
+                //check if subscription endpoint has a better lastexception
+                if (tmp2 != null && tmp2.LastException != null)
+                {
+                    if (!tmp2.LastException.Data["Redis-FailureType"].ToString().Equals(ConnectionFailureType.UnableToConnect.ToString()))
+                    {
+                        return tmp2.LastException;
+                    }
+                }
+                return tmp1?.LastException;
+            }
+        }
+
+        internal PhysicalBridge.State ConnectionState
+        {
+            get
+            {
+                var tmp = interactive;
+                return tmp.ConnectionState;
+            }
+        }
+
         public bool IsSlave { get { return isSlave; } set { SetConfig(ref isSlave, value); } }
 
         public long OperationCount
@@ -524,7 +552,7 @@ internal Task<T> QueueDirectAsync<T>(Message message, ResultProcessor<T> process
             if (bridge == null) bridge = GetBridge(message.Command);
             if (!bridge.TryEnqueue(message, isSlave))
             {
-                ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, this));
+                ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, this, multiplexer.GetServerSnapshot()));
             }
             return tcs.Task;
         }