Commit 59a7cb68 authored by Marc Gravell's avatar Marc Gravell

Merge pull request #369 from deepakverma/certificatevalidation

SE.Redis to surface error from serverendpoint as an innerexception 
parents c4b9fc34 17cc31e8
using NUnit.Framework;
using System.Threading;
namespace StackExchange.Redis.Tests
{
[TestFixture]
public class ConnectionFailedErrors : TestBase
{
[Test]
[TestCase(true)]
[TestCase(false)]
public void SSLCertificateValidationError(bool isCertValidationSucceeded)
{
string name, password;
GetAzureCredentials(out name, out password);
var options = new ConfigurationOptions();
options.EndPoints.Add(name + ".redis.cache.windows.net");
options.Ssl = true;
options.Password = password;
options.CertificateValidation += (sender, cert, chain, errors) => { return isCertValidationSucceeded; };
options.AbortOnConnectFail = false;
using (var connection = ConnectionMultiplexer.Connect(options))
{
connection.ConnectionFailed += (object sender, ConnectionFailedEventArgs e) =>
{
Assert.That(e.FailureType, Is.EqualTo(ConnectionFailureType.AuthenticationFailure));
};
if (!isCertValidationSucceeded)
{
//validate that in this case it throws an certificatevalidation exception
var ex = Assert.Throws<RedisConnectionException>(() => connection.GetDatabase().Ping());
var rde = (RedisConnectionException)ex.InnerException;
Assert.That(rde.FailureType, Is.EqualTo(ConnectionFailureType.AuthenticationFailure));
Assert.That(rde.InnerException.Message, Is.EqualTo("The remote certificate is invalid according to the validation procedure."));
}
else
{
Assert.DoesNotThrow(() => connection.GetDatabase().Ping());
}
//wait for a second for connectionfailed event to fire
Thread.Sleep(1000);
}
}
[Test]
public void AuthenticationFailureError()
{
string name, password;
GetAzureCredentials(out name, out password);
var options = new ConfigurationOptions();
options.EndPoints.Add(name + ".redis.cache.windows.net");
options.Ssl = true;
options.Password = "";
options.AbortOnConnectFail = false;
using (var muxer = ConnectionMultiplexer.Connect(options))
{
muxer.ConnectionFailed += (object sender, ConnectionFailedEventArgs e) =>
{
Assert.That(e.FailureType, Is.EqualTo(ConnectionFailureType.AuthenticationFailure));
};
var ex = Assert.Throws<RedisConnectionException>(() => muxer.GetDatabase().Ping());
var rde = (RedisConnectionException)ex.InnerException;
Assert.That(rde.FailureType, Is.EqualTo(ConnectionFailureType.AuthenticationFailure));
Assert.That(rde.InnerException.Message, Is.EqualTo("Error: NOAUTH Authentication required. Verify if the Redis password provided is correct."));
//wait for a second for connectionfailed event to fire
Thread.Sleep(1000);
}
}
[Test]
public void SocketFailureError()
{
var options = new ConfigurationOptions();
options.EndPoints.Add(".redis.cache.windows.net");
options.Ssl = true;
options.Password = "";
options.AbortOnConnectFail = false;
using (var muxer = ConnectionMultiplexer.Connect(options))
{
var ex = Assert.Throws<RedisConnectionException>(() => muxer.GetDatabase().Ping());
var rde = (RedisConnectionException)ex.InnerException;
Assert.That(rde.FailureType, Is.EqualTo(ConnectionFailureType.SocketFailure));
}
}
[Test]
public void CheckFailureRecovered()
{
try
{
using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
{
var conn = muxer.GetDatabase();
var server = muxer.GetServer(muxer.GetEndPoints()[0]);
muxer.AllowConnect = false;
SocketManager.ConnectCompletionType = CompletionType.Async;
server.SimulateConnectionFailure();
Assert.AreEqual(ConnectionFailureType.SocketFailure, ((RedisConnectionException)muxer.GetServerSnapshot()[0].LastException).FailureType);
// should reconnect within 1 keepalive interval
muxer.AllowConnect = true;
Thread.Sleep(2000);
Assert.Null(muxer.GetServerSnapshot()[0].LastException);
}
}
finally
{
SocketManager.ConnectCompletionType = CompletionType.Any;
ClearAmbientFailures();
}
}
}
}
using System;
using NUnit.Framework;
namespace StackExchange.Redis.Tests
{
[TestFixture]
public class ExceptionFactoryTests : TestBase
{
[Test]
public void NullLastException()
{
using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
{
var conn = muxer.GetDatabase();
Assert.Null(muxer.GetServerSnapshot()[0].LastException);
var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null, null, muxer.GetServerSnapshot());
Assert.Null(ex.InnerException);
}
}
[Test]
public void NullSnapshot()
{
var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null, null, null);
Assert.Null(ex.InnerException);
}
[Test]
public void MultipleEndpointsThrowAggregateException()
{
try
{
using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
{
var conn = muxer.GetDatabase();
muxer.AllowConnect = false;
SocketManager.ConnectCompletionType = CompletionType.Async;
foreach (var endpoint in muxer.GetEndPoints())
{
muxer.GetServer(endpoint).SimulateConnectionFailure();
}
var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null, null, muxer.GetServerSnapshot());
Assert.IsInstanceOf<RedisConnectionException>(ex);
Assert.IsInstanceOf<AggregateException>(ex.InnerException);
var aggException = (AggregateException)ex.InnerException;
Assert.That(aggException.InnerExceptions.Count, Is.EqualTo(2));
for (int i = 0; i < aggException.InnerExceptions.Count; i++)
{
Assert.That(((RedisConnectionException)aggException.InnerExceptions[i]).FailureType, Is.EqualTo(ConnectionFailureType.SocketFailure));
}
}
}
finally
{
SocketManager.ConnectCompletionType = CompletionType.Any;
ClearAmbientFailures();
}
}
[Test]
public void NullInnerExceptionForMultipleEndpointsWithNoLastException()
{
try
{
using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
{
var conn = muxer.GetDatabase();
muxer.AllowConnect = false;
SocketManager.ConnectCompletionType = CompletionType.Async;
var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null, null, muxer.GetServerSnapshot());
Assert.IsInstanceOf<RedisConnectionException>(ex);
Assert.Null(ex.InnerException);
}
}
finally
{
SocketManager.ConnectCompletionType = CompletionType.Any;
ClearAmbientFailures();
}
}
[Test]
public void ServerTakesPrecendenceOverSnapshot()
{
try
{
using (var muxer = Create(keepAlive: 1, connectTimeout: 10000, allowAdmin: true))
{
var conn = muxer.GetDatabase();
muxer.AllowConnect = false;
SocketManager.ConnectCompletionType = CompletionType.Async;
muxer.GetServer(muxer.GetEndPoints()[0]).SimulateConnectionFailure();
var ex = ExceptionFactory.NoConnectionAvailable(true, new RedisCommand(), null,muxer.GetServerSnapshot()[0], muxer.GetServerSnapshot());
Assert.IsInstanceOf<RedisConnectionException>(ex);
Assert.IsInstanceOf<Exception>(ex.InnerException);
Assert.That(muxer.GetServerSnapshot()[0].LastException, Is.EqualTo(ex.InnerException));
}
}
finally
{
SocketManager.ConnectCompletionType = CompletionType.Any;
ClearAmbientFailures();
}
}
}
}
......@@ -310,5 +310,15 @@ protected static TimeSpan RunConcurrent(Action work, int threads, int timeout =
return watch.Elapsed;
}
protected virtual void GetAzureCredentials(out string name, out string password)
{
var lines = File.ReadAllLines(@"d:\dev\azure.txt");
if (lines == null || lines.Length != 2)
Assert.Inconclusive("azure credentials missing");
name = lines[0];
password = lines[1];
}
}
}
......@@ -18,7 +18,7 @@ public enum ConnectionFailureType
/// </summary>
SocketFailure,
/// <summary>
/// The connection did not authenticate correctly
/// Either SSL Stream or Redis authentication failed
/// </summary>
AuthenticationFailure,
/// <summary>
......
......@@ -295,7 +295,7 @@ internal void MakeMaster(ServerEndPoint server, ReplicationChangeOptions options
if (server == null) throw new ArgumentNullException(nameof(server));
var srv = new RedisServer(this, server, null);
if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, RedisCommand.SLAVEOF, null, server);
if (!srv.IsConnected) throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, RedisCommand.SLAVEOF, null, server, GetServerSnapshot());
if (log == null) log = TextWriter.Null;
CommandMap.AssertAvailable(RedisCommand.SLAVEOF);
......@@ -1657,6 +1657,12 @@ internal void UpdateClusterRange(ClusterConfiguration configuration)
private readonly ServerSelectionStrategy serverSelectionStrategy;
internal ServerEndPoint[] GetServerSnapshot()
{
var tmp = serverSnapshot;
return tmp;
}
internal ServerEndPoint SelectServer(Message message)
{
if (message == null) return null;
......@@ -1866,7 +1872,7 @@ internal Task<T> ExecuteAsyncImpl<T>(Message message, ResultProcessor<T> process
var source = ResultBox<T>.Get(tcs);
if (!TryPushMessageToBridge(message, processor, source, ref server))
{
ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server));
ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server, GetServerSnapshot()));
}
return tcs.Task;
}
......@@ -1907,7 +1913,7 @@ internal T ExecuteSyncImpl<T>(Message message, ResultProcessor<T> processor, Ser
{
if (!TryPushMessageToBridge(message, processor, source, ref server))
{
throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server);
throw ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, message.Command, message, server, GetServerSnapshot());
}
if (Monitor.Wait(source, timeoutMilliseconds))
......
using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
namespace StackExchange.Redis
{
internal static class ExceptionFactory
{
const string DataCommandKey = "redis-command",
DataServerKey = "redis-server";
const string DataCommandKey = "redis-command",
DataServerKey = "redis-server",
DataServerEndpoint = "server-endpoint",
DataConnectionState = "connection-state",
DataLastFailure = "last-failure",
DataLastInnerException = "last-innerexception";
internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
......@@ -67,13 +75,49 @@ internal static Exception MultiSlot(bool includeDetail, Message message)
return ex;
}
internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, "No connection is available to service this operation: " + s);
if (includeDetail) AddDetail(ex, message, server, s);
return ex;
}
internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server, ServerEndPoint[] serverSnapshot)
{
string s = GetLabel(includeDetail, command, message);
if (server != null)
{
//if we already have the serverEndpoint for connection failure use that
//otherwise it would output state of all the endpoints
serverSnapshot = new ServerEndPoint[] { server };
}
string exceptionmessage = "No connection is available to service this operation: " + s ;
var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage, GetServerSnapshotInnerExceptions(serverSnapshot));
if (includeDetail)
{
AddDetail(ex, message, server, s);
}
return ex;
}
internal static Exception GetServerSnapshotInnerExceptions(ServerEndPoint[] serverSnapshot)
{
List<Exception> innerExceptions = new List<Exception>();
if (serverSnapshot != null)
{
for (int i = 0; i < serverSnapshot.Length; i++)
{
if (serverSnapshot[i].LastException != null)
{
var lastException = serverSnapshot[i].LastException;
innerExceptions.Add(lastException);
}
}
}
if (innerExceptions.Count == 1)
{
return innerExceptions[0];
}
else if(innerExceptions.Count > 1)
{
return new AggregateException(innerExceptions);
}
return null;
}
internal static Exception NotSupported(bool includeDetail, RedisCommand command)
{
......
......@@ -16,7 +16,7 @@ enum WriteResult
CompetingWriter,
NoConnection,
}
sealed partial class PhysicalBridge : IDisposable
{
internal readonly string Name;
......@@ -63,7 +63,9 @@ public enum State : byte
ConnectedEstablishing,
ConnectedEstablished,
Disconnected
}
}
public Exception LastException { get; private set; }
public ConnectionType ConnectionType { get; }
......@@ -264,7 +266,7 @@ internal void KeepAlive()
Multiplexer.Trace("Enqueue: " + msg);
if (!TryEnqueue(msg, ServerEndPoint.IsSlave))
{
OnInternalError(ExceptionFactory.NoConnectionAvailable(Multiplexer.IncludeDetailInExceptions, msg.Command, msg, ServerEndPoint));
OnInternalError(ExceptionFactory.NoConnectionAvailable(Multiplexer.IncludeDetailInExceptions, msg.Command, msg, ServerEndPoint, Multiplexer.GetServerSnapshot()));
}
}
}
......@@ -302,6 +304,7 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur
{
if (reportNextFailure)
{
LastException = innerException;
reportNextFailure = false; // until it is restored
var endpoint = ServerEndPoint.EndPoint;
Multiplexer.OnConnectionFailed(endpoint, ConnectionType, failureType, innerException, reconfigureNextFailure);
......@@ -350,6 +353,7 @@ internal void OnFullyEstablished(PhysicalConnection connection)
if (physical == connection && !isDisposed && ChangeState(State.ConnectedEstablishing, State.ConnectedEstablished))
{
reportNextFailure = reconfigureNextFailure = true;
LastException = null;
Interlocked.Exchange(ref failConnectCount, 0);
ServerEndPoint.OnFullyEstablished(connection);
Multiplexer.RequestWrite(this, true);
......@@ -381,6 +385,7 @@ internal void OnHeartbeat(bool ifConnectedOnly)
int connectTimeMilliseconds = unchecked(Environment.TickCount - VolatileWrapper.Read(ref connectStartTicks));
if (connectTimeMilliseconds >= Multiplexer.RawConfig.ConnectTimeout)
{
LastException = ExceptionFactory.UnableToConnect("ConnectTimeout");
Trace("Aborting connect");
// abort and reconnect
var snapshot = physical;
......
......@@ -781,9 +781,9 @@ SocketMode ISocketCallback.Connected(Stream stream, TextWriter log)
{
ssl.AuthenticateAsClient(host);
}
catch (AuthenticationException)
catch (AuthenticationException authexception)
{
RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure);
RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure, authexception);
Multiplexer.Trace("Encryption failure");
return SocketMode.Abort;
}
......
......@@ -30,13 +30,13 @@ public void Execute()
if (server == null)
{
FailNoServer(snapshot);
throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server,multiplexer.GetServerSnapshot());
}
var bridge = server.GetBridge(message.Command);
if (bridge == null)
{
FailNoServer(snapshot);
throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot());
}
// identity a list
......
......@@ -542,7 +542,7 @@ internal override Task<T> ExecuteAsync<T>(Message message, ResultProcessor<T> pr
// no need to deny exec-sync here; will be complete before they see if
var tcs = TaskSource.Create<T>(asyncState);
ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server));
ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot()));
return tcs.Task;
}
return base.ExecuteAsync<T>(message, processor, server);
......@@ -555,7 +555,7 @@ internal override T ExecuteSync<T>(Message message, ResultProcessor<T> processor
if (!server.IsConnected)
{
if (message == null || message.IsFireAndForget) return default(T);
throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server, multiplexer.GetServerSnapshot());
}
return base.ExecuteSync<T>(message, processor, server);
}
......
......@@ -1165,7 +1165,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, R
{
if (result.IsEqual(authFail) || result.IsEqual(authRequired))
{
connection.RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure);
connection.RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure, new Exception(result.ToString() + " Verify if the Redis password provided is correct."));
}
else if (result.AssertStarts(loading))
{
......
......@@ -91,6 +91,34 @@ public bool IsConnected
}
}
internal Exception LastException
{
get
{
var tmp1 = interactive;
var tmp2 = subscription;
//check if subscription endpoint has a better lastexception
if (tmp2 != null && tmp2.LastException != null)
{
if (!tmp2.LastException.Data["Redis-FailureType"].ToString().Equals(ConnectionFailureType.UnableToConnect.ToString()))
{
return tmp2.LastException;
}
}
return tmp1?.LastException;
}
}
internal PhysicalBridge.State ConnectionState
{
get
{
var tmp = interactive;
return tmp.ConnectionState;
}
}
public bool IsSlave { get { return isSlave; } set { SetConfig(ref isSlave, value); } }
public long OperationCount
......@@ -524,7 +552,7 @@ internal Task<T> QueueDirectAsync<T>(Message message, ResultProcessor<T> process
if (bridge == null) bridge = GetBridge(message.Command);
if (!bridge.TryEnqueue(message, isSlave))
{
ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, this));
ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, this, multiplexer.GetServerSnapshot()));
}
return tcs.Task;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment