Skip to content

S
StackExchange.Redis
Commit 5633b59a authored by Marc Gravell's avatar Marc Gravell
Browse files
Options

Merge branch 'feat/exceptiondata' of git@gitlab.stackexchange.com:core/stackexchange-redis.git

parents 7d3a92e0 8bea5864
Show whitespace changes
Inline Side-by-side
Showing with 72 additions and 28 deletions
StackExchange.Redis/StackExchange/Redis/ConnectionMultiplexer.cs
View file @ 5633b59a
......@@ -566,7 +566,9 @@ private void LogLockedWithThreadPoolStats(TextWriter log, string message)
{
var sb = new StringBuilder();
sb.Append(message);
AppendThreadPoolStats(sb);
string iocp, worker;
GetThreadPoolStats(out iocp, out worker);
sb.Append(", IOCP: ").Append(iocp).Append(", WORKER: ").Append(worker);
LogLocked(log, sb.ToString());
}
}
......@@ -1863,6 +1865,7 @@ internal T ExecuteSyncImpl<T>(Message message, ResultProcessor<T> processor, Ser
Trace("Timeout performing " + message.ToString());
Interlocked.Increment(ref syncTimeouts);
string errMessage;
List<Tuple<string, string>> data = null;
if (server == null || !IncludeDetailInExceptions)
{
errMessage = "Timeout performing " + message.Command.ToString();
......@@ -1870,21 +1873,35 @@ internal T ExecuteSyncImpl<T>(Message message, ResultProcessor<T> processor, Ser
else
{
int inst, qu, qs, qc, wr, wq, @in, ar;
string iocp, worker;
#if !__MonoCS__
var mgrState = socketManager.State;
#endif
var sb = new StringBuilder("Timeout performing ").Append(message.CommandAndKey);
data = new List<Tuple<string, string>> {Tuple.Create("Message", message.CommandAndKey)};
Action<string, string, string> add = (lk, sk, v) =>
{
data.Add(Tuple.Create(lk, v));
sb.Append(", " + sk + ": " + v);
};
int queue = server.GetOutstandingCount(message.Command, out inst, out qu, out qs, out qc, out wr, out wq, out @in, out ar);
var sb = new StringBuilder("Timeout performing ").Append(message.CommandAndKey)
.Append(", inst: ").Append(inst)
GetThreadPoolStats(out iocp, out worker);
add("Instantaneous", "inst", inst.ToString());
#if !__MonoCS__
.Append(", mgr: ").Append(mgrState)
add("Manager-State", "mgr", mgrState.ToString());
#endif
.Append(", queue: ").Append(queue).Append(", qu=").Append(qu)
.Append(", qs=").Append(qs).Append(", qc=").Append(qc)
.Append(", wr=").Append(wr).Append("/").Append(wq)
.Append(", in=").Append(@in).Append("/").Append(ar);
AppendThreadPoolStats(sb);
add("Queue-Length", "queue", queue.ToString());
add("Queue-Outstanding", "qu", qu.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
add("Queue-Completion-Outstanding", "qc", qc.ToString());
add("Active-Writers", "wr", wr.ToString());
add("Write-Queue", "wq", wq.ToString());
add("Inbound-Bytes", "in", @in.ToString());
add("Active-Readers", "ar", ar.ToString());
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
errMessage = sb.ToString();
if (stormLogThreshold >= 0 && queue >= stormLogThreshold && Interlocked.CompareExchange(ref haveStormLog, 1, 0) == 0)
......@@ -1894,7 +1911,15 @@ internal T ExecuteSyncImpl<T>(Message message, ResultProcessor<T> processor, Ser
else Interlocked.Exchange(ref stormLogSnapshot, log);
}
}
throw ExceptionFactory.Timeout(IncludeDetailInExceptions, errMessage, message, server);
var timeoutEx = ExceptionFactory.Timeout(IncludeDetailInExceptions, errMessage, message, server);
if (data != null)
{
foreach (var kv in data)
{
timeoutEx.Data["Redis-" + kv.Item1] = kv.Item2;
}
}
throw timeoutEx;
// very important not to return "source" to the pool here
}
}
......@@ -1907,7 +1932,7 @@ internal T ExecuteSyncImpl<T>(Message message, ResultProcessor<T> processor, Ser
return val;
}
}
private static void AppendThreadPoolStats(StringBuilder errorMessage)
private static void GetThreadPoolStats(out string iocp, out string worker)
{
//BusyThreads = TP.GetMaxThreads() –TP.GetAVailable();
//If BusyThreads >= TP.GetMinThreads(), then threadpool growth throttling is possible.
......@@ -1924,8 +1949,8 @@ private static void AppendThreadPoolStats(StringBuilder errorMessage)
int busyIoThreads = maxIoThreads - freeIoThreads;
int busyWorkerThreads = maxWorkerThreads - freeWorkerThreads;
errorMessage.AppendFormat(", IOCP:(Busy={0},Free={1},Min={2},Max={3})", busyIoThreads, freeIoThreads, minIoThreads, maxIoThreads);
errorMessage.AppendFormat(", WORKER:(Busy={0},Free={1},Min={2},Max={3})", busyWorkerThreads, freeWorkerThreads, minWorkerThreads, maxWorkerThreads);
iocp = string.Format("(Busy={0},Free={1},Min={2},Max={3})", busyIoThreads, freeIoThreads, minIoThreads, maxIoThreads);
worker = string.Format("(Busy={0},Free={1},Min={2},Max={3})", busyWorkerThreads, freeWorkerThreads, minWorkerThreads, maxWorkerThreads);
}
/// <summary>
......
StackExchange.Redis/StackExchange/Redis/PhysicalConnection.cs
View file @ 5633b59a
......@@ -177,23 +177,42 @@ public void RecordConnectionFailed(ConnectionFailureType failureType, Exception
lastBeat = Thread.VolatileRead(ref lastBeatTickCount);
int unansweredRead = Thread.VolatileRead(ref firstUnansweredWriteTickCount);
string message = failureType + " on " + Format.ToString(bridge.ServerEndPoint.EndPoint) + "/" + connectionType
+ ", origin: " + origin
+ ", input-buffer: " + ioBufferBytes + ", outstanding: " + GetSentAwaitingResponseCount()
+ ", last-read: " + unchecked(now - lastRead) / 1000 + "s ago, last-write: " + unchecked(now - lastWrite) / 1000 + "s ago"
+ ", unanswered-write: " + unchecked(now - unansweredRead) / 1000 + "s ago"
+ ", keep-alive: " + bridge.ServerEndPoint.WriteEverySeconds + "s, pending: "
+ bridge.GetPendingCount() + ", state: " + oldState + ", last-heartbeat: " + (lastBeat == 0 ? "never" : (unchecked(now - lastBeat) / 1000 + "s ago"))
+ (bridge.IsBeating ? " (mid-beat)" : "") + ", last-mbeat: " + multiplexer.LastHeartbeatSecondsAgo + "s ago, global: "
+ ConnectionMultiplexer.LastGlobalHeartbeatSecondsAgo + "s ago"
var exMessage = new StringBuilder(failureType + " on " + Format.ToString(bridge.ServerEndPoint.EndPoint) + "/" + connectionType);
var data = new List<Tuple<string, string>>
{
Tuple.Create("FailureType", failureType.ToString()),
Tuple.Create("EndPoint", Format.ToString(bridge.ServerEndPoint.EndPoint))
};
Action<string, string, string> add = (lk, sk, v) =>
{
data.Add(Tuple.Create(lk, v));
exMessage.Append(", " + sk + ": " + v);
};
add("Origin", "origin", origin);
add("Input-Buffer", "input-buffer", ioBufferBytes.ToString());
add("Outstanding-Responses", "outstanding", GetSentAwaitingResponseCount().ToString());
add("Last-Read", "last-read", unchecked(now - lastRead) / 1000 + "s ago");
add("Last-Write", "last-write", unchecked(now - lastWrite) / 1000 + "s ago");
add("Unanswered-Write", "unanswered-write", unchecked(now - unansweredRead) / 1000 + "s ago");
add("Keep-Alive", "keep-alive", bridge.ServerEndPoint.WriteEverySeconds + "s");
add("Pending", "pending", bridge.GetPendingCount().ToString());
add("Previous-Physical-State", "state", oldState.ToString());
add("Last-Heartbeat", "last-heartbeat", (lastBeat == 0 ? "never" : (unchecked(now - lastBeat)/1000 + "s ago"))+ (bridge.IsBeating ? " (mid-beat)" : "") );
add("Last-Multiplexer-Heartbeat", "last-mbeat", multiplexer.LastHeartbeatSecondsAgo + "s ago");
add("Last-Global-Heartbeat", "global", ConnectionMultiplexer.LastGlobalHeartbeatSecondsAgo + "s ago");
#if !__MonoCS__
+ ", mgr: "+ bridge.Multiplexer.SocketManager.State
add("SocketManager-State", "mgr", bridge.Multiplexer.SocketManager.State.ToString());
#endif
;
var ex = innerException == null
? new RedisConnectionException(failureType, message)
: new RedisConnectionException(failureType, message, innerException);
? new RedisConnectionException(failureType, exMessage.ToString())
: new RedisConnectionException(failureType, exMessage.ToString(), innerException);
foreach (var kv in data)
{
ex.Data["Redis-" + kv.Item1] = kv.Item2;
}
bridge.OnConnectionFailed(this, failureType, ex);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment