Improved heartbeat code; introduced dedicated reader (can be pooled via...

Improved heartbeat code; introduced dedicated reader (can be pooled via SocketManager); tidied solition directory

Redis Configs/redis-cli 7000.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7000
\ No newline at end of file

Redis Configs/redis-cli 7001.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7001
\ No newline at end of file

Redis Configs/redis-cli 7002.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7002
\ No newline at end of file

Redis Configs/redis-cli 7003.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7003
\ No newline at end of file

Redis Configs/redis-cli 7004.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7004
\ No newline at end of file

Redis Configs/redis-cli 7005.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7005
\ No newline at end of file

Redis Configs/redis-cli master.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -p 6379
\ No newline at end of file

Redis Configs/redis-cli secure.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -p 6381
\ No newline at end of file

Redis Configs/redis-cli slave.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-cli.exe -p 6380
\ No newline at end of file

Redis Configs/redis-server master.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-server.exe master.conf
\ No newline at end of file

Redis Configs/redis-server secure.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-server.exe secure.conf
\ No newline at end of file

Redis Configs/redis-server slave.cmd

+@..\packages\Redis-64.2.6.12.1\tools\redis-server.exe slave.conf
\ No newline at end of file

StackExchange.Redis.Tests/BasicOps.cs

@@ -8,7 +8,7 @@
 namespace StackExchange.Redis.Tests
 {
     [TestFixture]
-    public class ConnectTests : TestBase
+    public class BasicOpsTests : TestBase
     {
         [Test]
         [TestCase(true)]

StackExchange.Redis.Tests/Config.cs

@@ -67,7 +67,7 @@ public void TalkToNonsenseServer()
         [Test]
         public void TestManaulHeartbeat()
         {
-            using (var muxer = Create(keepAlive: 2000))
+            using (var muxer = Create(keepAlive: 2))
             {
                 var conn = muxer.GetDatabase();
                 conn.Ping();
@@ -79,7 +79,7 @@ public void TestManaulHeartbeat()
 
                 var after = muxer.OperationCount;
 
-                Assert.AreEqual(before + 2, after);
+                Assert.IsTrue(after >= before + 4);
 
             }
         }
@@ -296,7 +296,7 @@ public void TestAutomaticHeartbeat()
                         Thread.Sleep(TimeSpan.FromSeconds(8));
 
                         var after = innerMuxer.OperationCount;
-                        Assert.AreEqual(before + 2, after);
+                        Assert.IsTrue(after >= before + 4);
 
                     }
                 }

StackExchange.Redis.Tests/FloatingPoint.cs

 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using NUnit.Framework;
 
 namespace StackExchange.Redis.Tests

StackExchange.Redis.Tests/Naming.cs

@@ -106,6 +106,7 @@ static bool IgnoreMethodConventions(MethodInfo method)
                 case "CreateTransaction":
                 case "IsConnected":
                 case "SetScan":
+                case "SubscribedEndpoint":
                     return true;
             }
             return false;

StackExchange.Redis.Tests/PubSub.cs

 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using NUnit.Framework;
@@ -229,52 +230,98 @@ public void TestPatternPubSub(bool preserveOrder)
         }
 
         [Test]
-        public void SubscriptionsSurviceMasterSwitch()
+        [TestCase(false)]
+        [TestCase(true)]
+        public void SubscriptionsSurviveMasterSwitch(bool useSharedSocketManager)
         {
-            using (var a = Create(allowAdmin: true))
-            using (var b = Create(allowAdmin: true))
+            using (var a = Create(allowAdmin: true, useSharedSocketManager: useSharedSocketManager))
+            using (var b = Create(allowAdmin: true, useSharedSocketManager: useSharedSocketManager))
             {
                 RedisChannel channel = Me();
                 var subA = a.GetSubscriber();
                 var subB = b.GetSubscriber();
                 
                 long masterChanged = 0, aCount = 0, bCount = 0;
-                a.MasterChanged += delegate { Interlocked.Increment(ref masterChanged); };
-                subA.Subscribe(channel, delegate { Interlocked.Increment(ref aCount); });
-                subB.Subscribe(channel, delegate { Interlocked.Increment(ref bCount); });
-
-                //var epA = subA.IdentifyEndpoint(channel);
-                //var epB = subB.IdentifyEndpoint(channel);
-                //Console.WriteLine(epA);
-                //Console.WriteLine(epB);
-                subA.Publish(channel, "a");
-                subB.Publish(channel, "b");
+                a.ConfigurationChangedBroadcast += delegate {
+                    Console.WriteLine("a noticed config broadcast: " + Interlocked.Increment(ref masterChanged)); 
+                };
+                b.ConfigurationChangedBroadcast += delegate {
+                    Console.WriteLine("b noticed config broadcast: " + Interlocked.Increment(ref masterChanged));
+                };
+                subA.Subscribe(channel, (ch, message) => {
+                    Console.WriteLine("a got message: " + message);
+                    Interlocked.Increment(ref aCount);
+                });
+                subB.Subscribe(channel, (ch, message) => {
+                    Console.WriteLine("b got message: " + message);
+                    Interlocked.Increment(ref bCount);
+                });
+
+
+                Assert.IsFalse(a.GetServer(PrimaryServer, PrimaryPort).IsSlave, PrimaryPortString + " is master via a");
+                Assert.IsTrue(a.GetServer(PrimaryServer, SlavePort).IsSlave, SlavePortString + " is slave via a");
+                Assert.IsFalse(b.GetServer(PrimaryServer, PrimaryPort).IsSlave, PrimaryPortString + " is master via b");
+                Assert.IsTrue(b.GetServer(PrimaryServer, SlavePort).IsSlave, SlavePortString + " is slave via b");
+
+
+                var epA = subA.SubscribedEndpoint(channel);
+                var epB = subB.SubscribedEndpoint(channel);
+                Console.WriteLine("a: " + EndPointCollection.ToString(epA));
+                Console.WriteLine("b: " + EndPointCollection.ToString(epB));
+                subA.Publish(channel, "a1");
+                subB.Publish(channel, "b1");
                 subA.Ping();
                 subB.Ping();
-
-                Assert.AreEqual(0, Interlocked.Read(ref masterChanged), "master");
+                
                 Assert.AreEqual(2, Interlocked.Read(ref aCount), "a");
                 Assert.AreEqual(2, Interlocked.Read(ref bCount), "b");
+                Assert.AreEqual(0, Interlocked.Read(ref masterChanged), "master");
 
                 try
                 {
-                    b.GetServer(PrimaryServer, SlavePort).MakeMaster(ReplicationChangeOptions.All);
-                    Thread.Sleep(100);
-                    //epA = subA.IdentifyEndpoint(channel);
-                    //epB = subB.IdentifyEndpoint(channel);
-                    //Console.WriteLine(epA);
-                    //Console.WriteLine(epB);
-                    subA.Publish(channel, "a");
-                    subB.Publish(channel, "b");
+                    Interlocked.Exchange(ref masterChanged, 0);
+                    Interlocked.Exchange(ref aCount, 0);
+                    Interlocked.Exchange(ref bCount, 0);
+                    Console.WriteLine("Changing master...");
+                    using (var sw = new StringWriter())
+                    {
+                        a.GetServer(PrimaryServer, SlavePort).MakeMaster(ReplicationChangeOptions.All, sw);
+                        Console.WriteLine(sw);
+                    }
+                    subA.Ping();
+                    subB.Ping();
+                    Console.WriteLine("Pausing...");
+                    Thread.Sleep(2000);
+
+                    Assert.IsTrue(a.GetServer(PrimaryServer, PrimaryPort).IsSlave, PrimaryPortString + " is slave via a");
+                    Assert.IsFalse(a.GetServer(PrimaryServer, SlavePort).IsSlave, SlavePortString + " is master via a");
+                    Assert.IsTrue(b.GetServer(PrimaryServer, PrimaryPort).IsSlave, PrimaryPortString + " is slave via b");
+                    Assert.IsFalse(b.GetServer(PrimaryServer, SlavePort).IsSlave, SlavePortString + " is master via b");
+
+                    Console.WriteLine("Pause complete");
+                    var counters = a.GetCounters();
+                    Console.WriteLine("a outstanding: " + counters.TotalOutstanding);
+                    counters = b.GetCounters();
+                    Console.WriteLine("b outstanding: " + counters.TotalOutstanding);
                     subA.Ping();
+                    subB.Ping();
+                    epA = subA.SubscribedEndpoint(channel);
+                    epB = subB.SubscribedEndpoint(channel);
+                    Console.WriteLine("a: " + EndPointCollection.ToString(epA));
+                    Console.WriteLine("b: " + EndPointCollection.ToString(epB));
+                    Console.WriteLine("a2 sent to: " + subA.Publish(channel, "a2"));
+                    Console.WriteLine("b2 sent to: " + subB.Publish(channel, "b2"));
                     subA.Ping();
                     subB.Ping();
-                    Assert.AreEqual(2, Interlocked.Read(ref masterChanged), "master");
-                    Assert.AreEqual(4, Interlocked.Read(ref aCount), "a");
-                    Assert.AreEqual(4, Interlocked.Read(ref bCount), "b");
+                    Console.WriteLine("Checking...");
+                    
+                    Assert.AreEqual(2, Interlocked.Read(ref aCount), "a");
+                    Assert.AreEqual(2, Interlocked.Read(ref bCount), "b");
+                    Assert.AreEqual(4, Interlocked.CompareExchange(ref masterChanged, 0, 0), "master");
                 }
                 finally
                 {
+                    Console.WriteLine("Restoring configuration...");
                     try
                     {
                         a.GetServer(PrimaryServer, PrimaryPort).MakeMaster(ReplicationChangeOptions.All);

StackExchange.Redis.Tests/TestBase.cs

@@ -13,8 +13,19 @@
 namespace StackExchange.Redis.Tests
 {
 
-    public abstract class TestBase
+    public abstract class TestBase : IDisposable
     {
+
+        private readonly SocketManager socketManager;
+
+        protected TestBase()
+        {
+            socketManager = new SocketManager(GetType().Name);
+        }
+        public void Dispose()
+        {
+            socketManager.Dispose();
+        }
 #if VERBOSE
         protected const int AsyncOpsQty = 100, SyncOpsQty = 10;
 #else
@@ -85,7 +96,7 @@ public void Teardown()
         }
 
         protected const int PrimaryPort = 6379, SlavePort = 6380, SecurePort = 6381;
-        protected const string PrimaryServer = "127.0.0.1", SecurePassword = "changeme", PrimaryPortString = "6379", SecurePortString = "6381";
+        protected const string PrimaryServer = "127.0.0.1", SecurePassword = "changeme", PrimaryPortString = "6379", SlavePortString = "6380", SecurePortString = "6381";
         internal static Task Swallow(Task task)
         {
             if (task != null) task.ContinueWith(swallowErrors, TaskContinuationOptions.OnlyOnFaulted);
@@ -115,7 +126,7 @@ protected IServer GetServer(ConnectionMultiplexer muxer)
             string clientName = null, int? syncTimeout = null, bool? allowAdmin = null, int? keepAlive = null,
             int? connectTimeout = null, string password = null, string tieBreaker = null, TextWriter log = null,
             bool fail = true, string[] disabledCommands = null, bool checkConnect = true, bool pause = true, string failMessage = null,
-            string channelPrefix = null)
+            string channelPrefix = null, bool useSharedSocketManager = true)
         {
             if(pause) Thread.Sleep(500); // get a lot of glitches when hammering new socket creations etc; pace it out a bit
             string configuration = GetConfiguration();
@@ -127,6 +138,7 @@ protected IServer GetServer(ConnectionMultiplexer muxer)
                     map[cmd] = null;
                 config.CommandMap = CommandMap.Create(map);
             }
+            if (useSharedSocketManager) config.SocketManager = socketManager;
             if (channelPrefix != null) config.ChannelPrefix = channelPrefix;
             if (tieBreaker != null) config.TieBreaker = tieBreaker;
             if (password != null) config.Password = string.IsNullOrEmpty(password) ? null : password;

StackExchange.Redis.sln

@@ -12,23 +12,23 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{D3090D
 		.nuget\packages.config = .nuget\packages.config
 	EndProjectSection
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Redis", "Redis", "{29A8EF11-C420-41F5-B8DC-BB586EF4D827}"
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Redis Configs", "Redis Configs", "{29A8EF11-C420-41F5-B8DC-BB586EF4D827}"
 	ProjectSection(SolutionItems) = preProject
-		master.conf = master.conf
-		redis-cli 7000.cmd = redis-cli 7000.cmd
-		redis-cli 7001.cmd = redis-cli 7001.cmd
-		redis-cli 7002.cmd = redis-cli 7002.cmd
-		redis-cli 7003.cmd = redis-cli 7003.cmd
-		redis-cli 7004.cmd = redis-cli 7004.cmd
-		redis-cli 7005.cmd = redis-cli 7005.cmd
-		redis-cli master.cmd = redis-cli master.cmd
-		redis-cli secure.cmd = redis-cli secure.cmd
-		redis-cli slave.cmd = redis-cli slave.cmd
-		redis-server master.cmd = redis-server master.cmd
-		redis-server secure.cmd = redis-server secure.cmd
-		redis-server slave.cmd = redis-server slave.cmd
-		secure.conf = secure.conf
-		slave.conf = slave.conf
+		Redis Configs\master.conf = Redis Configs\master.conf
+		Redis Configs\redis-cli 7000.cmd = Redis Configs\redis-cli 7000.cmd
+		Redis Configs\redis-cli 7001.cmd = Redis Configs\redis-cli 7001.cmd
+		Redis Configs\redis-cli 7002.cmd = Redis Configs\redis-cli 7002.cmd
+		Redis Configs\redis-cli 7003.cmd = Redis Configs\redis-cli 7003.cmd
+		Redis Configs\redis-cli 7004.cmd = Redis Configs\redis-cli 7004.cmd
+		Redis Configs\redis-cli 7005.cmd = Redis Configs\redis-cli 7005.cmd
+		Redis Configs\redis-cli master.cmd = Redis Configs\redis-cli master.cmd
+		Redis Configs\redis-cli secure.cmd = Redis Configs\redis-cli secure.cmd
+		Redis Configs\redis-cli slave.cmd = Redis Configs\redis-cli slave.cmd
+		Redis Configs\redis-server master.cmd = Redis Configs\redis-server master.cmd
+		Redis Configs\redis-server secure.cmd = Redis Configs\redis-server secure.cmd
+		Redis Configs\redis-server slave.cmd = Redis Configs\redis-server slave.cmd
+		Redis Configs\secure.conf = Redis Configs\secure.conf
+		Redis Configs\slave.conf = Redis Configs\slave.conf
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{709E0CE4-F0BA-4933-A4FD-4A8B6668A5D4}"

StackExchange.Redis/StackExchange.Redis.csproj

@@ -63,7 +63,7 @@
     <Reference Include="System.IO.Compression" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="StackExchange\RedisChannel.cs" />
+    <Compile Include="StackExchange\Redis\RedisChannel.cs" />
     <Compile Include="StackExchange\Redis\Bitwise.cs" />
     <Compile Include="StackExchange\Redis\ClientFlags.cs" />
     <Compile Include="StackExchange\Redis\ClientInfo.cs" />
@@ -133,6 +133,7 @@
     <Compile Include="StackExchange\Redis\ServerSelectionStrategy.cs" />
     <Compile Include="StackExchange\Redis\ServerType.cs" />
     <Compile Include="StackExchange\Redis\SetOperation.cs" />
+    <Compile Include="StackExchange\Redis\SocketManager.cs" />
     <Compile Include="StackExchange\Redis\StringSplits.cs" />
     <Compile Include="StackExchange\Redis\TaskContinuationCheck.cs" />
     <Compile Include="StackExchange\Redis\When.cs" />

StackExchange.Redis/StackExchange/Redis/ConfigurationOptions.cs

@@ -59,6 +59,12 @@ public ConfigurationOptions()
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1009:DeclareEventHandlersCorrectly")]
         public event RemoteCertificateValidationCallback CertificateValidation;
 
+        /// <summary>
+        /// Gets or sets the SocketManager instance to be used with these options; if this is null a per-multiplexer
+        /// SocketManager is created automatically.
+        /// </summary>
+        public SocketManager SocketManager {  get;set; }
+
         /// <summary>
         /// Indicates whether admin operations should be allowed
         /// </summary>
@@ -100,7 +106,7 @@ public ConfigurationOptions()
         public EndPointCollection EndPoints { get { return endpoints; } }
 
         /// <summary>
-        /// Specifies the time in milliseconds at which connections should be pinged to ensure validity
+        /// Specifies the time in seconds at which connections should be pinged to ensure validity
         /// </summary>
         public int KeepAlive { get { return keepAlive.GetValueOrDefault(-1); } set { keepAlive = value; } }
 
@@ -178,7 +184,8 @@ public ConfigurationOptions Clone()
                 CommandMap = CommandMap,
                 CertificateValidation = CertificateValidation,
                 CertificateSelection = CertificateSelection,
-                ChannelPrefix = ChannelPrefix.Clone()
+                ChannelPrefix = ChannelPrefix.Clone(),
+                SocketManager = SocketManager,
             };
             foreach (var item in endpoints)
                 options.endpoints.Add(item);
@@ -298,6 +305,7 @@ void Clear()
             CertificateValidation = null;
             CommandMap = CommandMap.Default;
             ChannelPrefix = default(RedisChannel);
+            SocketManager = null;
         }
 
         object ICloneable.Clone() { return Clone(); }

StackExchange.Redis/StackExchange/Redis/ConnectionMultiplexer.ReaderWriter.cs

@@ -6,12 +6,19 @@ namespace StackExchange.Redis
 {
     partial class ConnectionMultiplexer
     {
-        partial void OnCreateReaderWriter()
+        internal SocketManager SocketManager {  get {  return socketManager; } }
+        private SocketManager socketManager;
+        private bool ownsSocketManager;
+
+        partial void OnCreateReaderWriter(ConfigurationOptions configuration)
         {
+            this.ownsSocketManager = configuration.SocketManager == null;
+            this.socketManager = configuration.SocketManager ?? new SocketManager(configuration.ClientName);
+
             // we need a dedicated writer, because when under heavy ambient load
             // (a busy asp.net site, for example), workers are not reliable enough
             Thread dedicatedWriter = new Thread(writeAllQueues);
-            dedicatedWriter.Name = "SE.Redis.Writer";
+            dedicatedWriter.Name = socketManager.Name + ":Write";
             dedicatedWriter.IsBackground = true; // should not keep process alive
             dedicatedWriter.Start(this); // will self-exit when disposed
         }
@@ -22,6 +29,8 @@ partial class ConnectionMultiplexer
             { // make sure writer threads know to exit
                 Monitor.PulseAll(writeQueue);
             }
+            if (ownsSocketManager) socketManager.Dispose();
+            socketManager = null;
         }
 
         private readonly Queue<PhysicalBridge> writeQueue = new Queue<PhysicalBridge>();

StackExchange.Redis/StackExchange/Redis/ISubscriber.cs

@@ -86,5 +86,13 @@ public interface ISubscriber : IRedis
         /// </summary>
         [IgnoreNamePrefix]
         Task<EndPoint> IdentifyEndpointAsync(RedisChannel channel, CommandFlags flags = CommandFlags.None);
+
+        /// <summary>
+        /// Inidicate to which redis server we are actively subscribed for a given channel; returns null if
+        /// the channel is not actively subscribed
+        /// </summary>
+        [IgnoreNamePrefix]
+        EndPoint SubscribedEndpoint(RedisChannel channel);
+
     }
 }
\ No newline at end of file

StackExchange.Redis/StackExchange/Redis/Message.cs

@@ -63,8 +63,8 @@ abstract class Message : ICompletable
 
         protected RedisCommand command;
 
-        private const CommandFlags AskingFlag = (CommandFlags)32,
-                                   InternalCallFlag = (CommandFlags)128;
+        private const CommandFlags AskingFlag = (CommandFlags)32;
+        internal const CommandFlags InternalCallFlag = (CommandFlags)128;
 
         
         public virtual void AppendStormLog(StringBuilder sb)
@@ -349,12 +349,13 @@ public bool IsMasterOnly()
         }
 
         /// <summary>
-        /// This does two important things:
+        /// This does a few important things:
         /// 1: it suppresses error events for commands that the user isn't interested in
         ///    (i.e. "why does my standalone server keep saying ERR unknown command 'cluster' ?")
         /// 2: it allows the initial PING and GET (during connect) to get queued rather
         ///    than be rejected as no-server-available (note that this doesn't apply to
         ///    handshake messages, as they bypass the queue completely)
+        /// 3: it disables non-pref logging, as it is usually server-targeted
         /// </summary>
         public void SetInternalCall()
         {
@@ -363,7 +364,7 @@ public void SetInternalCall()
 
         public override string ToString()
         {
-            return string.Format("[{0}]:{1} ({2})", Db, Command,
+            return string.Format("[{0}]:{1} ({2})", Db, CommandAndKey,
                 resultProcessor == null ? "(n/a)" : resultProcessor.GetType().Name);
         }
 

StackExchange.Redis/StackExchange/Redis/PhysicalBridge.cs

@@ -23,7 +23,7 @@ private static readonly Message
         private int beating;
         int failConnectCount = 0;
         volatile bool isDisposed;
-        private volatile int missedHeartbeats;
+        //private volatile int missedHeartbeats;
         private long operationCount, socketCount;
         private int pendingCount;
         private volatile PhysicalConnection physical;
@@ -116,7 +116,6 @@ public bool TryEnqueue(Message message, bool isSlave)
                     // you can go in the queue, but we won't be starting
                     // a worker, because the handshake has not completed
                     queue.Push(message);
-                    LogNonPreferred(message.Flags, isSlave);
                     Interlocked.Increment(ref pendingCount);
                     return true;
                 }
@@ -140,15 +139,18 @@ public bool TryEnqueue(Message message, bool isSlave)
         }
         private void LogNonPreferred(CommandFlags flags, bool isSlave)
         {
-            if (isSlave)
+            if ((flags & Message.InternalCallFlag) == 0) // don't log internal-call
             {
-                if (Message.GetMasterSlaveFlags(flags) == CommandFlags.PreferMaster)
-                    Interlocked.Increment(ref nonPreferredEndpointCount);
-            }
-            else
-            {
-                if (Message.GetMasterSlaveFlags(flags) == CommandFlags.PreferSlave)
-                    Interlocked.Increment(ref nonPreferredEndpointCount);
+                if (isSlave)
+                {
+                    if (Message.GetMasterSlaveFlags(flags) == CommandFlags.PreferMaster)
+                        Interlocked.Increment(ref nonPreferredEndpointCount);
+                }
+                else
+                {
+                    if (Message.GetMasterSlaveFlags(flags) == CommandFlags.PreferSlave)
+                        Interlocked.Increment(ref nonPreferredEndpointCount);
+                }
             }
         }
         long nonPreferredEndpointCount;
@@ -231,27 +233,31 @@ internal void IncrementOpCount()
         internal void KeepAlive()
         {
             var commandMap = multiplexer.CommandMap;
-            Message msg;
+            Message msg = null;
             switch (connectionType)
             {
                 case ConnectionType.Interactive:
                     if (commandMap.IsAvailable(RedisCommand.PING))
                     {
                         msg = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.PING);
-                        msg.SetInternalCall();
-                        serverEndPoint.QueueDirectFireAndForget(msg, ResultProcessor.DemandPONG);
+                        msg.SetSource(ResultProcessor.DemandPONG, null);
                     }
                     break;
                 case ConnectionType.Subscription:
                     if (commandMap.IsAvailable(RedisCommand.UNSUBSCRIBE))
                     {
-                        RedisKey channel = Guid.NewGuid().ToByteArray();
-                        msg = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.UNSUBSCRIBE, channel);
-                        msg.SetInternalCall();
-                        serverEndPoint.QueueDirectFireAndForget(msg, ResultProcessor.TrackSubscriptions);
+                        msg = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.UNSUBSCRIBE,
+                            (RedisChannel)Guid.NewGuid().ToByteArray());
+                        msg.SetSource(ResultProcessor.TrackSubscriptions, null);
                     }
                     break;
             }
+            if(msg != null)
+            {
+                msg.SetInternalCall();
+                multiplexer.Trace("Enqueue: " + msg);
+                TryEnqueue(msg, serverEndPoint.IsSlave);
+            }
         }
 
         internal void OnConnected(PhysicalConnection connection)
@@ -267,12 +273,13 @@ internal void OnConnected(PhysicalConnection connection)
             }
         }
 
-        internal void OnConnectionFailed(EndPoint endPoint, ConnectionFailureType failureType, Exception innerException)
+        internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailureType failureType, Exception innerException)
         {
-            if (reportNextFailure)
+            if (connection == physical && reportNextFailure)
             {
                 reportNextFailure = false; // until it is restored
-                multiplexer.OnConnectionFailed(endPoint, failureType, innerException, reconfigureNextFailure);
+                var endpoint = serverEndPoint.EndPoint;
+                multiplexer.OnConnectionFailed(endpoint, failureType, innerException, reconfigureNextFailure);
             }
         }
 
@@ -327,7 +334,10 @@ internal void OnFullyEstablished(PhysicalConnection connection)
                 try { connection.Dispose(); } catch { }
             }
         }
-
+        internal int GetPendingCount()
+        {
+            return Thread.VolatileRead(ref pendingCount);
+        }
         internal void OnHeartbeat()
         {
             bool runThisTime = false;
@@ -348,11 +358,10 @@ internal void OnHeartbeat()
                         var tmp = physical;
                         if (tmp != null)
                         {
-                            int maxMissed = serverEndPoint.MaxMissedHeartbeats;
-                            if (maxMissed > 0 && ++missedHeartbeats >= maxMissed && Thread.VolatileRead(ref pendingCount) == 0)
+                            int writeEvery = serverEndPoint.WriteEverySeconds;
+                            if (writeEvery > 0 && tmp.LastWriteSecondsAgo >= writeEvery && Thread.VolatileRead(ref pendingCount) == 0)
                             {
                                 Trace("OnHeartbeat - overdue");
-                                missedHeartbeats = 0;
                                 if (state == (int)State.ConnectedEstablished)
                                 {
                                     KeepAlive();
@@ -385,11 +394,6 @@ internal void RemovePhysical(PhysicalConnection connection)
             Interlocked.CompareExchange(ref physical, null, connection);
         }
 
-        internal void Seen()
-        {
-            missedHeartbeats = 0;
-        }
-
         [Conditional("VERBOSE")]
         internal void Trace(string message)
         {
@@ -662,23 +666,26 @@ internal WriteResult WriteQueue(int maxWork)
                 }
 
                 int count = 0;
+                Message last = null;
                 while (true)
                 {
                     var next = queue.Dequeue();
                     if (next == null)
                     {
                         Trace("Nothing to write; exiting");
-                        {
-                            conn.Flush();
-                            return WriteResult.QueueEmpty;
-                        }
+                        Trace(last != null, "Flushed up to: " + last);
+                        conn.Flush();
+                        return WriteResult.QueueEmpty;
                     }
+                    last = next;
                     var newPendingCount = Interlocked.Decrement(ref pendingCount);
                     Trace("Now pending: " + newPendingCount);
                     WriteMessageDirect(conn, next);
                     count++;
                     if (maxWork > 0 && count >= maxWork)
                     {
+                        Trace("Work limit; exiting");
+                        Trace(last != null, "Flushed up to: " + last);
                         conn.Flush();
                         break;
                     }

StackExchange.Redis/StackExchange/Redis/RedisSubscriber.cs

@@ -96,29 +96,43 @@ public Task<EndPoint> IdentifyEndpointAsync(RedisChannel channel, CommandFlags f
             msg.SetInternalCall();
             return ExecuteAsync(msg, ResultProcessor.ConnectionIdentity);
         }
+
+        public EndPoint SubscribedEndpoint(RedisChannel channel)
+        {
+            var server = multiplexer.GetSubscribedServer(channel);
+            return server == null ? null : server.EndPoint;
+        }
     }
     partial class ConnectionMultiplexer
     {
 
-        internal bool SubscriberConnected(RedisChannel channel = default(RedisChannel))
+        internal ServerEndPoint GetSubscribedServer(RedisChannel channel)
         {
-            ServerEndPoint server;
             if (!channel.IsNullOrEmpty)
             {
-                lock(subscriptions)
+                lock (subscriptions)
                 {
                     Subscription sub;
-                    if(subscriptions.TryGetValue(channel, out sub))
+                    if (subscriptions.TryGetValue(channel, out sub))
                     {
-                        server = sub.GetOwner();
+                        return sub.GetOwner();
                     }
                 }
             }
+            return null;
+        }
+
+        internal bool SubscriberConnected(RedisChannel channel = default(RedisChannel))
+        {
+            var server = GetSubscribedServer(channel);
+            if (server != null) return server.IsConnected;
 
             server = SelectServer(-1, RedisCommand.SUBSCRIBE, CommandFlags.DemandMaster, default(RedisKey));
             return server != null && server.IsConnected;
         }
 
+
+
         private sealed class Subscription
         {
             private Action<RedisChannel, RedisValue> handler;
@@ -326,6 +340,5 @@ internal void ResendSubscriptions(ServerEndPoint server)
             }
             return false;
         }
-
     }
 }

StackExchange.Redis/StackExchange/Redis/ResultProcessor.cs

@@ -376,20 +376,19 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes
                                 if (key.Assert(timeout) && arr[(i * 2) + 1].TryGetInt64(out i64))
                                 {
                                     // note the configuration is in seconds
-                                    int timeoutMilliseconds = checked((int)i64) * 1000,
-                                        targetMilliseconds;
-                                    if (timeoutMilliseconds > 0)
+                                    int timeoutSeconds = checked((int)i64), targetSeconds;
+                                    if (timeoutSeconds > 0)
                                     {
-                                        if (timeoutMilliseconds >= 60000)
+                                        if (timeoutSeconds >= 60)
                                         {
-                                            targetMilliseconds = timeoutMilliseconds - 15000; // time to spare...
+                                            targetSeconds = timeoutSeconds - 20; // time to spare...
                                         }
                                         else
                                         {
-                                            targetMilliseconds = (timeoutMilliseconds * 3) / 4;
+                                            targetSeconds = (timeoutSeconds * 3) / 4;
                                         }
-                                        server.Multiplexer.Trace("Auto-configured timeout: " + targetMilliseconds + "ms");
-                                        server.SetHeartbeatMilliseconds(targetMilliseconds);
+                                        server.Multiplexer.Trace("Auto-configured timeout: " + targetSeconds + "s");
+                                        server.WriteEverySeconds = targetSeconds;
                                     }
                                 }
                                 else if (key.Assert(databases) && arr[(i * 2) + 1].TryGetInt64(out i64))
@@ -800,7 +799,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes
                     case ResultType.BulkString:
                         string s = result.GetString();
                         RedisType value;
-                        if (!Enum.TryParse<RedisType>(s, true, out value)) value = StackExchange.Redis.RedisType.Unknown;
+                        if (!Enum.TryParse<RedisType>(s, true, out value)) value = global::StackExchange.Redis.RedisType.Unknown;
                         SetResult(message, value);
                         return true;
                 }

StackExchange.Redis/StackExchange/Redis/ServerEndPoint.cs

@@ -31,7 +31,7 @@ internal sealed partial class ServerEndPoint : IDisposable
 
         private readonly ConnectionMultiplexer multiplexer;
 
-        private int databases, maxMissedHeartbeats;
+        private int databases, writeEverySeconds;
 
         private PhysicalBridge interactive, subscription;
 
@@ -54,7 +54,7 @@ public ServerEndPoint(ConnectionMultiplexer multiplexer, EndPoint endpoint)
             slaveReadOnly = true;
             isSlave = false;
             databases = 0;
-            maxMissedHeartbeats = ComputeBeatsFromMilliseconds(config.KeepAlive);
+            writeEverySeconds = config.KeepAlive;
             interactive = CreateBridge(ConnectionType.Interactive);
             serverType = ServerType.Standalone;
         }
@@ -78,7 +78,7 @@ public bool IsConnected
 
         public bool IsSlave { get { return isSlave; } set { SetConfig(ref isSlave, value); } }
 
-        public int MaxMissedHeartbeats { get { return maxMissedHeartbeats; } set { SetConfig(ref maxMissedHeartbeats, value); } }
+        public int WriteEverySeconds { get { return writeEverySeconds; } set { SetConfig(ref writeEverySeconds, value); } }
 
         public long OperationCount
         {
@@ -357,11 +357,6 @@ internal void ReportNextFailure()
             if (tmp != null) tmp.ReportNextFailure();
         }
 
-        internal void SetHeartbeatMilliseconds(int value)
-        {
-            MaxMissedHeartbeats = ComputeBeatsFromMilliseconds(value);
-        }
-
         internal string Summary()
         {
             var sb = new StringBuilder(Format.ToString(endpoint))
@@ -369,9 +364,8 @@ internal string Summary()
             
 
             if (databases > 0) sb.Append("; ").Append(databases).Append(" databases");
-            if (maxMissedHeartbeats > 0)
-                sb.Append("; keep-alive: ").Append(
-                    TimeSpan.FromMilliseconds(maxMissedHeartbeats * ConnectionMultiplexer.MillisecondsPerHeartbeat));
+            if (writeEverySeconds > 0)
+                sb.Append("; keep-alive: ").Append(TimeSpan.FromSeconds(writeEverySeconds));
             var tmp = interactive;
             sb.Append("; int: ").Append(tmp == null ? "n/a" : tmp.ConnectionState.ToString());
             tmp = subscription;
@@ -418,19 +412,6 @@ internal void WriteDirectOrQueueFireAndForget<T>(PhysicalConnection connection,
             }
         }
 
-        private static int ComputeBeatsFromMilliseconds(int value)
-        {
-            if (value > 0)
-            {
-                int beats = value / ConnectionMultiplexer.MillisecondsPerHeartbeat;
-                if (beats == 0) beats = 1;
-                return beats;
-            }
-            else
-            {
-                return -1;
-            }
-        }
         private PhysicalBridge CreateBridge(ConnectionType type)
         {
             multiplexer.Trace(type.ToString());
@@ -491,7 +472,7 @@ void Handshake(PhysicalConnection connection)
                 var configChannel = multiplexer.ConfigurationChangedChannel;
                 if(configChannel != null)
                 {
-                    msg = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.SUBSCRIBE, (RedisValue)configChannel);
+                    msg = Message.Create(-1, CommandFlags.FireAndForget, RedisCommand.SUBSCRIBE, (RedisChannel)configChannel);
                     WriteDirectOrQueueFireAndForget(connection, msg, ResultProcessor.TrackSubscriptions);
                 }
             }
@@ -505,7 +486,7 @@ private void SetConfig<T>(ref T field, T value, [CallerMemberName] string caller
             {
                 multiplexer.Trace(caller + " changed from " + field + " to " + value, "Configuration");
                 field = value;
-                multiplexer.ReconfigureIfNeeded(endpoint, false);
+                multiplexer.ReconfigureIfNeeded(endpoint, false, caller);
             }
         }
 

redis-cli 7000.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7000
\ No newline at end of file

redis-cli 7001.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7001
\ No newline at end of file

redis-cli 7002.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7002
\ No newline at end of file

redis-cli 7003.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7003
\ No newline at end of file

redis-cli 7004.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7004
\ No newline at end of file

redis-cli 7005.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -h cluster -p 7005
\ No newline at end of file

redis-cli master.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -p 6379
\ No newline at end of file

redis-cli secure.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -p 6381
\ No newline at end of file

redis-cli slave.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-cli.exe -p 6380
\ No newline at end of file

redis-server master.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-server.exe master.conf
\ No newline at end of file

redis-server secure.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-server.exe secure.conf
\ No newline at end of file

redis-server slave.cmd

-@packages\Redis-64.2.6.12.1\tools\redis-server.exe slave.conf
\ No newline at end of file