Commit 9c5834ea authored by Nick Craver's avatar Nick Craver

Test: Split failover test servers out

This could be a terrible idea, but I hope it helps isolate failures and increase stability a bit.
parent 0717a2e5
port 6382
repl-diskless-sync yes
repl-diskless-sync-delay 0
databases 2000
maxmemory 2gb
dir "../Temp"
appendonly no
dbfilename "master-6382.rdb"
save ""
\ No newline at end of file
port 6383
slaveof 127.0.0.1 6382
repl-diskless-sync yes
repl-diskless-sync-delay 0
databases 2000
maxmemory 2gb
appendonly no
dir "../Temp"
dbfilename "slave-6383.rdb"
save ""
\ No newline at end of file
@echo off @echo off
echo Starting Redis servers for testing... echo Starting Redis servers for testing...
call %~dp0\start-basic.cmd call %~dp0\start-basic.cmd
call %~dp0\start-failover.cmd
call %~dp0\start-cluster.cmd call %~dp0\start-cluster.cmd
call %~dp0\start-sentinel.cmd call %~dp0\start-sentinel.cmd
echo Servers started (minimized). echo Servers started (minimized).
......
@echo off
echo Starting Failover:
pushd %~dp0\Failover
echo Master: 6382
@start "Redis (Failover Master): 6382" /min ..\3.0.503\redis-server.exe master-6382.conf
echo Slave: 6383
@start "Redis (Failover Slave): 6383" /min ..\3.0.503\redis-server.exe slave-6383.conf
popd
\ No newline at end of file
...@@ -14,40 +14,6 @@ public class Config : TestBase ...@@ -14,40 +14,6 @@ public class Config : TestBase
{ {
public Config(ITestOutputHelper output) : base (output) { } public Config(ITestOutputHelper output) : base (output) { }
[Fact]
public async Task VerifyReceiveConfigChangeBroadcast()
{
var config = GetConfiguration();
using (var sender = Create(allowAdmin: true))
using (var receiver = Create(syncTimeout: 2000))
{
int total = 0;
receiver.ConfigurationChangedBroadcast += (s, a) =>
{
Output.WriteLine("Config changed: " + (a.EndPoint == null ? "(none)" : a.EndPoint.ToString()));
Interlocked.Increment(ref total);
};
// send a reconfigure/reconnect message
long count = sender.PublishReconfigure();
GetServer(receiver).Ping();
GetServer(receiver).Ping();
await Task.Delay(10).ConfigureAwait(false);
Assert.True(count == -1 || count >= 2, "subscribers");
Assert.True(Interlocked.CompareExchange(ref total, 0, 0) >= 1, "total (1st)");
Interlocked.Exchange(ref total, 0);
// and send a second time via a re-master operation
var server = GetServer(sender);
if (server.IsSlave) Skip.Inconclusive("didn't expect a slave");
server.MakeMaster(ReplicationChangeOptions.Broadcast);
await Task.Delay(100).ConfigureAwait(false);
GetServer(receiver).Ping();
GetServer(receiver).Ping();
Assert.True(Interlocked.CompareExchange(ref total, 0, 0) >= 1, "total (2nd)");
}
}
[Fact] [Fact]
public void TalkToNonsenseServer() public void TalkToNonsenseServer()
{ {
......
...@@ -50,6 +50,15 @@ public class Config ...@@ -50,6 +50,15 @@ public class Config
public string SecurePassword { get; set; } = "changeme"; public string SecurePassword { get; set; } = "changeme";
public string SecureServerAndPort => SecureServer + ":" + SecurePort.ToString(); public string SecureServerAndPort => SecureServer + ":" + SecurePort.ToString();
// Separate servers for failover tests, so they don't wreak havoc on all others
public string FailoverMasterServer { get; set; } = "127.0.0.1";
public int FailoverMasterPort { get; set; } = 6382;
public string FailoverMasterServerAndPort => FailoverMasterServer + ":" + FailoverMasterPort.ToString();
public string FailoverSlaveServer { get; set; } = "127.0.0.1";
public int FailoverSlavePort { get; set; } = 6383;
public string FailoverSlaveServerAndPort => FailoverSlaveServer + ":" + FailoverSlavePort.ToString();
public string IPv4Server { get; set; } = "127.0.0.1"; public string IPv4Server { get; set; } = "127.0.0.1";
public int IPv4Port { get; set; } = 6379; public int IPv4Port { get; set; } = 6379;
public string IPv6Server { get; set; } = "::1"; public string IPv6Server { get; set; } = "::1";
......
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Threading.Tasks;
using Xunit; using Xunit;
using Xunit.Abstractions; using Xunit.Abstractions;
namespace StackExchange.Redis.Tests namespace StackExchange.Redis.Tests
{ {
[Collection(NonParallelCollection.Name)]
public class MultiMaster : TestBase public class MultiMaster : TestBase
{ {
protected override string GetConfiguration() => protected override string GetConfiguration() =>
TestConfig.Current.MasterServerAndPort + "," + TestConfig.Current.SecureServerAndPort + ",password=" + TestConfig.Current.SecurePassword; TestConfig.Current.MasterServerAndPort + "," + TestConfig.Current.SecureServerAndPort + ",password=" + TestConfig.Current.SecurePassword;
public MultiMaster(ITestOutputHelper output) : base (output) { } public MultiMaster(ITestOutputHelper output) : base (output) { }
[Fact] [Fact]
...@@ -20,8 +17,7 @@ public void CannotFlushSlave() ...@@ -20,8 +17,7 @@ public void CannotFlushSlave()
{ {
var ex = Assert.Throws<RedisCommandException>(() => var ex = Assert.Throws<RedisCommandException>(() =>
{ {
ConfigurationOptions config = GetMasterSlaveConfig(); using (var conn = ConnectionMultiplexer.Connect(TestConfig.Current.SlaveServerAndPort + ",allowAdmin=true"))
using (var conn = ConnectionMultiplexer.Connect(config))
{ {
var servers = conn.GetEndPoints().Select(e => conn.GetServer(e)); var servers = conn.GetEndPoints().Select(e => conn.GetServer(e));
var slave = servers.FirstOrDefault(x => x.IsSlave); var slave = servers.FirstOrDefault(x => x.IsSlave);
...@@ -32,107 +28,6 @@ public void CannotFlushSlave() ...@@ -32,107 +28,6 @@ public void CannotFlushSlave()
Assert.Equal("Command cannot be issued to a slave: FLUSHDB", ex.Message); Assert.Equal("Command cannot be issued to a slave: FLUSHDB", ex.Message);
} }
[Fact]
public async Task DeslaveGoesToPrimary()
{
ConfigurationOptions config = GetMasterSlaveConfig();
using (var conn = ConnectionMultiplexer.Connect(config))
{
var primary = conn.GetServer(TestConfig.Current.MasterServerAndPort);
var secondary = conn.GetServer(TestConfig.Current.SlaveServerAndPort);
primary.Ping();
secondary.Ping();
primary.MakeMaster(ReplicationChangeOptions.SetTiebreaker);
secondary.MakeMaster(ReplicationChangeOptions.None);
await Task.Delay(100).ConfigureAwait(false);
primary.Ping();
secondary.Ping();
using (var writer = new StringWriter())
{
conn.Configure(writer);
string log = writer.ToString();
Assert.True(log.Contains("tie-break is unanimous at " + TestConfig.Current.MasterServerAndPort), "unanimous");
}
// k, so we know everyone loves 6379; is that what we get?
var db = conn.GetDatabase();
RedisKey key = Me();
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferSlave));
var ex = Assert.Throws<RedisConnectionException>(() => db.IdentifyEndpoint(key, CommandFlags.DemandSlave));
Assert.StartsWith("No connection is available to service this operation: EXISTS " + Me(), ex.Message);
Writer.WriteLine("Invoking MakeMaster()...");
primary.MakeMaster(ReplicationChangeOptions.Broadcast | ReplicationChangeOptions.EnslaveSubordinates | ReplicationChangeOptions.SetTiebreaker, Writer);
Writer.WriteLine("Finished MakeMaster() call.");
await Task.Delay(100).ConfigureAwait(false);
Writer.WriteLine("Invoking Ping() (post-master)");
primary.Ping();
secondary.Ping();
Writer.WriteLine("Finished Ping() (post-master)");
Assert.True(primary.IsConnected, $"{primary.EndPoint} is not connected.");
Assert.True(secondary.IsConnected, $"{secondary.EndPoint} is not connected.");
Writer.WriteLine($"{primary.EndPoint}: {primary.ServerType}, Mode: {(primary.IsSlave ? "Slave" : "Master")}");
Writer.WriteLine($"{secondary.EndPoint}: {secondary.ServerType}, Mode: {(secondary.IsSlave ? "Slave" : "Master")}");
// Create a separate multiplexer with a valid view of the world to distinguish between failures of
// server topology changes from failures to recognize those changes
Writer.WriteLine("Connecting to secondary validation connection.");
using (var conn2 = ConnectionMultiplexer.Connect(config))
{
var primary2 = conn.GetServer(TestConfig.Current.MasterServerAndPort);
var secondary2 = conn.GetServer(TestConfig.Current.SlaveServerAndPort);
Writer.WriteLine($"Check: {primary2.EndPoint}: {primary2.ServerType}, Mode: {(primary2.IsSlave ? "Slave" : "Master")}");
Writer.WriteLine($"Check: {secondary2.EndPoint}: {secondary2.ServerType}, Mode: {(secondary2.IsSlave ? "Slave" : "Master")}");
Assert.False(primary2.IsSlave, $"{primary2.EndPoint} should be a master (verification connection).");
Assert.True(secondary2.IsSlave, $"{secondary2.EndPoint} should be a slave (verification connection).");
var db2 = conn.GetDatabase();
Assert.Equal(primary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(secondary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.PreferSlave));
Assert.Equal(secondary2.EndPoint, db2.IdentifyEndpoint(key, CommandFlags.DemandSlave));
}
Assert.False(primary.IsSlave, $"{primary.EndPoint} should be a master.");
Assert.True(secondary.IsSlave, $"{secondary.EndPoint} should be a slave.");
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferMaster));
Assert.Equal(primary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandMaster));
Assert.Equal(secondary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.PreferSlave));
Assert.Equal(secondary.EndPoint, db.IdentifyEndpoint(key, CommandFlags.DemandSlave));
}
}
private static ConfigurationOptions GetMasterSlaveConfig()
{
return new ConfigurationOptions
{
AllowAdmin = true,
SyncTimeout = 100000,
EndPoints =
{
{ TestConfig.Current.MasterServer, TestConfig.Current.MasterPort },
{ TestConfig.Current.SlaveServer, TestConfig.Current.SlavePort },
}
};
}
[Fact] [Fact]
public void TestMultiNoTieBreak() public void TestMultiNoTieBreak()
{ {
......
...@@ -7,7 +7,6 @@ ...@@ -7,7 +7,6 @@
namespace StackExchange.Redis.Tests namespace StackExchange.Redis.Tests
{ {
[Collection(NonParallelCollection.Name)]
public class PubSub : TestBase public class PubSub : TestBase
{ {
public PubSub(ITestOutputHelper output) : base(output) { } public PubSub(ITestOutputHelper output) : base(output) { }
...@@ -194,11 +193,6 @@ private static void Ping(ConnectionMultiplexer muxer, IServer pub, ISubscriber s ...@@ -194,11 +193,6 @@ private static void Ping(ConnectionMultiplexer muxer, IServer pub, ISubscriber s
} }
} }
//protected override string GetConfiguration()
//{
// return TestConfig.Current.MasterServer + ":" + TestConfig.Current.MasterPort;
//}
[Theory] [Theory]
[InlineData(true)] [InlineData(true)]
[InlineData(false)] [InlineData(false)]
......
...@@ -13,6 +13,10 @@ install: ...@@ -13,6 +13,10 @@ install:
redis-server.exe --service-install --service-name "redis-6381" "..\Basic\secure-6381.conf" redis-server.exe --service-install --service-name "redis-6381" "..\Basic\secure-6381.conf"
redis-server.exe --service-install --service-name "redis-6382" "..\Failover\master-6382.conf"
redis-server.exe --service-install --service-name "redis-6382" "..\Failover\slave-6383.conf"
redis-server.exe --service-install --service-name "redis-7000" "..\Cluster\cluster-7000.conf" --dir "..\Cluster" redis-server.exe --service-install --service-name "redis-7000" "..\Cluster\cluster-7000.conf" --dir "..\Cluster"
redis-server.exe --service-install --service-name "redis-7001" "..\Cluster\cluster-7001.conf" --dir "..\Cluster" redis-server.exe --service-install --service-name "redis-7001" "..\Cluster\cluster-7001.conf" --dir "..\Cluster"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment