MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use per-shard circuit breakers for additional Redis clusters

This commit is contained in:
Chris Eager 2024-04-12 11:34:28 -05:00 committed by Chris Eager
parent be6f4e38b8
commit fc1f471369
5 changed files with 35 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
View File

@ -421,18 +421,18 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
ConnectionEventLogger.logConnectionEvents(redisClientResources); ConnectionEventLogger.logConnectionEvents(redisClientResources);
FaultTolerantRedisCluster cacheCluster = new ClusterFaultTolerantRedisCluster("main_cache_cluster", FaultTolerantRedisCluster cacheCluster = new ShardFaultTolerantRedisCluster("main_cache",
config.getCacheClusterConfiguration(), redisClientResources); config.getCacheClusterConfiguration(), redisClientResourcesBuilder);
FaultTolerantRedisCluster messagesCluster = new ClusterFaultTolerantRedisCluster("messages_cluster", FaultTolerantRedisCluster messagesCluster = new ClusterFaultTolerantRedisCluster("messages_cluster",
config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResources); config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResources);
FaultTolerantRedisCluster clientPresenceCluster = new ClusterFaultTolerantRedisCluster("client_presence_cluster", FaultTolerantRedisCluster clientPresenceCluster = new ShardFaultTolerantRedisCluster("client_presence",
config.getClientPresenceClusterConfiguration(), redisClientResources); config.getClientPresenceClusterConfiguration(), redisClientResourcesBuilder);
FaultTolerantRedisCluster metricsCluster = new ShardFaultTolerantRedisCluster("metrics", FaultTolerantRedisCluster metricsCluster = new ShardFaultTolerantRedisCluster("metrics",
config.getMetricsClusterConfiguration(), redisClientResourcesBuilder); config.getMetricsClusterConfiguration(), redisClientResourcesBuilder);
FaultTolerantRedisCluster pushSchedulerCluster = new ClusterFaultTolerantRedisCluster("push_scheduler", FaultTolerantRedisCluster pushSchedulerCluster = new ShardFaultTolerantRedisCluster("push_scheduler",
config.getPushSchedulerCluster(), redisClientResources); config.getPushSchedulerCluster(), redisClientResourcesBuilder);
FaultTolerantRedisCluster rateLimitersCluster = new ClusterFaultTolerantRedisCluster("rate_limiters", FaultTolerantRedisCluster rateLimitersCluster = new ShardFaultTolerantRedisCluster("rate_limiters",
config.getRateLimitersCluster(), redisClientResources); config.getRateLimitersCluster(), redisClientResourcesBuilder);
final BlockingQueue<Runnable> keyspaceNotificationDispatchQueue = new ArrayBlockingQueue<>(100_000); final BlockingQueue<Runnable> keyspaceNotificationDispatchQueue = new ArrayBlockingQueue<>(100_000);
Metrics.gaugeCollectionSize(name(getClass(), "keyspaceNotificationDispatchQueueSize"), Collections.emptyList(), Metrics.gaugeCollectionSize(name(getClass(), "keyspaceNotificationDispatchQueueSize"), Collections.emptyList(),

16
service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignUsernameCommand.java
View File

@ -31,6 +31,7 @@ import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery2Controll
import org.whispersystems.textsecuregcm.push.ClientPresenceManager; import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.redis.ShardFaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.Account;
@ -106,10 +107,11 @@ public class AssignUsernameCommand extends EnvironmentCommand<WhisperServerConfi
configuration.getAppConfig().getConfigurationName(), DynamicConfiguration.class, dynamicConfigurationExecutor); configuration.getAppConfig().getConfigurationName(), DynamicConfiguration.class, dynamicConfigurationExecutor);
dynamicConfigurationManager.start(); dynamicConfigurationManager.start();
ClientResources redisClusterClientResources = ClientResources.builder().build(); final ClientResources.Builder redisClientResourcesBuilder = ClientResources.builder();
final ClientResources redisClusterClientResources = redisClientResourcesBuilder.build();
FaultTolerantRedisCluster cacheCluster = new ClusterFaultTolerantRedisCluster("main_cache_cluster", FaultTolerantRedisCluster cacheCluster = new ShardFaultTolerantRedisCluster("main_cache_cluster",
configuration.getCacheClusterConfiguration(), redisClusterClientResources); configuration.getCacheClusterConfiguration(), redisClientResourcesBuilder);
Scheduler messageDeliveryScheduler = Schedulers.fromExecutorService( Scheduler messageDeliveryScheduler = Schedulers.fromExecutorService(
environment.lifecycle().executorService("messageDelivery-%d").maxThreads(4) environment.lifecycle().executorService("messageDelivery-%d").maxThreads(4)
@ -179,10 +181,10 @@ public class AssignUsernameCommand extends EnvironmentCommand<WhisperServerConfi
FaultTolerantRedisCluster messageReadDeleteCluster = new ClusterFaultTolerantRedisCluster( FaultTolerantRedisCluster messageReadDeleteCluster = new ClusterFaultTolerantRedisCluster(
"message_read_delete_cluster", "message_read_delete_cluster",
configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources); configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
FaultTolerantRedisCluster clientPresenceCluster = new ClusterFaultTolerantRedisCluster("client_presence_cluster", FaultTolerantRedisCluster clientPresenceCluster = new ShardFaultTolerantRedisCluster("client_presence",
configuration.getClientPresenceClusterConfiguration(), redisClusterClientResources); configuration.getClientPresenceClusterConfiguration(), redisClientResourcesBuilder);
FaultTolerantRedisCluster rateLimitersCluster = new ClusterFaultTolerantRedisCluster("rate_limiters", FaultTolerantRedisCluster rateLimitersCluster = new ShardFaultTolerantRedisCluster("rate_limiters",
configuration.getRateLimitersCluster(), redisClusterClientResources); configuration.getRateLimitersCluster(), redisClientResourcesBuilder);
SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client( SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(
secureValueRecoveryCredentialsGenerator, secureValueRecoveryExecutor, secureValueRecoveryServiceRetryExecutor, secureValueRecoveryCredentialsGenerator, secureValueRecoveryExecutor, secureValueRecoveryServiceRetryExecutor,
configuration.getSvr2Configuration()); configuration.getSvr2Configuration());

20
service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java
View File

@ -31,6 +31,7 @@ import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.push.ClientPresenceManager; import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.redis.ShardFaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
import org.whispersystems.textsecuregcm.storage.AccountLockManager; import org.whispersystems.textsecuregcm.storage.AccountLockManager;
@ -67,7 +68,7 @@ record CommandDependencies(
ClientPresenceManager clientPresenceManager, ClientPresenceManager clientPresenceManager,
KeysManager keysManager, KeysManager keysManager,
FaultTolerantRedisCluster cacheCluster, FaultTolerantRedisCluster cacheCluster,
ClientResources redisClusterClientResources, ClientResources.Builder redisClusterClientResourcesBuilder,
BackupManager backupManager, BackupManager backupManager,
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) { DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
@ -89,10 +90,11 @@ record CommandDependencies(
MetricsUtil.configureRegistries(configuration, environment, dynamicConfigurationManager); MetricsUtil.configureRegistries(configuration, environment, dynamicConfigurationManager);
ClientResources redisClusterClientResources = ClientResources.builder().build(); final ClientResources.Builder redisClientResourcesBuilder = ClientResources.builder();
ClientResources redisClusterClientResources = redisClientResourcesBuilder.build();
FaultTolerantRedisCluster cacheCluster = new ClusterFaultTolerantRedisCluster("main_cache_cluster", FaultTolerantRedisCluster cacheCluster = new ShardFaultTolerantRedisCluster("main_cache",
configuration.getCacheClusterConfiguration(), redisClusterClientResources); configuration.getCacheClusterConfiguration(), redisClientResourcesBuilder);
ScheduledExecutorService recurringJobExecutor = environment.lifecycle() ScheduledExecutorService recurringJobExecutor = environment.lifecycle()
.scheduledExecutorService(name(name, "recurringJob-%d")).threads(2).build(); .scheduledExecutorService(name(name, "recurringJob-%d")).threads(2).build();
@ -167,10 +169,10 @@ record CommandDependencies(
FaultTolerantRedisCluster messageReadDeleteCluster = new ClusterFaultTolerantRedisCluster( FaultTolerantRedisCluster messageReadDeleteCluster = new ClusterFaultTolerantRedisCluster(
"message_read_delete_cluster", "message_read_delete_cluster",
configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources); configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
FaultTolerantRedisCluster clientPresenceCluster = new ClusterFaultTolerantRedisCluster("client_presence_cluster", FaultTolerantRedisCluster clientPresenceCluster = new ShardFaultTolerantRedisCluster("client_presence",
configuration.getClientPresenceClusterConfiguration(), redisClusterClientResources); configuration.getClientPresenceClusterConfiguration(), redisClientResourcesBuilder);
FaultTolerantRedisCluster rateLimitersCluster = new ClusterFaultTolerantRedisCluster("rate_limiters", FaultTolerantRedisCluster rateLimitersCluster = new ShardFaultTolerantRedisCluster("rate_limiters",
configuration.getRateLimitersCluster(), redisClusterClientResources); configuration.getRateLimitersCluster(), redisClientResourcesBuilder);
SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client( SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(
secureValueRecoveryCredentialsGenerator, secureValueRecoveryServiceExecutor, secureValueRecoveryCredentialsGenerator, secureValueRecoveryServiceExecutor,
secureValueRecoveryServiceRetryExecutor, secureValueRecoveryServiceRetryExecutor,
@ -231,7 +233,7 @@ record CommandDependencies(
clientPresenceManager, clientPresenceManager,
keys, keys,
cacheCluster, cacheCluster,
redisClusterClientResources, redisClientResourcesBuilder,
backupManager, backupManager,
dynamicConfigurationManager dynamicConfigurationManager
); );

6
service/src/main/java/org/whispersystems/textsecuregcm/workers/ScheduledApnPushNotificationSenderServiceCommand.java
View File

@ -19,8 +19,8 @@ import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil; import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.push.APNSender; import org.whispersystems.textsecuregcm.push.APNSender;
import org.whispersystems.textsecuregcm.push.ApnPushNotificationScheduler; import org.whispersystems.textsecuregcm.push.ApnPushNotificationScheduler;
import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.redis.ShardFaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.util.logging.UncaughtExceptionHandler; import org.whispersystems.textsecuregcm.util.logging.UncaughtExceptionHandler;
public class ScheduledApnPushNotificationSenderServiceCommand extends ServerCommand<WhisperServerConfiguration> { public class ScheduledApnPushNotificationSenderServiceCommand extends ServerCommand<WhisperServerConfiguration> {
@ -64,8 +64,8 @@ public class ScheduledApnPushNotificationSenderServiceCommand extends ServerComm
}); });
} }
final FaultTolerantRedisCluster pushSchedulerCluster = new ClusterFaultTolerantRedisCluster("push_scheduler", final FaultTolerantRedisCluster pushSchedulerCluster = new ShardFaultTolerantRedisCluster("push_scheduler",
configuration.getPushSchedulerCluster(), deps.redisClusterClientResources()); configuration.getPushSchedulerCluster(), deps.redisClusterClientResourcesBuilder());
final ExecutorService apnSenderExecutor = environment.lifecycle().executorService(name(getClass(), "apnSender-%d")) final ExecutorService apnSenderExecutor = environment.lifecycle().executorService(name(getClass(), "apnSender-%d"))
.maxThreads(1).minThreads(1).build(); .maxThreads(1).minThreads(1).build();

7
service/src/test/java/org/whispersystems/textsecuregcm/redis/RedisClusterExtension.java
View File

@ -12,8 +12,8 @@ import io.lettuce.core.RedisException;
import io.lettuce.core.RedisURI; import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.api.sync.RedisCommands; import io.lettuce.core.api.sync.RedisCommands;
import io.lettuce.core.cluster.RedisClusterClient;
import io.lettuce.core.cluster.SlotHash; import io.lettuce.core.cluster.SlotHash;
import io.lettuce.core.resource.ClientResources;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.ServerSocket; import java.net.ServerSocket;
@ -81,8 +81,9 @@ public class RedisClusterExtension implements BeforeAllCallback, BeforeEachCallb
@Override @Override
public void beforeEach(final ExtensionContext context) throws Exception { public void beforeEach(final ExtensionContext context) throws Exception {
redisCluster = new ClusterFaultTolerantRedisCluster("test-cluster", redisCluster = new ShardFaultTolerantRedisCluster("test-cluster",
RedisClusterClient.create(getRedisURIs()), ClientResources.builder(),
getRedisURIs(),
timeout, timeout,
new CircuitBreakerConfiguration(), new CircuitBreakerConfiguration(),
retryConfiguration); retryConfiguration);