MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert Lettuce connection pooling.

This commit is contained in:
Jon Chambers 2020-08-18 15:55:01 -04:00 committed by Jon Chambers
parent 809750b995
commit 42ea7a9814
5 changed files with 22 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
service/src/main/java/org/whispersystems/textsecuregcm/configuration/RedisClusterConfiguration.java
View File

@ -23,10 +23,6 @@ public class RedisClusterConfiguration {
@Valid @Valid
private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration(); private CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
@JsonProperty
@Valid
private RedisConnectionPoolConfiguration connectionPool = new RedisConnectionPoolConfiguration();
public List<String> getUrls() { public List<String> getUrls() {
return urls; return urls;
} }
@ -38,8 +34,4 @@ public class RedisClusterConfiguration {
public CircuitBreakerConfiguration getCircuitBreakerConfiguration() { public CircuitBreakerConfiguration getCircuitBreakerConfiguration() {
return circuitBreaker; return circuitBreaker;
} }
public RedisConnectionPoolConfiguration getConnectionPoolConfiguration() {
return connectionPool;
}
} }

26
service/src/main/java/org/whispersystems/textsecuregcm/configuration/RedisConnectionPoolConfiguration.java
View File

@ -1,26 +0,0 @@
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import java.time.Duration;
public class RedisConnectionPoolConfiguration {
@JsonProperty
@Min(1)
private int poolSize = 16;
@JsonProperty
@NotNull
private Duration maxWait = Duration.ofSeconds(10);
public int getPoolSize() {
return poolSize;
}
public Duration getMaxWait() {
return maxWait;
}
}

63
service/src/main/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisCluster.java
View File

@ -10,14 +10,10 @@ import io.lettuce.core.cluster.RedisClusterClient;
import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection; import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
import io.lettuce.core.codec.ByteArrayCodec; import io.lettuce.core.codec.ByteArrayCodec;
import io.lettuce.core.support.ConnectionPoolSupport;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration; import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
import org.whispersystems.textsecuregcm.configuration.RedisConnectionPoolConfiguration;
import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil; import org.whispersystems.textsecuregcm.util.CircuitBreakerUtil;
import org.whispersystems.textsecuregcm.util.Constants; import org.whispersystems.textsecuregcm.util.Constants;
@ -32,7 +28,7 @@ import static com.codahale.metrics.MetricRegistry.name;
/** /**
* A fault-tolerant access manager for a Redis cluster. A fault-tolerant Redis cluster provides managed, * A fault-tolerant access manager for a Redis cluster. A fault-tolerant Redis cluster provides managed,
* circuit-breaker-protected access to a pool of connections. * circuit-breaker-protected access to connections.
*/ */
public class FaultTolerantRedisCluster { public class FaultTolerantRedisCluster {
@ -40,15 +36,14 @@ public class FaultTolerantRedisCluster {
private final RedisClusterClient clusterClient; private final RedisClusterClient clusterClient;
private final GenericObjectPool<StatefulRedisClusterConnection<String, String>> stringConnectionPool; private final StatefulRedisClusterConnection<String, String> stringConnection;
private final GenericObjectPool<StatefulRedisClusterConnection<byte[], byte[]>> binaryConnectionPool; private final StatefulRedisClusterConnection<byte[], byte[]> binaryConnection;
private final List<StatefulRedisClusterPubSubConnection<?, ?>> pubSubConnections = new ArrayList<>(); private final List<StatefulRedisClusterPubSubConnection<?, ?>> pubSubConnections = new ArrayList<>();
private final CircuitBreakerConfiguration circuitBreakerConfiguration; private final CircuitBreakerConfiguration circuitBreakerConfiguration;
private final CircuitBreaker circuitBreaker; private final CircuitBreaker circuitBreaker;
private final Timer acquireAndExecuteTimer;
private final Timer executeTimer; private final Timer executeTimer;
private static final Logger log = LoggerFactory.getLogger(FaultTolerantRedisCluster.class); private static final Logger log = LoggerFactory.getLogger(FaultTolerantRedisCluster.class);
@ -57,27 +52,18 @@ public class FaultTolerantRedisCluster {
this(name, this(name,
RedisClusterClient.create(clusterConfiguration.getUrls().stream().map(RedisURI::create).collect(Collectors.toList())), RedisClusterClient.create(clusterConfiguration.getUrls().stream().map(RedisURI::create).collect(Collectors.toList())),
clusterConfiguration.getTimeout(), clusterConfiguration.getTimeout(),
clusterConfiguration.getCircuitBreakerConfiguration(), clusterConfiguration.getCircuitBreakerConfiguration());
clusterConfiguration.getConnectionPoolConfiguration());
} }
@VisibleForTesting @VisibleForTesting
FaultTolerantRedisCluster(final String name, final RedisClusterClient clusterClient, final Duration timeout, final CircuitBreakerConfiguration circuitBreakerConfiguration, final RedisConnectionPoolConfiguration connectionPoolConfiguration) { FaultTolerantRedisCluster(final String name, final RedisClusterClient clusterClient, final Duration timeout, final CircuitBreakerConfiguration circuitBreakerConfiguration) {
this.name = name; this.name = name;
this.clusterClient = clusterClient; this.clusterClient = clusterClient;
this.clusterClient.setDefaultTimeout(timeout); this.clusterClient.setDefaultTimeout(timeout);
@SuppressWarnings("rawtypes") final GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig(); this.stringConnection = clusterClient.connect();
poolConfig.setMaxIdle(connectionPoolConfiguration.getPoolSize()); this.binaryConnection = clusterClient.connect(ByteArrayCodec.INSTANCE);
poolConfig.setMaxTotal(connectionPoolConfiguration.getPoolSize());
poolConfig.setMaxWaitMillis(connectionPoolConfiguration.getMaxWait().toMillis());
//noinspection unchecked
this.stringConnectionPool = ConnectionPoolSupport.createGenericObjectPool(clusterClient::connect, poolConfig);
//noinspection unchecked
this.binaryConnectionPool = ConnectionPoolSupport.createGenericObjectPool(() -> clusterClient.connect(ByteArrayCodec.INSTANCE), poolConfig);
this.circuitBreakerConfiguration = circuitBreakerConfiguration; this.circuitBreakerConfiguration = circuitBreakerConfiguration;
this.circuitBreaker = CircuitBreaker.of(name + "-read", circuitBreakerConfiguration.toCircuitBreakerConfig()); this.circuitBreaker = CircuitBreaker.of(name + "-read", circuitBreakerConfiguration.toCircuitBreakerConfig());
@ -88,13 +74,12 @@ public class FaultTolerantRedisCluster {
final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME); final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
this.acquireAndExecuteTimer = metricRegistry.timer(name(getClass(), name, "acquireConnectionAndExecute")); this.executeTimer = metricRegistry.timer(name(getClass(), name, "execute"));
this.executeTimer = metricRegistry.timer(name(getClass(), name, "execute"));
} }
void shutdown() { void shutdown() {
stringConnectionPool.close(); stringConnection.close();
binaryConnectionPool.close(); binaryConnection.close();
for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) { for (final StatefulRedisClusterPubSubConnection<?, ?> pubSubConnection : pubSubConnections) {
pubSubConnection.close(); pubSubConnection.close();
@ -104,30 +89,26 @@ public class FaultTolerantRedisCluster {
} }
public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) { public void useCluster(final Consumer<StatefulRedisClusterConnection<String, String>> consumer) {
acceptPooledConnection(stringConnectionPool, consumer); useConnection(stringConnection, consumer);
} }
public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) { public <T> T withCluster(final Function<StatefulRedisClusterConnection<String, String>, T> function) {
return applyToPooledConnection(stringConnectionPool, function); return withConnection(stringConnection, function);
} }
public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) { public void useBinaryCluster(final Consumer<StatefulRedisClusterConnection<byte[], byte[]>> consumer) {
acceptPooledConnection(binaryConnectionPool, consumer); useConnection(binaryConnection, consumer);
} }
public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) { public <T> T withBinaryCluster(final Function<StatefulRedisClusterConnection<byte[], byte[]>, T> function) {
return applyToPooledConnection(binaryConnectionPool, function); return withConnection(binaryConnection, function);
} }
private <K, V> void acceptPooledConnection(final GenericObjectPool<StatefulRedisClusterConnection<K, V>> pool, final Consumer<StatefulRedisClusterConnection<K, V>> consumer) { private <K, V> void useConnection(final StatefulRedisClusterConnection<K, V> connection, final Consumer<StatefulRedisClusterConnection<K, V>> consumer) {
try { try {
circuitBreaker.executeCheckedRunnable(() -> { circuitBreaker.executeCheckedRunnable(() -> {
try (final Timer.Context ignored = acquireAndExecuteTimer.time(); try (final Timer.Context ignored = executeTimer.time()) {
final StatefulRedisClusterConnection<K, V> connection = pool.borrowObject()) { consumer.accept(connection);
try (final Timer.Context ignored2 = executeTimer.time()) {
consumer.accept(connection);
}
} }
}); });
} catch (final Throwable t) { } catch (final Throwable t) {
@ -141,15 +122,11 @@ public class FaultTolerantRedisCluster {
} }
} }
private <T, K, V> T applyToPooledConnection(final GenericObjectPool<StatefulRedisClusterConnection<K, V>> pool, final Function<StatefulRedisClusterConnection<K, V>, T> function) { private <T, K, V> T withConnection(final StatefulRedisClusterConnection<K, V> connection, final Function<StatefulRedisClusterConnection<K, V>, T> function) {
try { try {
return circuitBreaker.executeCheckedSupplier(() -> { return circuitBreaker.executeCheckedSupplier(() -> {
try (final Timer.Context ignored = acquireAndExecuteTimer.time(); try (final Timer.Context ignored = executeTimer.time()) {
final StatefulRedisClusterConnection<K, V> connection = pool.borrowObject()) { return function.apply(connection);
try (final Timer.Context ignored2 = executeTimer.time()) {
return function.apply(connection);
}
} }
}); });
} catch (final Throwable t) { } catch (final Throwable t) {

5
service/src/test/java/org/whispersystems/textsecuregcm/redis/AbstractRedisClusterTest.java
View File

@ -12,8 +12,6 @@ import org.junit.AfterClass;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RedisClusterConfiguration;
import org.whispersystems.textsecuregcm.configuration.RedisConnectionPoolConfiguration;
import org.whispersystems.textsecuregcm.util.RedisClusterUtil; import org.whispersystems.textsecuregcm.util.RedisClusterUtil;
import redis.embedded.RedisServer; import redis.embedded.RedisServer;
@ -62,8 +60,7 @@ public abstract class AbstractRedisClusterTest {
redisCluster = new FaultTolerantRedisCluster("test-cluster", redisCluster = new FaultTolerantRedisCluster("test-cluster",
RedisClusterClient.create(urls.stream().map(RedisURI::create).collect(Collectors.toList())), RedisClusterClient.create(urls.stream().map(RedisURI::create).collect(Collectors.toList())),
Duration.ofSeconds(2), Duration.ofSeconds(2),
new CircuitBreakerConfiguration(), new CircuitBreakerConfiguration());
new RedisConnectionPoolConfiguration());
redisCluster.useCluster(connection -> { redisCluster.useCluster(connection -> {
boolean setAll = false; boolean setAll = false;

3
service/src/test/java/org/whispersystems/textsecuregcm/redis/FaultTolerantRedisClusterTest.java
View File

@ -9,7 +9,6 @@ import io.lettuce.core.cluster.pubsub.StatefulRedisClusterPubSubConnection;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RedisConnectionPoolConfiguration;
import java.time.Duration; import java.time.Duration;
@ -41,7 +40,7 @@ public class FaultTolerantRedisClusterTest {
breakerConfiguration.setRingBufferSizeInClosedState(1); breakerConfiguration.setRingBufferSizeInClosedState(1);
breakerConfiguration.setWaitDurationInOpenStateInSeconds(Integer.MAX_VALUE); breakerConfiguration.setWaitDurationInOpenStateInSeconds(Integer.MAX_VALUE);
faultTolerantCluster = new FaultTolerantRedisCluster("test", clusterClient, Duration.ofSeconds(2), breakerConfiguration, new RedisConnectionPoolConfiguration()); faultTolerantCluster = new FaultTolerantRedisCluster("test", clusterClient, Duration.ofSeconds(2), breakerConfiguration);
} }
@Test @Test