From 7faf143a977f28a52ceeb8947e4f19e4eeb7328d Mon Sep 17 00:00:00 2001 From: Jon Chambers Date: Tue, 16 Jun 2020 18:22:46 -0400 Subject: [PATCH] Subdivide the account database crawler cache experiment and add logging to track down lingering disagreements. --- .../storage/AccountDatabaseCrawlerCache.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java index 2bcc40378..91e982004 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java @@ -18,6 +18,8 @@ package org.whispersystems.textsecuregcm.storage; import io.lettuce.core.ScriptOutputType; import io.lettuce.core.SetArgs; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.whispersystems.textsecuregcm.experiment.Experiment; import org.whispersystems.textsecuregcm.redis.ClusterLuaScript; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; @@ -41,11 +43,14 @@ public class AccountDatabaseCrawlerCache { private static final long LAST_NUMBER_TTL_MS = 86400_000L; + private static final Logger log = LoggerFactory.getLogger(AccountDatabaseCrawlerCache.class); + private final ReplicatedJedisPool jedisPool; private final FaultTolerantRedisCluster cacheCluster; private final LuaScript unlockScript; private final ClusterLuaScript unlockClusterScript; - private final Experiment redisClusterExperiment = new Experiment("RedisCluster", "AccountDatabaseCrawlerCache"); + private final Experiment isAcceleratedExperiment = new Experiment("RedisCluster", "AccountDatabaseCrawlerCache", "isAccelerated"); + private final Experiment getLastUuidExperiment = new Experiment("RedisCluster", "AccountDatabaseCrawlerCache", "getLastUuid"); public AccountDatabaseCrawlerCache(ReplicatedJedisPool jedisPool, FaultTolerantRedisCluster cacheCluster) throws IOException { this.jedisPool = jedisPool; @@ -64,7 +69,7 @@ public class AccountDatabaseCrawlerCache { public boolean isAccelerated() { try (Jedis jedis = jedisPool.getWriteResource()) { final String accelerated = jedis.get(ACCELERATE_KEY); - redisClusterExperiment.compareSupplierResult(accelerated, () -> cacheCluster.withReadCluster(connection -> connection.sync().get(ACCELERATE_KEY))); + isAcceleratedExperiment.compareSupplierResult(accelerated, () -> cacheCluster.withReadCluster(connection -> connection.sync().get(ACCELERATE_KEY))); return "1".equals(accelerated); } @@ -87,13 +92,18 @@ public class AccountDatabaseCrawlerCache { List keys = Arrays.asList(ACTIVE_WORKER_KEY.getBytes()); List args = Arrays.asList(workerId.getBytes()); unlockScript.execute(keys, args); - unlockClusterScript.execute(List.of(ACTIVE_WORKER_KEY), List.of(workerId)); + + try { + unlockClusterScript.execute(List.of(ACTIVE_WORKER_KEY), List.of(workerId)); + } catch (Exception e) { + log.warn("Failed to execute clustered unlock script", e); + } } public Optional getLastUuid() { try (Jedis jedis = jedisPool.getWriteResource()) { String lastUuidString = jedis.get(LAST_UUID_KEY); - redisClusterExperiment.compareSupplierResult(lastUuidString, () -> cacheCluster.withWriteCluster(connection -> connection.sync().get(LAST_UUID_KEY))); + getLastUuidExperiment.compareSupplierResult(lastUuidString, () -> cacheCluster.withWriteCluster(connection -> connection.sync().get(LAST_UUID_KEY))); if (lastUuidString == null) return Optional.empty(); else return Optional.of(UUID.fromString(lastUuidString));