diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index de0ce4b08..84f6ee5bc 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -46,9 +46,7 @@ import java.util.Optional; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; import javax.servlet.FilterRegistration; @@ -62,13 +60,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.whispersystems.dispatch.DispatchManager; import org.whispersystems.textsecuregcm.auth.AccountAuthenticator; -import org.whispersystems.textsecuregcm.auth.WebsocketRefreshApplicationEventListener; import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount; import org.whispersystems.textsecuregcm.auth.CertificateGenerator; import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccountAuthenticator; import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount; import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator; import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator; +import org.whispersystems.textsecuregcm.auth.WebsocketRefreshApplicationEventListener; import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter; import org.whispersystems.textsecuregcm.badges.ProfileBadgeConverter; import org.whispersystems.textsecuregcm.configuration.DirectoryServerConfiguration; @@ -156,9 +154,7 @@ import org.whispersystems.textsecuregcm.storage.AccountCleaner; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener; -import org.whispersystems.textsecuregcm.storage.Accounts; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; -import org.whispersystems.textsecuregcm.storage.AccountsDynamoDbMigrator; import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.ContactDiscoveryWriter; import org.whispersystems.textsecuregcm.storage.DeletedAccounts; @@ -174,11 +170,6 @@ import org.whispersystems.textsecuregcm.storage.MessagePersister; import org.whispersystems.textsecuregcm.storage.MessagesCache; import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesManager; -import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts; -import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts; -import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccountsTableCrawler; -import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts; -import org.whispersystems.textsecuregcm.storage.MigrationRetryAccountsTableCrawler; import org.whispersystems.textsecuregcm.storage.Profiles; import org.whispersystems.textsecuregcm.storage.ProfilesManager; import org.whispersystems.textsecuregcm.storage.PubSubManager; @@ -211,14 +202,12 @@ import org.whispersystems.textsecuregcm.workers.ServerVersionCommand; import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask; import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask; import org.whispersystems.textsecuregcm.workers.SetUserDiscoverabilityCommand; -import org.whispersystems.textsecuregcm.workers.VacuumCommand; import org.whispersystems.textsecuregcm.workers.ZkParamsCommand; import org.whispersystems.websocket.WebSocketResourceProviderFactory; import org.whispersystems.websocket.setup.WebSocketEnvironment; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.s3.S3Client; @@ -228,7 +217,6 @@ public class WhisperServerService extends Application bootstrap) { - bootstrap.addCommand(new VacuumCommand()); bootstrap.addCommand(new DeleteUserCommand()); bootstrap.addCommand(new CertificateCommand()); bootstrap.addCommand(new ZkParamsCommand()); @@ -243,7 +231,6 @@ public class WhisperServerService extends Application("abusedb", "abusedb.xml") { @Override public PooledDataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) { @@ -316,20 +303,9 @@ public class WhisperServerService extends Application()); - - DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(config.getAccountsDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(), - accountsDynamoDbMigrationThreadPool); - DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(config.getDeletedAccountsDynamoDbConfiguration(), software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - DynamoDbClient recentlyDeletedAccountsDynamoDb = DynamoDbFromConfig.client(config.getMigrationDeletedAccountsDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - DynamoDbClient pushChallengeDynamoDbClient = DynamoDbFromConfig.client( config.getPushChallengeDynamoDbConfiguration(), software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); @@ -338,14 +314,6 @@ public class WhisperServerService extends Application listeners; - private final ExecutorService chunkPreReadExecutorService; - - private final DynamicConfigurationManager dynamicConfigurationManager; private AtomicBoolean running = new AtomicBoolean(false); private boolean finished; - // temporary to control behavior during the Postgres → Dynamo transition - private boolean dedicatedDynamoMigrationCrawler; - public AccountDatabaseCrawler(AccountsManager accounts, AccountDatabaseCrawlerCache cache, List listeners, int chunkSize, - long chunkIntervalMs, - ExecutorService chunkPreReadExecutorService, - DynamicConfigurationManager dynamicConfigurationManager) { + long chunkIntervalMs) { this.accounts = accounts; this.chunkSize = chunkSize; this.chunkIntervalMs = chunkIntervalMs; this.workerId = UUID.randomUUID().toString(); this.cache = cache; this.listeners = listeners; - this.chunkPreReadExecutorService = chunkPreReadExecutorService; - this.dynamicConfigurationManager = dynamicConfigurationManager; } @Override @@ -131,25 +120,19 @@ public class AccountDatabaseCrawler implements Managed, Runnable { try (Timer.Context timer = processChunkTimer.time()) { - final boolean useDynamo = !dedicatedDynamoMigrationCrawler && dynamicConfigurationManager.getConfiguration() - .getAccountsDynamoDbMigrationConfiguration() - .isDynamoCrawlerEnabled(); - - final Optional fromUuid = getLastUuid(useDynamo); + final Optional fromUuid = getLastUuid(); if (fromUuid.isEmpty()) { logger.info("Started crawl"); listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart); } - final AccountCrawlChunk chunkAccounts = readChunk(fromUuid, chunkSize, useDynamo); - - primeDatabaseForNextChunkAsync(chunkAccounts.getLastUuid(), chunkSize, useDynamo); + final AccountCrawlChunk chunkAccounts = readChunk(fromUuid, chunkSize); if (chunkAccounts.getAccounts().isEmpty()) { logger.info("Finished crawl"); listeners.forEach(listener -> listener.onCrawlEnd(fromUuid)); - cacheLastUuid(Optional.empty(), useDynamo); + cacheLastUuid(Optional.empty()); cache.setAccelerated(false); } else { logger.info("Processing chunk"); @@ -157,70 +140,42 @@ public class AccountDatabaseCrawler implements Managed, Runnable { for (AccountDatabaseCrawlerListener listener : listeners) { listener.timeAndProcessCrawlChunk(fromUuid, chunkAccounts.getAccounts()); } - cacheLastUuid(chunkAccounts.getLastUuid(), useDynamo); + cacheLastUuid(chunkAccounts.getLastUuid()); } catch (AccountDatabaseCrawlerRestartException e) { - cacheLastUuid(Optional.empty(), useDynamo); + cacheLastUuid(Optional.empty()); cache.setAccelerated(false); } } } } - /** - * This is an optimization based on the observation that cold reads of chunks are slow, but subsequent reads of the - * same chunk (within a few minutes) are fast. We can’t easily store the actual result data, since the next chunk - * might be processed elsewhere, but the time savings are still substantial. - */ - private void primeDatabaseForNextChunkAsync(Optional fromUuid, int chunkSize, boolean useDynamo) { - if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration() - .isCrawlerPreReadNextChunkEnabled()) { - if (!useDynamo && fromUuid.isPresent()) { - chunkPreReadExecutorService.submit(() -> readChunk(fromUuid, chunkSize, false, preReadChunkTimer)); - } - } + private AccountCrawlChunk readChunk(Optional fromUuid, int chunkSize) { + return readChunk(fromUuid, chunkSize, readChunkTimer); } - private AccountCrawlChunk readChunk(Optional fromUuid, int chunkSize, boolean useDynamo) { - return readChunk(fromUuid, chunkSize, useDynamo, readChunkTimer); - } - - private AccountCrawlChunk readChunk(Optional fromUuid, int chunkSize, boolean useDynamo, Timer readTimer) { + private AccountCrawlChunk readChunk(Optional fromUuid, int chunkSize, Timer readTimer) { try (Timer.Context timer = readTimer.time()) { if (fromUuid.isPresent()) { - return useDynamo - ? accounts.getAllFromDynamo(fromUuid.get(), chunkSize) - : accounts.getAllFrom(fromUuid.get(), chunkSize); + return accounts.getAllFromDynamo(fromUuid.get(), chunkSize); } - return useDynamo - ? accounts.getAllFromDynamo(chunkSize) - : accounts.getAllFrom(chunkSize); + return accounts.getAllFromDynamo(chunkSize); } } - private Optional getLastUuid(final boolean useDynamo) { - if (useDynamo) { - return cache.getLastUuidDynamo(); - } else { - return cache.getLastUuid(); - } + private Optional getLastUuid() { + return cache.getLastUuidDynamo(); } - private void cacheLastUuid(final Optional lastUuid, final boolean useDynamo) { - if (useDynamo) { - cache.setLastUuidDynamo(lastUuid); - } else { - cache.setLastUuid(lastUuid); - } - } - - public void setDedicatedDynamoMigrationCrawler(final boolean dedicatedDynamoMigrationCrawler) { - this.dedicatedDynamoMigrationCrawler = dedicatedDynamoMigrationCrawler; + private void cacheLastUuid(final Optional lastUuid) { + cache.setLastUuidDynamo(lastUuid); } private synchronized void sleepWhileRunning(long delayMs) { - if (running.get()) Util.wait(this, delayMs); + if (running.get()) { + Util.wait(this, delayMs); + } } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java index 77c680f14..314aa0f91 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java @@ -27,8 +27,6 @@ public class AccountDatabaseCrawlerCache { private final FaultTolerantRedisCluster cacheCluster; private final ClusterLuaScript unlockClusterScript; - private String prefix = ""; - public AccountDatabaseCrawlerCache(FaultTolerantRedisCluster cacheCluster) throws IOException { this.cacheCluster = cacheCluster; this.unlockClusterScript = ClusterLuaScript.fromResource(cacheCluster, "lua/account_database_crawler/unlock.lua", @@ -37,9 +35,9 @@ public class AccountDatabaseCrawlerCache { public void setAccelerated(final boolean accelerated) { if (accelerated) { - cacheCluster.useCluster(connection -> connection.sync().set(getPrefixedKey(ACCELERATE_KEY), "1")); + cacheCluster.useCluster(connection -> connection.sync().set(ACCELERATE_KEY, "1")); } else { - cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(ACCELERATE_KEY))); + cacheCluster.useCluster(connection -> connection.sync().del(ACCELERATE_KEY)); } } @@ -49,16 +47,16 @@ public class AccountDatabaseCrawlerCache { public boolean claimActiveWork(String workerId, long ttlMs) { return "OK".equals(cacheCluster.withCluster(connection -> connection.sync() - .set(getPrefixedKey(ACTIVE_WORKER_KEY), workerId, SetArgs.Builder.nx().px(ttlMs)))); + .set(ACTIVE_WORKER_KEY, workerId, SetArgs.Builder.nx().px(ttlMs)))); } public void releaseActiveWork(String workerId) { - unlockClusterScript.execute(List.of(getPrefixedKey(ACTIVE_WORKER_KEY)), List.of(workerId)); + unlockClusterScript.execute(List.of(ACTIVE_WORKER_KEY), List.of(workerId)); } public Optional getLastUuid() { final String lastUuidString = cacheCluster.withCluster( - connection -> connection.sync().get(getPrefixedKey(LAST_UUID_KEY))); + connection -> connection.sync().get(LAST_UUID_KEY)); if (lastUuidString == null) { return Optional.empty(); @@ -70,15 +68,15 @@ public class AccountDatabaseCrawlerCache { public void setLastUuid(Optional lastUuid) { if (lastUuid.isPresent()) { cacheCluster.useCluster(connection -> connection.sync() - .psetex(getPrefixedKey(LAST_UUID_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString())); + .psetex(LAST_UUID_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString())); } else { - cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_KEY))); + cacheCluster.useCluster(connection -> connection.sync().del(LAST_UUID_KEY)); } } public Optional getLastUuidDynamo() { final String lastUuidString = cacheCluster.withCluster( - connection -> connection.sync().get(getPrefixedKey(LAST_UUID_DYNAMO_KEY))); + connection -> connection.sync().get(LAST_UUID_DYNAMO_KEY)); if (lastUuidString == null) { return Optional.empty(); @@ -91,21 +89,9 @@ public class AccountDatabaseCrawlerCache { if (lastUuid.isPresent()) { cacheCluster.useCluster( connection -> connection.sync() - .psetex(getPrefixedKey(LAST_UUID_DYNAMO_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString())); + .psetex(LAST_UUID_DYNAMO_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString())); } else { - cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_DYNAMO_KEY))); + cacheCluster.useCluster(connection -> connection.sync().del(LAST_UUID_DYNAMO_KEY)); } } - - private String getPrefixedKey(final String key) { - return prefix + key; - } - - /** - * Set a cache key prefix, allowing for uses beyond the canonical crawler - */ - public void setPrefix(final String prefix) { - this.prefix = prefix + "::"; - } - } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java deleted file mode 100644 index ebbcf8ec5..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ -package org.whispersystems.textsecuregcm.storage; - -import static com.codahale.metrics.MetricRegistry.name; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.SharedMetricRegistries; -import com.codahale.metrics.Timer; -import com.codahale.metrics.Timer.Context; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import org.jdbi.v3.core.transaction.TransactionIsolationLevel; -import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper; -import org.whispersystems.textsecuregcm.util.Constants; -import org.whispersystems.textsecuregcm.util.SystemMapper; - -public class Accounts implements AccountStore { - - public static final String ID = "id"; - public static final String UID = "uuid"; - public static final String NUMBER = "number"; - public static final String DATA = "data"; - public static final String VERSION = "version"; - - private static final ObjectMapper mapper = SystemMapper.getMapper(); - - private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME); - private final Timer createTimer = metricRegistry.timer(name(Accounts.class, "create" )); - private final Timer updateTimer = metricRegistry.timer(name(Accounts.class, "update" )); - private final Timer getByNumberTimer = metricRegistry.timer(name(Accounts.class, "getByNumber" )); - private final Timer getByUuidTimer = metricRegistry.timer(name(Accounts.class, "getByUuid" )); - private final Timer getAllFromTimer = metricRegistry.timer(name(Accounts.class, "getAllFrom" )); - private final Timer getAllFromOffsetTimer = metricRegistry.timer(name(Accounts.class, "getAllFromOffset")); - private final Timer deleteTimer = metricRegistry.timer(name(Accounts.class, "delete" )); - private final Timer vacuumTimer = metricRegistry.timer(name(Accounts.class, "vacuum" )); - - private final FaultTolerantDatabase database; - - public Accounts(FaultTolerantDatabase database) { - this.database = database; - this.database.getDatabase().registerRowMapper(new AccountRowMapper()); - } - - @Override - public boolean create(Account account) { - return database.with(jdbi -> jdbi.inTransaction(TransactionIsolationLevel.SERIALIZABLE, handle -> { - try (Timer.Context ignored = createTimer.time()) { - final Map resultMap = handle.createQuery("INSERT INTO accounts (" + NUMBER + ", " + UID + ", " + DATA + ") VALUES (:number, :uuid, CAST(:data AS json)) ON CONFLICT(number) DO UPDATE SET " + DATA + " = EXCLUDED.data, " + VERSION + " = accounts.version + 1 RETURNING uuid, version") - .bind("number", account.getNumber()) - .bind("uuid", account.getUuid()) - .bind("data", mapper.writeValueAsString(account)) - .mapToMap() - .findOnly(); - - final UUID uuid = (UUID) resultMap.get(UID); - final int version = (int) resultMap.get(VERSION); - - boolean isNew = uuid.equals(account.getUuid()); - account.setUuid(uuid); - account.setVersion(version); - return isNew; - } catch (JsonProcessingException e) { - throw new IllegalArgumentException(e); - } - })); - } - - @Override - public void update(Account account) throws ContestedOptimisticLockException { - database.use(jdbi -> jdbi.useHandle(handle -> { - try (Timer.Context ignored = updateTimer.time()) { - final int newVersion = account.getVersion() + 1; - int rowsModified = handle.createUpdate("UPDATE accounts SET " + DATA + " = CAST(:data AS json), " + VERSION + " = :newVersion WHERE " + UID + " = :uuid AND " + VERSION + " = :version") - .bind("uuid", account.getUuid()) - .bind("data", mapper.writeValueAsString(account)) - .bind("version", account.getVersion()) - .bind("newVersion", newVersion) - .execute(); - - if (rowsModified == 0) { - throw new ContestedOptimisticLockException(); - } - - account.setVersion(newVersion); - - } catch (JsonProcessingException e) { - throw new IllegalArgumentException(e); - } - })); - } - - @Override - public Optional get(String number) { - return database.with(jdbi -> jdbi.withHandle(handle -> { - try (Timer.Context ignored = getByNumberTimer.time()) { - return handle.createQuery("SELECT * FROM accounts WHERE " + NUMBER + " = :number") - .bind("number", number) - .mapTo(Account.class) - .findFirst(); - } - })); - } - - @Override - public Optional get(UUID uuid) { - return database.with(jdbi -> jdbi.withHandle(handle -> { - try (Timer.Context ignored = getByUuidTimer.time()) { - return handle.createQuery("SELECT * FROM accounts WHERE " + UID + " = :uuid") - .bind("uuid", uuid) - .mapTo(Account.class) - .findFirst(); - } - })); - } - - public AccountCrawlChunk getAllFrom(UUID from, int length) { - final List accounts = database.with(jdbi -> jdbi.withHandle(handle -> { - try (Context ignored = getAllFromOffsetTimer.time()) { - return handle.createQuery("SELECT * FROM accounts WHERE " + UID + " > :from ORDER BY " + UID + " LIMIT :limit") - .bind("from", from) - .bind("limit", length) - .mapTo(Account.class) - .list(); - } - })); - return buildChunkForAccounts(accounts); - } - - public AccountCrawlChunk getAllFrom(int length) { - final List accounts = database.with(jdbi -> jdbi.withHandle(handle -> { - try (Timer.Context ignored = getAllFromTimer.time()) { - return handle.createQuery("SELECT * FROM accounts ORDER BY " + UID + " LIMIT :limit") - .bind("limit", length) - .mapTo(Account.class) - .list(); - } - })); - - return buildChunkForAccounts(accounts); - } - - private AccountCrawlChunk buildChunkForAccounts(final List accounts) { - return new AccountCrawlChunk(accounts, accounts.isEmpty() ? null : accounts.get(accounts.size() - 1).getUuid()); - } - - @Override - public void delete(final UUID uuid) { - database.use(jdbi -> jdbi.useHandle(handle -> { - try (Timer.Context ignored = deleteTimer.time()) { - handle.createUpdate("DELETE FROM accounts WHERE " + UID + " = :uuid") - .bind("uuid", uuid) - .execute(); - } - })); - } - - public void vacuum() { - database.use(jdbi -> jdbi.useHandle(handle -> { - try (Timer.Context ignored = vacuumTimer.time()) { - handle.execute("VACUUM accounts"); - } - })); - } - -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDb.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDb.java index 3737561f9..527c16e98 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDb.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDb.java @@ -8,7 +8,6 @@ import static com.codahale.metrics.MetricRegistry.name; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.annotations.VisibleForTesting; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Timer; import java.io.IOException; @@ -17,16 +16,10 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.whispersystems.textsecuregcm.util.AttributeValues; import org.whispersystems.textsecuregcm.util.SystemMapper; import org.whispersystems.textsecuregcm.util.UUIDUtil; -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.CancellationReason; @@ -59,12 +52,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt static final String ATTR_CANONICALLY_DISCOVERABLE = "C"; private final DynamoDbClient client; - private final DynamoDbAsyncClient asyncClient; - - private final ThreadPoolExecutor migrationThreadPool; - - private final MigrationDeletedAccounts migrationDeletedAccounts; - private final MigrationRetryAccounts migrationRetryAccounts; private final String phoneNumbersTableName; private final String accountsTableName; @@ -76,26 +63,15 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFrom")); private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFromOffset")); private static final Timer DELETE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "delete")); - private static final Timer DELETE_RECENTLY_DELETED_UUIDS_TIMER = Metrics.timer( - name(AccountsDynamoDb.class, "deleteRecentlyDeletedUuids")); - private final Logger logger = LoggerFactory.getLogger(AccountsDynamoDb.class); - public AccountsDynamoDb(DynamoDbClient client, DynamoDbAsyncClient asyncClient, - ThreadPoolExecutor migrationThreadPool, String accountsTableName, String phoneNumbersTableName, - MigrationDeletedAccounts migrationDeletedAccounts, - MigrationRetryAccounts accountsMigrationErrors) { + public AccountsDynamoDb(DynamoDbClient client, String accountsTableName, String phoneNumbersTableName) { super(client); this.client = client; - this.asyncClient = asyncClient; this.phoneNumbersTableName = phoneNumbersTableName; this.accountsTableName = accountsTableName; - this.migrationThreadPool = migrationThreadPool; - - this.migrationDeletedAccounts = migrationDeletedAccounts; - this.migrationRetryAccounts = accountsMigrationErrors; } @Override @@ -215,29 +191,19 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt } try { - try { - UpdateItemResponse response = client.updateItem(updateItemRequest); + UpdateItemResponse response = client.updateItem(updateItemRequest); - account.setVersion(AttributeValues.getInt(response.attributes(), "V", account.getVersion() + 1)); - } catch (final TransactionConflictException e) { + account.setVersion(AttributeValues.getInt(response.attributes(), "V", account.getVersion() + 1)); + } catch (final TransactionConflictException e) { - throw new ContestedOptimisticLockException(); + throw new ContestedOptimisticLockException(); - } catch (final ConditionalCheckFailedException e) { + } catch (final ConditionalCheckFailedException e) { - // the exception doesn’t give details about which condition failed, - // but we can infer it was an optimistic locking failure if the UUID is known - throw get(account.getUuid()).isPresent() ? new ContestedOptimisticLockException() : e; - } - } catch (final Exception e) { - if (!(e instanceof ContestedOptimisticLockException)) { - // the Dynamo account now lags the Postgres account version. Put it in the migration retry table so that it will - // get updated faster—otherwise it will be stale until the accounts crawler runs again - migrationRetryAccounts.put(account.getUuid()); - } - throw e; + // the exception doesn’t give details about which condition failed, + // but we can infer it was an optimistic locking failure if the UUID is known + throw get(account.getUuid()).isPresent() ? new ContestedOptimisticLockException() : e; } - }); } @@ -279,10 +245,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt DELETE_TIMER.record(() -> delete(uuid, true)); } - public void deleteInvalidMigration(UUID uuid) { - DELETE_TIMER.record(() -> delete(uuid, false)); - } - public AccountCrawlChunk getAllFrom(final UUID from, final int maxCount, final int pageSize) { final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder() .limit(pageSize) @@ -312,10 +274,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt private void delete(UUID uuid, boolean saveInDeletedAccountsTable) { - if (saveInDeletedAccountsTable) { - migrationDeletedAccounts.put(uuid); - } - Optional maybeAccount = get(uuid); maybeAccount.ifPresent(account -> { @@ -341,105 +299,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt }); } - private static final Counter MIGRATED_COUNTER = Metrics.counter(name(AccountsDynamoDb.class, "migration", "count")); - private static final Counter ERROR_COUNTER = Metrics.counter(name(AccountsDynamoDb.class, "migration", "error")); - - public CompletableFuture migrate(List accounts, int threads) { - - if (threads > migrationThreadPool.getMaximumPoolSize()) { - migrationThreadPool.setMaximumPoolSize(threads); - migrationThreadPool.setCorePoolSize(threads); - } else { - migrationThreadPool.setCorePoolSize(threads); - migrationThreadPool.setMaximumPoolSize(threads); - } - - final List> futures = accounts.stream() - .map(this::migrate) - .map(f -> f.whenCompleteAsync((migrated, e) -> { - if (e == null) { - MIGRATED_COUNTER.increment(migrated ? 1 : 0); - } else { - ERROR_COUNTER.increment(); - } - }, migrationThreadPool)) - .collect(Collectors.toList()); - - CompletableFuture migrationBatch = CompletableFuture.allOf(futures.toArray(new CompletableFuture[]{})); - - return migrationBatch.whenCompleteAsync((result, exception) -> { - if (exception != null) { - logger.warn("Exception migrating batch", exception); - } - deleteRecentlyDeletedUuids(); - }, migrationThreadPool); - } - - public void deleteRecentlyDeletedUuids() { - - DELETE_RECENTLY_DELETED_UUIDS_TIMER.record(() -> { - - final List recentlyDeletedUuids = migrationDeletedAccounts.getRecentlyDeletedUuids(); - - for (UUID recentlyDeletedUuid : recentlyDeletedUuids) { - delete(recentlyDeletedUuid, false); - } - - migrationDeletedAccounts.delete(recentlyDeletedUuids); - }); - } - - public CompletableFuture migrate(Account account) { - try { - TransactWriteItem phoneNumberConstraintPut = buildPutWriteItemForPhoneNumberConstraint(account, account.getUuid()); - - TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid(), Put.builder() - .conditionExpression("attribute_not_exists(#uuid) OR (attribute_exists(#uuid) AND #version < :version)") - .expressionAttributeNames(Map.of( - "#uuid", KEY_ACCOUNT_UUID, - "#version", ATTR_VERSION)) - .expressionAttributeValues(Map.of( - ":version", AttributeValues.fromInt(account.getVersion())))); - - final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder() - .transactItems(phoneNumberConstraintPut, accountPut).build(); - - final CompletableFuture resultFuture = new CompletableFuture<>(); - asyncClient.transactWriteItems(request).whenCompleteAsync((result, exception) -> { - if (result != null) { - resultFuture.complete(true); - return; - } - if (exception instanceof CompletionException) { - // whenCompleteAsync can wrap exceptions in a CompletionException; unwrap it to get to the root cause. - exception = exception.getCause(); - } - if (exception instanceof TransactionCanceledException) { - // account is already migrated - resultFuture.complete(false); - return; - } - try { - migrationRetryAccounts.put(account.getUuid()); - } catch (final Exception e) { - logger.error("Could not store account {}", account.getUuid()); - } - resultFuture.completeExceptionally(exception); - }, migrationThreadPool); - return resultFuture; - } catch (Exception e) { - return CompletableFuture.failedFuture(e); - } - } - - void putUuidForMigrationRetry(final UUID uuid) { - try { - migrationRetryAccounts.put(uuid); - } catch (final Exception e) { - logger.error("Failed to store for retry: {}", uuid, e); - } - } - private static String extractCancellationReasonCodes(final TransactionCanceledException exception) { return exception.cancellationReasons().stream() .map(CancellationReason::code) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbMigrator.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbMigrator.java deleted file mode 100644 index 7ae8be997..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbMigrator.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.whispersystems.textsecuregcm.storage; - -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; - -public class AccountsDynamoDbMigrator extends AccountDatabaseCrawlerListener { - - private final AccountsDynamoDb accountsDynamoDb; - private final DynamicConfigurationManager dynamicConfigurationManager; - - public AccountsDynamoDbMigrator(final AccountsDynamoDb accountsDynamoDb, final DynamicConfigurationManager dynamicConfigurationManager) { - this.accountsDynamoDb = accountsDynamoDb; - this.dynamicConfigurationManager = dynamicConfigurationManager; - } - - @Override - public void onCrawlStart() { - - } - - @Override - public void onCrawlEnd(Optional fromUuid) { - - } - - @Override - protected void onCrawlChunk(Optional fromUuid, List chunkAccounts) { - - if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration() - .isBackgroundMigrationEnabled()) { - return; - } - - final CompletableFuture migrationBatch = accountsDynamoDb.migrate(chunkAccounts, - dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().getBackgroundMigrationExecutorThreads()); - - migrationBatch.join(); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java index 3b030fc12..9f85a605e 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java @@ -11,22 +11,17 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.SharedMetricRegistries; import com.codahale.metrics.Timer; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.lettuce.core.RedisException; import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Tags; import java.io.IOException; import java.util.Arrays; -import java.util.Objects; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -37,7 +32,6 @@ import org.slf4j.LoggerFactory; import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials; import org.whispersystems.textsecuregcm.controllers.AccountController; import org.whispersystems.textsecuregcm.entities.AccountAttributes; -import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; @@ -69,20 +63,14 @@ public class AccountsManager { private static final String COUNTRY_CODE_TAG_NAME = "country"; private static final String DELETION_REASON_TAG_NAME = "reason"; - private static final String DYNAMO_MIGRATION_ERROR_COUNTER_NAME = name(AccountsManager.class, "migration", "error"); - private static final Counter DYNAMO_MIGRATION_COMPARISON_COUNTER = Metrics.counter(name(AccountsManager.class, "migration", "comparisons")); - private static final String DYNAMO_MIGRATION_MISMATCH_COUNTER_NAME = name(AccountsManager.class, "migration", "mismatches"); - private final Logger logger = LoggerFactory.getLogger(AccountsManager.class); - private final Accounts accounts; private final AccountsDynamoDb accountsDynamoDb; private final FaultTolerantRedisCluster cacheCluster; private final DeletedAccountsManager deletedAccountsManager; private final DirectoryQueue directoryQueue; private final KeysDynamoDb keysDynamoDb; private final MessagesManager messagesManager; - private final MigrationMismatchedAccounts mismatchedAccounts; private final UsernamesManager usernamesManager; private final ProfilesManager profilesManager; private final StoredVerificationCodeManager pendingAccounts; @@ -90,10 +78,7 @@ public class AccountsManager { private final SecureBackupClient secureBackupClient; private final ObjectMapper mapper; - private final ObjectMapper migrationComparisonMapper; - private final DynamicConfigurationManager dynamicConfigurationManager; - private final ExperimentEnrollmentManager experimentEnrollmentManager; public enum DeletionReason { ADMIN_DELETED("admin"), @@ -107,25 +92,22 @@ public class AccountsManager { } } - public AccountsManager(Accounts accounts, AccountsDynamoDb accountsDynamoDb, FaultTolerantRedisCluster cacheCluster, + public AccountsManager(AccountsDynamoDb accountsDynamoDb, FaultTolerantRedisCluster cacheCluster, final DeletedAccountsManager deletedAccountsManager, final DirectoryQueue directoryQueue, final KeysDynamoDb keysDynamoDb, final MessagesManager messagesManager, - final MigrationMismatchedAccounts mismatchedAccounts, final UsernamesManager usernamesManager, + final UsernamesManager usernamesManager, final ProfilesManager profilesManager, final StoredVerificationCodeManager pendingAccounts, final SecureStorageClient secureStorageClient, final SecureBackupClient secureBackupClient, - final ExperimentEnrollmentManager experimentEnrollmentManager, final DynamicConfigurationManager dynamicConfigurationManager) { - this.accounts = accounts; - this.accountsDynamoDb = accountsDynamoDb; + this.accountsDynamoDb = accountsDynamoDb; this.cacheCluster = cacheCluster; this.deletedAccountsManager = deletedAccountsManager; this.directoryQueue = directoryQueue; this.keysDynamoDb = keysDynamoDb; this.messagesManager = messagesManager; - this.mismatchedAccounts = mismatchedAccounts; this.usernamesManager = usernamesManager; this.profilesManager = profilesManager; this.pendingAccounts = pendingAccounts; @@ -133,11 +115,7 @@ public class AccountsManager { this.secureBackupClient = secureBackupClient; this.mapper = SystemMapper.getMapper(); - this.migrationComparisonMapper = mapper.copy(); - migrationComparisonMapper.addMixIn(Device.class, DeviceComparisonMixin.class); - this.dynamicConfigurationManager = dynamicConfigurationManager; - this.experimentEnrollmentManager = experimentEnrollmentManager; } public Account create(final String number, @@ -170,36 +148,12 @@ public class AccountsManager { final UUID originalUuid = account.getUuid(); - boolean freshUser = primaryCreate(account); + boolean freshUser = dynamoCreate(account); // create() sometimes updates the UUID, if there was a number conflict. // for metrics, we want secondary to run with the same original UUID final UUID actualUuid = account.getUuid(); - try { - if (secondaryWriteEnabled()) { - - account.setUuid(originalUuid); - - runSafelyAndRecordMetrics(() -> secondaryCreate(account), Optional.of(account.getUuid()), freshUser, - (primaryResult, secondaryResult) -> { - - if (primaryResult.equals(secondaryResult)) { - return Optional.empty(); - } - - if (secondaryResult) { - return Optional.of("secondaryFreshUser"); - } - - return Optional.of("primaryFreshUser"); - }, - "create"); - } - } finally { - account.setUuid(actualUuid); - } - redisSet(account); pendingAccounts.remove(number); @@ -293,26 +247,7 @@ public class AccountsManager { final UUID uuid = account.getUuid(); - updatedAccount = updateWithRetries(account, updater, this::primaryUpdate, () -> primaryGet(uuid).get()); - - if (secondaryWriteEnabled()) { - runSafelyAndRecordMetrics(() -> secondaryGet(uuid).map(secondaryAccount -> { - try { - return updateWithRetries(secondaryAccount, updater, this::secondaryUpdate, () -> secondaryGet(uuid).get()); - } catch (final OptimisticLockRetryLimitExceededException e) { - if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration() - .isDynamoPrimary()) { - accountsDynamoDb.putUuidForMigrationRetry(uuid); - } - - throw e; - } - }), - Optional.of(uuid), - Optional.of(updatedAccount), - this::compareAccounts, - "update"); - } + updatedAccount = updateWithRetries(account, updater, this::dynamoUpdate, () -> dynamoGet(uuid).get()); redisSet(updatedAccount); } @@ -378,14 +313,9 @@ public class AccountsManager { try (Timer.Context ignored = getByNumberTimer.time()) { Optional account = redisGet(number); - if (!account.isPresent()) { - account = primaryGet(number); - account.ifPresent(value -> redisSet(value)); - - if (secondaryReadEnabled()) { - runSafelyAndRecordMetrics(() -> secondaryGet(number), Optional.empty(), account, this::compareAccounts, - "getByNumber"); - } + if (account.isEmpty()) { + account = dynamoGet(number); + account.ifPresent(this::redisSet); } return account; @@ -396,29 +326,15 @@ public class AccountsManager { try (Timer.Context ignored = getByUuidTimer.time()) { Optional account = redisGet(uuid); - if (!account.isPresent()) { - account = primaryGet(uuid); - account.ifPresent(value -> redisSet(value)); - - if (secondaryReadEnabled()) { - runSafelyAndRecordMetrics(() -> secondaryGet(uuid), Optional.of(uuid), account, this::compareAccounts, - "getByUuid"); - } + if (account.isEmpty()) { + account = dynamoGet(uuid); + account.ifPresent(this::redisSet); } return account; } } - - public AccountCrawlChunk getAllFrom(int length) { - return accounts.getAllFrom(length); - } - - public AccountCrawlChunk getAllFrom(UUID uuid, int length) { - return accounts.getAllFrom(uuid, length); - } - public AccountCrawlChunk getAllFromDynamo(int length) { final int maxPageSize = dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration() .getDynamoCrawlerScanPageSize(); @@ -447,16 +363,7 @@ public class AccountsManager { deleteBackupServiceDataFuture.join(); redisDelete(account); - primaryDelete(account); - - if (secondaryDeleteEnabled()) { - try { - secondaryDelete(account); - } catch (final Exception e) { - logger.error("Could not delete account {} from secondary", account.getUuid().toString()); - Metrics.counter(DYNAMO_MIGRATION_ERROR_COUNTER_NAME, "action", "delete").increment(); - } - } + dynamoDelete(account); return account.getUuid(); }); @@ -537,100 +444,6 @@ public class AccountsManager { } } - private Optional primaryGet(String number) { - return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary() - ? - dynamoGet(number) : - databaseGet(number); - } - - private Optional secondaryGet(String number) { - return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary() - ? - databaseGet(number) : - dynamoGet(number); - } - - private Optional primaryGet(UUID uuid) { - return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary() - ? - dynamoGet(uuid) : - databaseGet(uuid); - } - - private Optional secondaryGet(UUID uuid) { - return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary() - ? - databaseGet(uuid) : - dynamoGet(uuid); - } - - private boolean primaryCreate(Account account) { - return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary() - ? - dynamoCreate(account) : - databaseCreate(account); - } - - private boolean secondaryCreate(Account account) { - return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary() - ? - databaseCreate(account) : - dynamoCreate(account); - } - - private void primaryUpdate(Account account) { - if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) { - dynamoUpdate(account); - } else { - databaseUpdate(account); - } - } - - private void secondaryUpdate(Account account) { - if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) { - databaseUpdate(account); - } else { - dynamoUpdate(account); - } - } - - private void primaryDelete(Account account) { - if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) { - dynamoDelete(account); - } else { - databaseDelete(account); - } - } - - private void secondaryDelete(Account account) { - if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) { - databaseDelete(account); - } else { - dynamoDelete(account); - } - } - - private Optional databaseGet(String number) { - return accounts.get(number); - } - - private Optional databaseGet(UUID uuid) { - return accounts.get(uuid); - } - - private boolean databaseCreate(Account account) { - return accounts.create(account); - } - - private void databaseUpdate(Account account) { - accounts.update(account); - } - - private void databaseDelete(final Account account) { - accounts.delete(account.getUuid()); - } - private Optional dynamoGet(String number) { return accountsDynamoDb.get(number); } @@ -651,175 +464,14 @@ public class AccountsManager { accountsDynamoDb.delete(account.getUuid()); } - private boolean secondaryDeleteEnabled() { - return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDeleteEnabled(); - } - - private boolean secondaryReadEnabled() { - return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isReadEnabled(); - } - - private boolean secondaryWriteEnabled() { - return secondaryDeleteEnabled() - && dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isWriteEnabled(); - } - + // TODO delete @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + @Deprecated public Optional compareAccounts(final Optional maybePrimaryAccount, final Optional maybeSecondaryAccount) { - - if (maybePrimaryAccount.isEmpty() && maybeSecondaryAccount.isEmpty()) { - return Optional.empty(); - } - - if (maybePrimaryAccount.isEmpty()) { - return Optional.of("primaryMissing"); - } - - if (maybeSecondaryAccount.isEmpty()) { - return Optional.of("secondaryMissing"); - } - - final Account primaryAccount = maybePrimaryAccount.get(); - final Account secondaryAccount = maybeSecondaryAccount.get(); - - final int uuidCompare = primaryAccount.getUuid().compareTo(secondaryAccount.getUuid()); - - if (uuidCompare != 0) { - return Optional.of("uuid"); - } - - final int numberCompare = primaryAccount.getNumber().compareTo(secondaryAccount.getNumber()); - - if (numberCompare != 0) { - return Optional.of("number"); - } - - if (!Objects.equals(primaryAccount.getIdentityKey(), secondaryAccount.getIdentityKey())) { - return Optional.of("identityKey"); - } - - if (!Objects.equals(primaryAccount.getCurrentProfileVersion(), secondaryAccount.getCurrentProfileVersion())) { - return Optional.of("currentProfileVersion"); - } - - if (!Objects.equals(primaryAccount.getProfileName(), secondaryAccount.getProfileName())) { - return Optional.of("profileName"); - } - - if (!Objects.equals(primaryAccount.getAvatar(), secondaryAccount.getAvatar())) { - return Optional.of("avatar"); - } - - if (!Objects.equals(primaryAccount.getUnidentifiedAccessKey(), secondaryAccount.getUnidentifiedAccessKey())) { - if (primaryAccount.getUnidentifiedAccessKey().isPresent() && secondaryAccount.getUnidentifiedAccessKey() - .isPresent()) { - - if (Arrays.compare(primaryAccount.getUnidentifiedAccessKey().get(), - secondaryAccount.getUnidentifiedAccessKey().get()) != 0) { - return Optional.of("unidentifiedAccessKey"); - } - - } else { - return Optional.of("unidentifiedAccessKey"); - } - } - - if (!Objects.equals(primaryAccount.isUnrestrictedUnidentifiedAccess(), - secondaryAccount.isUnrestrictedUnidentifiedAccess())) { - return Optional.of("unrestrictedUnidentifiedAccess"); - } - - if (!Objects.equals(primaryAccount.isDiscoverableByPhoneNumber(), secondaryAccount.isDiscoverableByPhoneNumber())) { - return Optional.of("discoverableByPhoneNumber"); - } - - if (primaryAccount.getMasterDevice().isPresent() && secondaryAccount.getMasterDevice().isPresent()) { - if (!Objects.equals(primaryAccount.getMasterDevice().get().getSignedPreKey(), - secondaryAccount.getMasterDevice().get().getSignedPreKey())) { - return Optional.of("masterDeviceSignedPreKey"); - } - } - - try { - if (!serializedEquals(primaryAccount.getDevices(), secondaryAccount.getDevices())) { - return Optional.of("devices"); - } - - if (primaryAccount.getVersion() != secondaryAccount.getVersion()) { - return Optional.of("version"); - } - - if (primaryAccount.getMasterDevice().isPresent() && secondaryAccount.getMasterDevice().isPresent()) { - if (Math.abs(primaryAccount.getMasterDevice().get().getPushTimestamp() - - secondaryAccount.getMasterDevice().get().getPushTimestamp()) > 60 * 1_000L) { - // These are generally few milliseconds off, because the setter uses System.currentTimeMillis() internally, - // but we can be more relaxed - return Optional.of("masterDevicePushTimestamp"); - } - } - - if (!serializedEquals(primaryAccount, secondaryAccount)) { - return Optional.of("serialization"); - } - - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - return Optional.empty(); } - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private void runSafelyAndRecordMetrics(Callable callable, Optional maybeUuid, final T primaryResult, - final BiFunction> mismatchClassifier, final String action) { - - if (maybeUuid.isPresent()) { - // the only time we don’t have a UUID is in getByNumber, which is sufficiently low volume to not be a concern, and - // it will also be gated by the global readEnabled configuration - final boolean enrolled = experimentEnrollmentManager.isEnrolled(maybeUuid.get(), "accountsDynamoDbMigration"); - - if (!enrolled) { - return; - } - } - - try { - - final T secondaryResult = callable.call(); - compare(primaryResult, secondaryResult, mismatchClassifier, action, maybeUuid); - - } catch (final Exception e) { - logger.error("Error running " + action + " in Dynamo", e); - - Metrics.counter(DYNAMO_MIGRATION_ERROR_COUNTER_NAME, "action", action).increment(); - } - } - - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private void compare(final T primaryResult, final T secondaryResult, - final BiFunction> mismatchClassifier, final String action, - final Optional maybeUUid) { - - DYNAMO_MIGRATION_COMPARISON_COUNTER.increment(); - - mismatchClassifier.apply(primaryResult, secondaryResult) - .ifPresent(mismatchType -> { - final String mismatchDescription = action + ":" + mismatchType; - Metrics.counter(DYNAMO_MIGRATION_MISMATCH_COUNTER_NAME, - "mismatchType", mismatchDescription) - .increment(); - - maybeUUid.ifPresent(uuid -> { - - if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration() - .isPostCheckMismatches()) { - mismatchedAccounts.put(uuid); - } - }); - }); - } - private String getAbbreviatedCallChain(final StackTraceElement[] stackTrace) { return Arrays.stream(stackTrace) .filter(stackTraceElement -> stackTraceElement.getClassName().contains("org.whispersystems")) @@ -827,22 +479,4 @@ public class AccountsManager { .map(stackTraceElement -> StringUtils.substringAfterLast(stackTraceElement.getClassName(), ".") + ":" + stackTraceElement.getMethodName()) .collect(Collectors.joining(" -> ")); } - - private static abstract class DeviceComparisonMixin extends Device { - - @JsonIgnore - private long lastSeen; - - @JsonIgnore - private long pushTimestamp; - - } - - private boolean serializedEquals(final Object primary, final Object secondary) throws JsonProcessingException { - final byte[] primarySerialized = migrationComparisonMapper.writeValueAsBytes(primary); - final byte[] secondarySerialized = migrationComparisonMapper.writeValueAsBytes(secondary); - final int serializeCompare = Arrays.compare(primarySerialized, secondarySerialized); - - return serializeCompare == 0; - } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationDeletedAccounts.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationDeletedAccounts.java deleted file mode 100644 index 1957fa415..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationDeletedAccounts.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.whispersystems.textsecuregcm.storage; - -import com.google.common.annotations.VisibleForTesting; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Collectors; -import org.whispersystems.textsecuregcm.util.AttributeValues; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.DeleteRequest; -import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanResponse; -import software.amazon.awssdk.services.dynamodb.model.WriteRequest; - -public class MigrationDeletedAccounts extends AbstractDynamoDbStore { - - private final String tableName; - - static final String KEY_UUID = "U"; - - public MigrationDeletedAccounts(DynamoDbClient dynamoDb, String tableName) { - super(dynamoDb); - this.tableName = tableName; - } - - public void put(UUID uuid) { - db().putItem(PutItemRequest.builder() - .tableName(tableName) - .item(primaryKey(uuid)) - .build()); - } - - public List getRecentlyDeletedUuids() { - - final List uuids = new ArrayList<>(); - Optional firstPage = db().scanPaginator(ScanRequest.builder() - .tableName(tableName) - .build()).stream().findAny(); // get the first available response - - if (firstPage.isPresent()) { - for (Map item : firstPage.get().items()) { - // only process one page each time. If we have a significant backlog at the end of the migration - // we can handle it separately - uuids.add(AttributeValues.getUUID(item, KEY_UUID, null)); - } - } - - return uuids; - } - - public void delete(List uuids) { - - writeInBatches(uuids, (batch) -> { - List deletes = batch.stream().map((uuid) -> WriteRequest.builder().deleteRequest(DeleteRequest.builder() - .key(primaryKey(uuid)) - .build()).build()).collect(Collectors.toList()); - - executeTableWriteItemsUntilComplete(Map.of(tableName, deletes)); - }); - } - - @VisibleForTesting - public static Map primaryKey(UUID uuid) { - return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid)); - } - -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchedAccounts.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchedAccounts.java deleted file mode 100644 index 11e6326d9..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchedAccounts.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import com.google.common.annotations.VisibleForTesting; -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import org.whispersystems.textsecuregcm.util.AttributeValues; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.DeleteRequest; -import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanResponse; -import software.amazon.awssdk.services.dynamodb.model.WriteRequest; -import software.amazon.awssdk.services.dynamodb.paginators.ScanIterable; - -public class MigrationMismatchedAccounts extends AbstractDynamoDbStore { - - static final String KEY_UUID = "U"; - static final String ATTR_TIMESTAMP = "T"; - - @VisibleForTesting - static final long MISMATCH_CHECK_DELAY_MILLIS = Duration.ofMinutes(1).toMillis(); - - private final String tableName; - private final Clock clock; - - public void put(UUID uuid) { - final Map item = primaryKey(uuid); - item.put("T", AttributeValues.fromLong(clock.millis())); - db().putItem(PutItemRequest.builder() - .tableName(tableName) - .item(item) - .build()); - } - - public MigrationMismatchedAccounts(DynamoDbClient dynamoDb, String tableName) { - this(dynamoDb, tableName, Clock.systemUTC()); - } - - @VisibleForTesting - MigrationMismatchedAccounts(DynamoDbClient dynamoDb, String tableName, final Clock clock) { - super(dynamoDb); - - this.tableName = tableName; - this.clock = clock; - } - - /** - * returns a list of UUIDs stored in the table that have passed {@link #MISMATCH_CHECK_DELAY_MILLIS} - */ - public List getUuids(int max) { - - final List uuids = new ArrayList<>(); - - final ScanIterable scanPaginator = db().scanPaginator(ScanRequest.builder() - .tableName(tableName) - .filterExpression("#timestamp <= :timestamp") - .expressionAttributeNames(Map.of("#timestamp", ATTR_TIMESTAMP)) - .expressionAttributeValues(Map.of(":timestamp", - AttributeValues.fromLong(clock.millis() - MISMATCH_CHECK_DELAY_MILLIS))) - .build()); - - for (ScanResponse response : scanPaginator) { - - for (Map item : response.items()) { - uuids.add(AttributeValues.getUUID(item, KEY_UUID, null)); - - if (uuids.size() >= max) { - break; - } - } - - if (uuids.size() >= max) { - break; - } - } - - return uuids; - } - - @VisibleForTesting - public static Map primaryKey(UUID uuid) { - final HashMap item = new HashMap<>(); - item.put(KEY_UUID, AttributeValues.fromUUID(uuid)); - return item; - } - - public void delete(final List uuidsToDelete) { - - writeInBatches(uuidsToDelete, (uuids -> { - - final List deletes = uuids.stream() - .map(uuid -> WriteRequest.builder().deleteRequest( - DeleteRequest.builder().key(Map.of(KEY_UUID, AttributeValues.fromUUID(uuid))).build()).build()) - .collect(Collectors.toList()); - - executeTableWriteItemsUntilComplete(Map.of(tableName, deletes)); - })); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchedAccountsTableCrawler.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchedAccountsTableCrawler.java deleted file mode 100644 index b141d9ba6..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchedAccountsTableCrawler.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ -package org.whispersystems.textsecuregcm.storage; - - -import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name; - -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Metrics; -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ScheduledExecutorService; -import net.logstash.logback.argument.StructuredArguments; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; - -public class MigrationMismatchedAccountsTableCrawler extends ManagedPeriodicWork { - - private static final Logger logger = LoggerFactory.getLogger(MigrationMismatchedAccountsTableCrawler.class); - - private static final Duration WORKER_TTL = Duration.ofMinutes(2); - private static final Duration RUN_INTERVAL = Duration.ofMinutes(1); - private static final String ACTIVE_WORKER_KEY = "migration_mismatched_accounts_crawler_cache_active_worker"; - - private static final int MAX_BATCH_SIZE = 5_000; - - private static final Counter COMPARISONS_COUNTER = Metrics.counter( - name(MigrationMismatchedAccountsTableCrawler.class, "comparisons")); - private static final String MISMATCH_COUNTER_NAME = name(MigrationMismatchedAccountsTableCrawler.class, "mismatches"); - private static final Counter ERRORS_COUNTER = Metrics.counter( - name(MigrationMismatchedAccountsTableCrawler.class, "errors")); - - private final MigrationMismatchedAccounts mismatchedAccounts; - private final AccountsManager accountsManager; - private final Accounts accountsDb; - private final AccountsDynamoDb accountsDynamoDb; - - private final DynamicConfigurationManager dynamicConfigurationManager; - - public MigrationMismatchedAccountsTableCrawler( - final MigrationMismatchedAccounts mismatchedAccounts, - final AccountsManager accountsManager, - final Accounts accountsDb, - final AccountsDynamoDb accountsDynamoDb, - final DynamicConfigurationManager dynamicConfigurationManager, - final FaultTolerantRedisCluster cluster, - final ScheduledExecutorService executorService) throws IOException { - - super(new ManagedPeriodicWorkLock(ACTIVE_WORKER_KEY, cluster), WORKER_TTL, RUN_INTERVAL, executorService); - - this.mismatchedAccounts = mismatchedAccounts; - this.accountsManager = accountsManager; - this.accountsDb = accountsDb; - this.accountsDynamoDb = accountsDynamoDb; - this.dynamicConfigurationManager = dynamicConfigurationManager; - } - - @Override - public void doPeriodicWork() { - - if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration() - .isPostCheckMismatches()) { - return; - } - - final List uuids = this.mismatchedAccounts.getUuids(MAX_BATCH_SIZE); - - final List processedUuids = new ArrayList<>(uuids.size()); - - try { - for (UUID uuid : uuids) { - - try { - - final Optional result = accountsManager.compareAccounts(accountsDb.get(uuid), - accountsDynamoDb.get(uuid)); - - COMPARISONS_COUNTER.increment(); - - result.ifPresent(mismatchType -> { - Metrics.counter(MISMATCH_COUNTER_NAME, "type", mismatchType) - .increment(); - - if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration() - .isLogMismatches()) { - logger.info("Mismatch: {}", StructuredArguments.entries(Map.of( - "type", mismatchType, - "uuid", uuid))); - } - }); - - processedUuids.add(uuid); - - } catch (final Exception e) { - ERRORS_COUNTER.increment(); - logger.warn("Failed to check account mismatch", e); - } - - } - } finally { - this.mismatchedAccounts.delete(processedUuids); - } - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccounts.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccounts.java deleted file mode 100644 index 440669fed..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccounts.java +++ /dev/null @@ -1,76 +0,0 @@ -package org.whispersystems.textsecuregcm.storage; - -import com.google.common.annotations.VisibleForTesting; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import org.whispersystems.textsecuregcm.util.AttributeValues; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.DeleteRequest; -import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanResponse; -import software.amazon.awssdk.services.dynamodb.model.WriteRequest; - -public class MigrationRetryAccounts extends AbstractDynamoDbStore { - - private final String tableName; - - static final String KEY_UUID = "U"; - - public MigrationRetryAccounts(DynamoDbClient dynamoDb, String tableName) { - super(dynamoDb); - - this.tableName = tableName; - } - - public void put(UUID uuid) { - db().putItem(PutItemRequest.builder() - .tableName(tableName) - .item(primaryKey(uuid)) - .build()); - } - - public List getUuids(int max) { - - final List uuids = new ArrayList<>(); - - for (ScanResponse response : db().scanPaginator(ScanRequest.builder().tableName(tableName).build())) { - - for (Map item : response.items()) { - uuids.add(AttributeValues.getUUID(item, KEY_UUID, null)); - - if (uuids.size() >= max) { - break; - } - } - - if (uuids.size() >= max) { - break; - } - } - - return uuids; - } - - @VisibleForTesting - public static Map primaryKey(UUID uuid) { - return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid)); - } - - public void delete(final List uuidsToDelete) { - - writeInBatches(uuidsToDelete, (uuids -> { - - final List deletes = uuids.stream() - .map(uuid -> WriteRequest.builder().deleteRequest( - DeleteRequest.builder().key(Map.of(KEY_UUID, AttributeValues.fromUUID(uuid))).build()).build()) - .collect(Collectors.toList()); - - executeTableWriteItemsUntilComplete(Map.of(tableName, deletes)); - })); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccountsTableCrawler.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccountsTableCrawler.java deleted file mode 100644 index 4cf9857ee..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccountsTableCrawler.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2013-2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ -package org.whispersystems.textsecuregcm.storage; - -import static com.codahale.metrics.MetricRegistry.name; - -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Metrics; -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ScheduledExecutorService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; - -public class MigrationRetryAccountsTableCrawler extends ManagedPeriodicWork { - - private static final Logger logger = LoggerFactory.getLogger(MigrationRetryAccountsTableCrawler.class); - - private static final Duration WORKER_TTL = Duration.ofMinutes(2); - private static final Duration RUN_INTERVAL = Duration.ofMinutes(15); - private static final String ACTIVE_WORKER_KEY = "migration_retry_accounts_crawler_cache_active_worker"; - - private static final int MAX_BATCH_SIZE = 5_000; - - private static final Counter MIGRATED_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "migrated")); - private static final Counter ERROR_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "error")); - private static final Counter TOTAL_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "total")); - - private final MigrationRetryAccounts retryAccounts; - private final AccountsManager accountsManager; - private final AccountsDynamoDb accountsDynamoDb; - - public MigrationRetryAccountsTableCrawler( - final MigrationRetryAccounts retryAccounts, - final AccountsManager accountsManager, - final AccountsDynamoDb accountsDynamoDb, - final FaultTolerantRedisCluster cluster, - final ScheduledExecutorService executorService) throws IOException { - - super(new ManagedPeriodicWorkLock(ACTIVE_WORKER_KEY, cluster), WORKER_TTL, RUN_INTERVAL, executorService); - - this.retryAccounts = retryAccounts; - this.accountsManager = accountsManager; - this.accountsDynamoDb = accountsDynamoDb; - } - - @Override - public void doPeriodicWork() { - - final List uuids = this.retryAccounts.getUuids(MAX_BATCH_SIZE); - - final List processedUuids = new ArrayList<>(uuids.size()); - - try { - for (UUID uuid : uuids) { - - try { - final Optional maybeDynamoAccount = accountsDynamoDb.get(uuid); - - if (maybeDynamoAccount.isEmpty()) { - accountsManager.get(uuid).ifPresent(account -> { - accountsDynamoDb.migrate(account); - MIGRATED_COUNTER.increment(); - }); - } - - processedUuids.add(uuid); - - TOTAL_COUNTER.increment(); - - } catch (final Exception e) { - ERROR_COUNTER.increment(); - logger.warn("Failed to migrate account"); - } - - } - } finally { - this.retryAccounts.delete(processedUuids); - } - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/Usernames.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/Usernames.java index c09c31e81..2ebd00117 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/Usernames.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/Usernames.java @@ -5,18 +5,16 @@ package org.whispersystems.textsecuregcm.storage; +import static com.codahale.metrics.MetricRegistry.name; + import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.SharedMetricRegistries; import com.codahale.metrics.Timer; -import org.jdbi.v3.core.JdbiException; -import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper; -import org.whispersystems.textsecuregcm.util.Constants; - import java.sql.SQLException; import java.util.Optional; import java.util.UUID; - -import static com.codahale.metrics.MetricRegistry.name; +import org.jdbi.v3.core.JdbiException; +import org.whispersystems.textsecuregcm.util.Constants; public class Usernames { @@ -34,7 +32,6 @@ public class Usernames { public Usernames(FaultTolerantDatabase database) { this.database = database; - this.database.getDatabase().registerRowMapper(new AccountRowMapper()); } public boolean put(UUID uuid, String username) { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/mappers/AccountRowMapper.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/mappers/AccountRowMapper.java deleted file mode 100644 index 33c3a535e..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/mappers/AccountRowMapper.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage.mappers; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; -import org.whispersystems.textsecuregcm.storage.Account; -import org.whispersystems.textsecuregcm.storage.Accounts; -import org.whispersystems.textsecuregcm.util.SystemMapper; - -import java.io.IOException; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.UUID; - -public class AccountRowMapper implements RowMapper { - - private static ObjectMapper mapper = SystemMapper.getMapper(); - - @Override - public Account map(ResultSet resultSet, StatementContext ctx) throws SQLException { - try { - Account account = mapper.readValue(resultSet.getString(Accounts.DATA), Account.class); - account.setNumber(resultSet.getString(Accounts.NUMBER)); - account.setUuid(UUID.fromString(resultSet.getString(Accounts.UID))); - account.setVersion(resultSet.getInt(Accounts.VERSION)); - return account; - } catch (IOException e) { - throw new SQLException(e); - } - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java index 1b7c55217..780f74d8c 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java @@ -20,9 +20,6 @@ import io.lettuce.core.resource.ClientResources; import io.micrometer.core.instrument.Metrics; import java.util.Optional; import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import net.sourceforge.argparse4j.inf.Namespace; import net.sourceforge.argparse4j.inf.Subparser; import org.jdbi.v3.core.Jdbi; @@ -30,14 +27,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.whispersystems.textsecuregcm.WhisperServerConfiguration; import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator; -import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager; import org.whispersystems.textsecuregcm.metrics.PushLatencyManager; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; import org.whispersystems.textsecuregcm.storage.Account; -import org.whispersystems.textsecuregcm.storage.Accounts; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.AccountsManager.DeletionReason; @@ -49,9 +44,6 @@ import org.whispersystems.textsecuregcm.storage.KeysDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesCache; import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesManager; -import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts; -import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts; -import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts; import org.whispersystems.textsecuregcm.storage.Profiles; import org.whispersystems.textsecuregcm.storage.ProfilesManager; import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb; @@ -62,7 +54,6 @@ import org.whispersystems.textsecuregcm.storage.Usernames; import org.whispersystems.textsecuregcm.storage.UsernamesManager; import org.whispersystems.textsecuregcm.storage.VerificationCodeStore; import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig; -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; public class DeleteUserCommand extends EnvironmentCommand { @@ -70,11 +61,9 @@ public class DeleteUserCommand extends EnvironmentCommand() { + super(new Application<>() { @Override - public void run(WhisperServerConfiguration configuration, Environment environment) - throws Exception - { + public void run(WhisperServerConfiguration configuration, Environment environment) { } }, "rmuser", "remove user"); @@ -105,9 +94,6 @@ public class DeleteUserCommand extends EnvironmentCommand()); - DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig.client( configuration.getReportMessageDynamoDbConfiguration(), software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); @@ -118,16 +104,9 @@ public class DeleteUserCommand extends EnvironmentCommand { @@ -105,9 +96,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand()); - DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig .client(configuration.getReportMessageDynamoDbConfiguration(), software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); @@ -118,10 +106,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand maybeAccount; diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/VacuumCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/VacuumCommand.java deleted file mode 100644 index 855d7be3b..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/VacuumCommand.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.workers; - -import net.sourceforge.argparse4j.inf.Namespace; -import org.jdbi.v3.core.Jdbi; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.whispersystems.textsecuregcm.WhisperServerConfiguration; -import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration; -import org.whispersystems.textsecuregcm.storage.Accounts; -import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; - -import io.dropwizard.cli.ConfiguredCommand; -import io.dropwizard.setup.Bootstrap; - - -public class VacuumCommand extends ConfiguredCommand { - - private final Logger logger = LoggerFactory.getLogger(VacuumCommand.class); - - public VacuumCommand() { - super("vacuum", "Vacuum Postgres Tables"); - } - - @Override - protected void run(Bootstrap bootstrap, - Namespace namespace, - WhisperServerConfiguration config) - throws Exception - { - DatabaseConfiguration accountDbConfig = config.getAbuseDatabaseConfiguration(); - Jdbi accountJdbi = Jdbi.create(accountDbConfig.getUrl(), accountDbConfig.getUser(), accountDbConfig.getPassword()); - FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_vacuum", accountJdbi, accountDbConfig.getCircuitBreakerConfiguration()); - - Accounts accounts = new Accounts(accountDatabase); - - logger.info("Vacuuming accounts..."); - accounts.vacuum(); - - Thread.sleep(3000); - System.exit(0); - } -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicConfigurationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicConfigurationTest.java index f86639390..cf6f5cc50 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicConfigurationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicConfigurationTest.java @@ -320,30 +320,19 @@ class DynamicConfigurationTest { final DynamicConfiguration emptyConfig = DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow(); - assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isBackgroundMigrationEnabled()); - assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isDeleteEnabled()); - assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isWriteEnabled()); - assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isReadEnabled()); + assertEquals(10, emptyConfig.getAccountsDynamoDbMigrationConfiguration().getDynamoCrawlerScanPageSize()); } { final String accountsDynamoDbMigrationConfig = "accountsDynamoDbMigration:\n" - + " backgroundMigrationEnabled: true\n" - + " backgroundMigrationExecutorThreads: 100\n" - + " deleteEnabled: true\n" - + " readEnabled: true\n" - + " writeEnabled: true"; + + " dynamoCrawlerScanPageSize: 5000"; final DynamicAccountsDynamoDbMigrationConfiguration config = DynamicConfigurationManager.parseConfiguration(accountsDynamoDbMigrationConfig).orElseThrow() .getAccountsDynamoDbMigrationConfiguration(); - assertTrue(config.isBackgroundMigrationEnabled()); - assertEquals(100, config.getBackgroundMigrationExecutorThreads()); - assertTrue(config.isDeleteEnabled()); - assertTrue(config.isWriteEnabled()); - assertTrue(config.isReadEnabled()); + assertEquals(5000, config.getDynamoCrawlerScanPageSize()); } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java index 4fcad83f7..01fc6ab09 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java @@ -18,11 +18,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutorService; import org.junit.Before; import org.junit.Test; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest; public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterTest { @@ -58,18 +55,15 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT when(firstAccount.getUuid()).thenReturn(FIRST_UUID); when(secondAccount.getUuid()).thenReturn(SECOND_UUID); - when(accountsManager.getAllFrom(CHUNK_SIZE)).thenReturn(new AccountCrawlChunk(List.of(firstAccount), FIRST_UUID)); - when(accountsManager.getAllFrom(any(UUID.class), eq(CHUNK_SIZE))) + when(accountsManager.getAllFromDynamo(CHUNK_SIZE)).thenReturn( + new AccountCrawlChunk(List.of(firstAccount), FIRST_UUID)); + when(accountsManager.getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE))) .thenReturn(new AccountCrawlChunk(List.of(secondAccount), SECOND_UUID)) .thenReturn(new AccountCrawlChunk(Collections.emptyList(), null)); - final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class); - when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration); - when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn(mock(DynamicAccountsDynamoDbMigrationConfiguration.class)); - final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache(getRedisCluster()); accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, crawlerCache, List.of(listener), CHUNK_SIZE, - CHUNK_INTERVAL_MS, mock(ExecutorService.class), dynamicConfigurationManager); + CHUNK_INTERVAL_MS); } @Test @@ -78,9 +72,9 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT assertFalse(accountDatabaseCrawler.doPeriodicWork()); assertFalse(accountDatabaseCrawler.doPeriodicWork()); - verify(accountsManager).getAllFrom(CHUNK_SIZE); - verify(accountsManager).getAllFrom(FIRST_UUID, CHUNK_SIZE); - verify(accountsManager).getAllFrom(SECOND_UUID, CHUNK_SIZE); + verify(accountsManager).getAllFromDynamo(CHUNK_SIZE); + verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE); + verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE); verify(listener).onCrawlStart(); verify(listener).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount)); @@ -98,9 +92,9 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT assertFalse(accountDatabaseCrawler.doPeriodicWork()); assertFalse(accountDatabaseCrawler.doPeriodicWork()); - verify(accountsManager, times(2)).getAllFrom(CHUNK_SIZE); - verify(accountsManager).getAllFrom(FIRST_UUID, CHUNK_SIZE); - verify(accountsManager).getAllFrom(SECOND_UUID, CHUNK_SIZE); + verify(accountsManager, times(2)).getAllFromDynamo(CHUNK_SIZE); + verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE); + verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE); verify(listener, times(2)).onCrawlStart(); verify(listener, times(2)).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount)); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbMigrationCrawlerIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbMigrationCrawlerIntegrationTest.java deleted file mode 100644 index f72378d9f..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbMigrationCrawlerIntegrationTest.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.opentable.db.postgres.embedded.LiquibasePreparer; -import com.opentable.db.postgres.junit5.EmbeddedPostgresExtension; -import com.opentable.db.postgres.junit5.PreparedDbExtension; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; -import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager; -import org.whispersystems.textsecuregcm.limits.RateLimiter; -import org.whispersystems.textsecuregcm.limits.RateLimiters; -import org.whispersystems.textsecuregcm.redis.RedisClusterExtension; -import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; -import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; -import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; -import org.whispersystems.textsecuregcm.tests.util.SynchronousExecutorService; -import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; -import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest; -import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex; -import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; -import software.amazon.awssdk.services.dynamodb.model.KeyType; -import software.amazon.awssdk.services.dynamodb.model.Projection; -import software.amazon.awssdk.services.dynamodb.model.ProjectionType; -import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput; -import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; - -class AccountsDynamoDbMigrationCrawlerIntegrationTest { - - private static final int CHUNK_SIZE = 20; - private static final long CHUNK_INTERVAL_MS = 0; - - private static final String ACCOUNTS_TABLE_NAME = "accounts_test"; - private static final String KEYS_TABLE_NAME = "keys_test"; - private static final String MIGRATION_DELETED_ACCOUNTS_TABLE_NAME = "migration_deleted_accounts_test"; - private static final String MIGRATION_RETRY_ACCOUNTS_TABLE_NAME = "migration_retry_accounts_test"; - private static final String NUMBERS_TABLE_NAME = "numbers_test"; - private static final String VERIFICATION_CODE_TABLE_NAME = "verification_code_test"; - - @RegisterExtension - static final RedisClusterExtension REDIS_CLUSTER_EXTENSION = RedisClusterExtension.builder().build(); - - @RegisterExtension - static final DynamoDbExtension KEYS_DYNAMODB_EXTENSION = DynamoDbExtension.builder() - .tableName(KEYS_TABLE_NAME) - .hashKey("U") - .rangeKey("DK") - .attributeDefinition(AttributeDefinition.builder() - .attributeName("U") - .attributeType(ScalarAttributeType.B) - .build()) - .attributeDefinition(AttributeDefinition.builder() - .attributeName("DK") - .attributeType(ScalarAttributeType.B) - .build()) - .build(); - - @RegisterExtension - static final DynamoDbExtension VERIFICATION_CODE_DYNAMODB_EXTENSION = DynamoDbExtension.builder() - .tableName(VERIFICATION_CODE_TABLE_NAME) - .hashKey("P") - .attributeDefinition(AttributeDefinition.builder() - .attributeName("P") - .attributeType(ScalarAttributeType.S) - .build()) - .build(); - - @RegisterExtension - static PreparedDbExtension db = EmbeddedPostgresExtension - .preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml")); - - @RegisterExtension - static DynamoDbExtension ACCOUNTS_DYNAMODB_EXTENSION = DynamoDbExtension.builder() - .tableName(ACCOUNTS_TABLE_NAME) - .hashKey("U") - .attributeDefinition(AttributeDefinition.builder() - .attributeName("U") - .attributeType(ScalarAttributeType.B) - .build()) - .build(); - - private static final String NEEDS_RECONCILIATION_INDEX_NAME = "needs_reconciliation_test"; - - @RegisterExtension - static final DynamoDbExtension DELETED_ACCOUNTS_DYNAMODB_EXTENSION = DynamoDbExtension.builder() - .tableName("deleted_accounts_test") - .hashKey(DeletedAccounts.KEY_ACCOUNT_E164) - .attributeDefinition(AttributeDefinition.builder() - .attributeName(DeletedAccounts.KEY_ACCOUNT_E164) - .attributeType(ScalarAttributeType.S).build()) - .attributeDefinition(AttributeDefinition.builder() - .attributeName(DeletedAccounts.ATTR_NEEDS_CDS_RECONCILIATION) - .attributeType(ScalarAttributeType.N) - .build()) - .globalSecondaryIndex(GlobalSecondaryIndex.builder() - .indexName(NEEDS_RECONCILIATION_INDEX_NAME) - .keySchema( - KeySchemaElement.builder().attributeName(DeletedAccounts.KEY_ACCOUNT_E164).keyType(KeyType.HASH).build(), - KeySchemaElement.builder().attributeName(DeletedAccounts.ATTR_NEEDS_CDS_RECONCILIATION) - .keyType(KeyType.RANGE).build()) - .projection(Projection.builder().projectionType(ProjectionType.INCLUDE) - .nonKeyAttributes(DeletedAccounts.ATTR_ACCOUNT_UUID).build()) - .provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(10L).writeCapacityUnits(10L).build()) - .build()) - .build(); - - @RegisterExtension - static DynamoDbExtension DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION = DynamoDbExtension.builder() - .tableName("deleted_accounts_lock_test") - .hashKey(DeletedAccounts.KEY_ACCOUNT_E164) - .attributeDefinition(AttributeDefinition.builder() - .attributeName(DeletedAccounts.KEY_ACCOUNT_E164) - .attributeType(ScalarAttributeType.S).build()) - .build(); - - private DynamicAccountsDynamoDbMigrationConfiguration accountMigrationConfiguration; - - private AccountsManager accountsManager; - private AccountDatabaseCrawler accountDatabaseCrawler; - private Accounts accounts; - private AccountsDynamoDb accountsDynamoDb; - - @BeforeEach - void setUp() throws Exception { - - createAdditionalDynamoDbTables(); - - final DeletedAccounts deletedAccounts = new DeletedAccounts(DELETED_ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(), - DELETED_ACCOUNTS_DYNAMODB_EXTENSION.getTableName(), - NEEDS_RECONCILIATION_INDEX_NAME); - - final DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts, - DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION.getLegacyDynamoClient(), - DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION.getTableName()); - - MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts( - ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(), MIGRATION_DELETED_ACCOUNTS_TABLE_NAME); - - MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts( - (ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient()), - MIGRATION_RETRY_ACCOUNTS_TABLE_NAME); - - accountsDynamoDb = new AccountsDynamoDb( - ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(), - ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbAsyncClient(), - new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()), - ACCOUNTS_DYNAMODB_EXTENSION.getTableName(), - NUMBERS_TABLE_NAME, - migrationDeletedAccounts, - migrationRetryAccounts); - - final KeysDynamoDb keysDynamoDb = new KeysDynamoDb(KEYS_DYNAMODB_EXTENSION.getDynamoDbClient(), KEYS_TABLE_NAME); - - accounts = new Accounts(new FaultTolerantDatabase("accountsTest", - Jdbi.create(db.getTestDatabase()), - new CircuitBreakerConfiguration())); - - final DirectoryQueue directoryQueue = mock(DirectoryQueue.class); - - final RateLimiters rateLimiters = mock(RateLimiters.class); - when(rateLimiters.getVerifyLimiter()).thenReturn(mock(RateLimiter.class)); - - final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class); - final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class); - accountMigrationConfiguration = new DynamicAccountsDynamoDbMigrationConfiguration(); - accountMigrationConfiguration.setBackgroundMigrationEnabled(true); - accountMigrationConfiguration.setLogMismatches(true); - - when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration); - when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn(accountMigrationConfiguration); - - final ExperimentEnrollmentManager experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class); - when(experimentEnrollmentManager.isEnrolled(any(UUID.class), eq("accountsDynamoDbMigration"))).thenReturn(true); - - accountsManager = new AccountsManager( - accounts, - accountsDynamoDb, - REDIS_CLUSTER_EXTENSION.getRedisCluster(), - deletedAccountsManager, - directoryQueue, - keysDynamoDb, - mock(MessagesManager.class), - mock(MigrationMismatchedAccounts.class), - mock(UsernamesManager.class), - mock(ProfilesManager.class), - mock(StoredVerificationCodeManager.class), - mock(SecureStorageClient.class), - mock(SecureBackupClient.class), - experimentEnrollmentManager, - dynamicConfigurationManager); - - final AccountsDynamoDbMigrator dynamoDbMigrator = new AccountsDynamoDbMigrator(accountsDynamoDb, - dynamicConfigurationManager); - final PushFeedbackProcessor pushFeedbackProcessor = new PushFeedbackProcessor(accountsManager); - - final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache( - REDIS_CLUSTER_EXTENSION.getRedisCluster()); - - // Using a synchronous service doesn’t meaningfully impact the test - final ExecutorService chunkPreReadExecutorService = new SynchronousExecutorService(); - - accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, crawlerCache, - List.of(dynamoDbMigrator, pushFeedbackProcessor), CHUNK_SIZE, - CHUNK_INTERVAL_MS, chunkPreReadExecutorService, dynamicConfigurationManager); - } - - void createAdditionalDynamoDbTables() { - CreateTableRequest createNumbersTableRequest = CreateTableRequest.builder() - .tableName(NUMBERS_TABLE_NAME) - .keySchema(KeySchemaElement.builder() - .attributeName("P") - .keyType(KeyType.HASH) - .build()) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName("P") - .attributeType(ScalarAttributeType.S) - .build()) - .provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT) - .build(); - - ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createNumbersTableRequest); - - final CreateTableRequest createMigrationDeletedAccountsTableRequest = CreateTableRequest.builder() - .tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME) - .keySchema(KeySchemaElement.builder() - .attributeName("U") - .keyType(KeyType.HASH) - .build()) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName("U") - .attributeType(ScalarAttributeType.B) - .build()) - .provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT) - .build(); - - ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createMigrationDeletedAccountsTableRequest); - - final CreateTableRequest createMigrationRetryAccountsTableRequest = CreateTableRequest.builder() - .tableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME) - .keySchema(KeySchemaElement.builder() - .attributeName("U") - .keyType(KeyType.HASH) - .build()) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName("U") - .attributeType(ScalarAttributeType.B) - .build()) - .provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT) - .build(); - - ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createMigrationRetryAccountsTableRequest); - } -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbTest.java index 7d5367b93..dc32c9f46 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsDynamoDbTest.java @@ -24,10 +24,6 @@ import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import org.jdbi.v3.core.transaction.TransactionException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -36,21 +32,15 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; import org.whispersystems.textsecuregcm.entities.SignedPreKey; import org.whispersystems.textsecuregcm.util.AttributeValues; -import org.whispersystems.textsecuregcm.util.SystemMapper; -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException; import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest; import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; import software.amazon.awssdk.services.dynamodb.model.KeyType; -import software.amazon.awssdk.services.dynamodb.model.ReturnValue; import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; -import software.amazon.awssdk.services.dynamodb.model.ScanRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanResponse; import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest; import software.amazon.awssdk.services.dynamodb.model.TransactionConflictException; import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest; @@ -59,8 +49,6 @@ class AccountsDynamoDbTest { private static final String ACCOUNTS_TABLE_NAME = "accounts_test"; private static final String NUMBERS_TABLE_NAME = "numbers_test"; - private static final String MIGRATION_DELETED_ACCOUNTS_TABLE_NAME = "migration_deleted_accounts_test"; - private static final String MIGRATION_RETRY_ACCOUNTS_TABLE_NAME = "migration_retry_accounts_test"; @RegisterExtension static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder() @@ -91,50 +79,11 @@ class AccountsDynamoDbTest { dynamoDbExtension.getDynamoDbClient().createTable(createNumbersTableRequest); - final CreateTableRequest createMigrationDeletedAccountsTableRequest = CreateTableRequest.builder() - .tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME) - .keySchema(KeySchemaElement.builder() - .attributeName(MigrationDeletedAccounts.KEY_UUID) - .keyType(KeyType.HASH) - .build()) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName(MigrationDeletedAccounts.KEY_UUID) - .attributeType(ScalarAttributeType.B) - .build()) - .provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT) - .build(); - - dynamoDbExtension.getDynamoDbClient().createTable(createMigrationDeletedAccountsTableRequest); - - MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts( - dynamoDbExtension.getDynamoDbClient(), MIGRATION_DELETED_ACCOUNTS_TABLE_NAME); - - final CreateTableRequest createMigrationRetryAccountsTableRequest = CreateTableRequest.builder() - .tableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME) - .keySchema(KeySchemaElement.builder() - .attributeName(MigrationRetryAccounts.KEY_UUID) - .keyType(KeyType.HASH) - .build()) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName(MigrationRetryAccounts.KEY_UUID) - .attributeType(ScalarAttributeType.B) - .build()) - .provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT) - .build(); - - dynamoDbExtension.getDynamoDbClient().createTable(createMigrationRetryAccountsTableRequest); - - MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts((dynamoDbExtension.getDynamoDbClient()), - MIGRATION_RETRY_ACCOUNTS_TABLE_NAME); - this.accountsDynamoDb = new AccountsDynamoDb( dynamoDbExtension.getDynamoDbClient(), - dynamoDbExtension.getDynamoDbAsyncClient(), - new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()), dynamoDbExtension.getTableName(), - NUMBERS_TABLE_NAME, - migrationDeletedAccounts, - migrationRetryAccounts); + NUMBERS_TABLE_NAME + ); } @Test @@ -283,15 +232,13 @@ class AccountsDynamoDbTest { void testUpdateWithMockTransactionConflictException() { final DynamoDbClient dynamoDbClient = mock(DynamoDbClient.class); - accountsDynamoDb = new AccountsDynamoDb(dynamoDbClient, mock(DynamoDbAsyncClient.class), - new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()), - dynamoDbExtension.getTableName(), NUMBERS_TABLE_NAME, mock(MigrationDeletedAccounts.class), - mock(MigrationRetryAccounts.class)); + accountsDynamoDb = new AccountsDynamoDb(dynamoDbClient, + dynamoDbExtension.getTableName(), NUMBERS_TABLE_NAME); when(dynamoDbClient.updateItem(any(UpdateItemRequest.class))) .thenThrow(TransactionConflictException.class); - Device device = generateDevice (1 ); + Device device = generateDevice(1); Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device)); assertThatThrownBy(() -> accountsDynamoDb.update(account)).isInstanceOfAny(ContestedOptimisticLockException.class); @@ -376,33 +323,6 @@ class AccountsDynamoDbTest { verifyStoredState(recreatedAccount.getNumber(), recreatedAccount.getUuid(), accountsDynamoDb.get(recreatedAccount.getUuid()).get(), recreatedAccount); } - - verifyRecentlyDeletedAccountsTableItemCount(1); - - Map primaryKey = MigrationDeletedAccounts.primaryKey(deletedAccount.getUuid()); - assertThat(dynamoDbExtension.getDynamoDbClient().getItem(GetItemRequest.builder() - .tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME) - .key(Map.of(MigrationDeletedAccounts.KEY_UUID, primaryKey.get(MigrationDeletedAccounts.KEY_UUID))) - .build())) - .isNotNull(); - - accountsDynamoDb.deleteRecentlyDeletedUuids(); - - verifyRecentlyDeletedAccountsTableItemCount(0); - } - - private void verifyRecentlyDeletedAccountsTableItemCount(int expectedItemCount) { - int totalItems = 0; - - for (ScanResponse page : dynamoDbExtension.getDynamoDbClient().scanPaginator(ScanRequest.builder() - .tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME) - .build())) { - for (Map item : page.items()) { - totalItems++; - } - } - - assertThat(totalItems).isEqualTo(expectedItemCount); } @Test @@ -437,9 +357,8 @@ class AccountsDynamoDbTest { when(client.updateItem(any(UpdateItemRequest.class))) .thenThrow(RuntimeException.class); - AccountsDynamoDb accounts = new AccountsDynamoDb(client, mock(DynamoDbAsyncClient.class), mock(ThreadPoolExecutor.class), ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME, mock( - MigrationDeletedAccounts.class), mock(MigrationRetryAccounts.class)); - Account account = generateAccount("+14151112222", UUID.randomUUID()); + AccountsDynamoDb accounts = new AccountsDynamoDb(client, ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME); + Account account = generateAccount("+14151112222", UUID.randomUUID()); try { accounts.update(account); @@ -472,42 +391,6 @@ class AccountsDynamoDbTest { } } - @Test - void testMigrate() throws ExecutionException, InterruptedException { - - Device device = generateDevice (1 ); - UUID firstUuid = UUID.randomUUID(); - Account account = generateAccount("+14151112222", firstUuid, Collections.singleton(device)); - - boolean migrated = accountsDynamoDb.migrate(account).get(); - - assertThat(migrated).isTrue(); - - verifyStoredState("+14151112222", account.getUuid(), account, true); - - migrated = accountsDynamoDb.migrate(account).get(); - - assertThat(migrated).isFalse(); - - verifyStoredState("+14151112222", account.getUuid(), account, true); - - UUID secondUuid = UUID.randomUUID(); - - device = generateDevice(1); - Account accountRemigrationWithDifferentUuid = generateAccount("+14151112222", secondUuid, Collections.singleton(device)); - - migrated = accountsDynamoDb.migrate(accountRemigrationWithDifferentUuid).get(); - - assertThat(migrated).isFalse(); - verifyStoredState("+14151112222", firstUuid, account, true); - - account.setVersion(account.getVersion() + 1); - - migrated = accountsDynamoDb.migrate(account).get(); - - assertThat(migrated).isTrue(); - } - @Test void testCanonicallyDiscoverableSet() { Device device = generateDevice(1); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java index 284a7e0b6..24cc4292e 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java @@ -23,7 +23,6 @@ import com.opentable.db.postgres.junit5.PreparedDbExtension; import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands; import java.io.IOException; import java.time.Instant; -import java.util.List; import java.util.Optional; import java.util.Random; import java.util.UUID; @@ -34,18 +33,14 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Stream; -import org.jdbi.v3.core.Jdbi; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.mockito.ArgumentCaptor; import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials; -import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.entities.AccountAttributes; import org.whispersystems.textsecuregcm.entities.SignedPreKey; -import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; @@ -76,8 +71,6 @@ class AccountsManagerConcurrentModificationIntegrationTest { .build()) .build(); - private Accounts accounts; - private AccountsDynamoDb accountsDynamoDb; private AccountsManager accountsManager; @@ -108,22 +101,9 @@ class AccountsManagerConcurrentModificationIntegrationTest { accountsDynamoDb = new AccountsDynamoDb( dynamoDbExtension.getDynamoDbClient(), - dynamoDbExtension.getDynamoDbAsyncClient(), - new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()), dynamoDbExtension.getTableName(), - NUMBERS_TABLE_NAME, - mock(MigrationDeletedAccounts.class), - mock(MigrationRetryAccounts.class)); - - { - final CircuitBreakerConfiguration circuitBreakerConfiguration = new CircuitBreakerConfiguration(); - circuitBreakerConfiguration.setIgnoredExceptions(List.of("org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException")); - FaultTolerantDatabase faultTolerantDatabase = new FaultTolerantDatabase("accountsTest", - Jdbi.create(db.getTestDatabase()), - circuitBreakerConfiguration); - - accounts = new Accounts(faultTolerantDatabase); - } + NUMBERS_TABLE_NAME + ); { final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class); @@ -131,17 +111,6 @@ class AccountsManagerConcurrentModificationIntegrationTest { DynamicConfiguration dynamicConfiguration = new DynamicConfiguration(); when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration); - final ExperimentEnrollmentManager experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class); - - final DynamicAccountsDynamoDbMigrationConfiguration config = dynamicConfiguration - .getAccountsDynamoDbMigrationConfiguration(); - - config.setDeleteEnabled(true); - config.setReadEnabled(true); - config.setWriteEnabled(true); - - when(experimentEnrollmentManager.isEnrolled(any(UUID.class), anyString())).thenReturn(true); - commands = mock(RedisAdvancedClusterCommands.class); final DeletedAccountsManager deletedAccountsManager = mock(DeletedAccountsManager.class); @@ -153,20 +122,17 @@ class AccountsManagerConcurrentModificationIntegrationTest { }).when(deletedAccountsManager).lockAndTake(anyString(), any()); accountsManager = new AccountsManager( - accounts, accountsDynamoDb, RedisClusterHelper.buildMockRedisCluster(commands), deletedAccountsManager, mock(DirectoryQueue.class), mock(KeysDynamoDb.class), mock(MessagesManager.class), - mock(MigrationMismatchedAccounts.class), mock(UsernamesManager.class), mock(ProfilesManager.class), mock(StoredVerificationCodeManager.class), mock(SecureStorageClient.class), mock(SecureBackupClient.class), - experimentEnrollmentManager, dynamicConfigurationManager); } } @@ -225,14 +191,12 @@ class AccountsManagerConcurrentModificationIntegrationTest { ).join(); final Account managerAccount = accountsManager.get(uuid).get(); - final Account dbAccount = accounts.get(uuid).get(); final Account dynamoAccount = accountsDynamoDb.get(uuid).get(); final Account redisAccount = getLastAccountFromRedisMock(commands); Stream.of( new Pair<>("manager", managerAccount), - new Pair<>("db", dbAccount), new Pair<>("dynamo", dynamoAccount), new Pair<>("redis", redisAccount) ).forEach(pair -> diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationDeletedAccountsTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationDeletedAccountsTest.java deleted file mode 100644 index 272131acd..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationDeletedAccountsTest.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.whispersystems.textsecuregcm.storage; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import java.util.UUID; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; -import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; - -class MigrationDeletedAccountsTest { - - @RegisterExtension - static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder() - .tableName("deleted_accounts_test") - .hashKey(MigrationDeletedAccounts.KEY_UUID) - .attributeDefinition(AttributeDefinition.builder() - .attributeName(MigrationDeletedAccounts.KEY_UUID) - .attributeType(ScalarAttributeType.B) - .build()) - .build(); - - @Test - void test() { - - final MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDbClient(), - dynamoDbExtension.getTableName()); - - UUID firstUuid = UUID.randomUUID(); - UUID secondUuid = UUID.randomUUID(); - - assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().isEmpty()); - - migrationDeletedAccounts.put(firstUuid); - migrationDeletedAccounts.put(secondUuid); - - assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().containsAll(List.of(firstUuid, secondUuid))); - - migrationDeletedAccounts.delete(List.of(firstUuid, secondUuid)); - - assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().isEmpty()); - } -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchAccountsTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchAccountsTest.java deleted file mode 100644 index 987eae4ff..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationMismatchAccountsTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.time.Clock; -import java.util.List; -import java.util.UUID; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; -import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; - -class MigrationMismatchAccountsTest { - - @RegisterExtension - static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder() - .tableName("account_migration_mismatches_test") - .hashKey(MigrationRetryAccounts.KEY_UUID) - .attributeDefinition(AttributeDefinition.builder() - .attributeName(MigrationRetryAccounts.KEY_UUID) - .attributeType(ScalarAttributeType.B) - .build()) - .build(); - - @Test - void test() { - - final Clock clock = mock(Clock.class); - when(clock.millis()).thenReturn(0L); - - final MigrationMismatchedAccounts migrationMismatchedAccounts = new MigrationMismatchedAccounts( - dynamoDbExtension.getDynamoDbClient(), - dynamoDbExtension.getTableName(), clock); - - UUID firstUuid = UUID.randomUUID(); - UUID secondUuid = UUID.randomUUID(); - - assertTrue(migrationMismatchedAccounts.getUuids(10).isEmpty()); - - migrationMismatchedAccounts.put(firstUuid); - migrationMismatchedAccounts.put(secondUuid); - - assertTrue(migrationMismatchedAccounts.getUuids(10).isEmpty()); - - when(clock.millis()).thenReturn(MigrationMismatchedAccounts.MISMATCH_CHECK_DELAY_MILLIS); - - assertTrue(migrationMismatchedAccounts.getUuids(10).containsAll(List.of(firstUuid, secondUuid))); - } -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccountsTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccountsTest.java deleted file mode 100644 index 7af23fa90..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MigrationRetryAccountsTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.whispersystems.textsecuregcm.storage; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import java.util.UUID; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; -import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; - -class MigrationRetryAccountsTest { - - @RegisterExtension - static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder() - .tableName("account_migration_errors_test") - .hashKey(MigrationRetryAccounts.KEY_UUID) - .attributeDefinition(AttributeDefinition.builder() - .attributeName(MigrationRetryAccounts.KEY_UUID) - .attributeType(ScalarAttributeType.B) - .build()) - .build(); - - @Test - void test() { - - final MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(dynamoDbExtension.getDynamoDbClient(), - dynamoDbExtension.getTableName()); - - UUID firstUuid = UUID.randomUUID(); - UUID secondUuid = UUID.randomUUID(); - - assertTrue(migrationRetryAccounts.getUuids(10).isEmpty()); - - migrationRetryAccounts.put(firstUuid); - migrationRetryAccounts.put(secondUuid); - - assertTrue(migrationRetryAccounts.getUuids(10).containsAll(List.of(firstUuid, secondUuid))); - } -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java index 6b2743d7b..0b513d8ca 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java @@ -16,7 +16,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.util.Arrays; @@ -25,10 +24,7 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutorService; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; +import org.junit.jupiter.api.Test; import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler; @@ -36,7 +32,6 @@ import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerRestartException; import org.whispersystems.textsecuregcm.storage.AccountsManager; -import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; class AccountDatabaseCrawlerTest { @@ -55,22 +50,14 @@ class AccountDatabaseCrawlerTest { private final ExecutorService chunkPreReadExecutorService = mock(ExecutorService.class); - private final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class); - private final AccountDatabaseCrawler crawler = new AccountDatabaseCrawler(accounts, cache, Arrays.asList(listener), - CHUNK_SIZE, CHUNK_INTERVAL_MS, chunkPreReadExecutorService, dynamicConfigurationManager); - private DynamicAccountsDynamoDbMigrationConfiguration dynamicAccountsDynamoDbMigrationConfiguration; + CHUNK_SIZE, CHUNK_INTERVAL_MS); @BeforeEach void setup() { when(account1.getUuid()).thenReturn(ACCOUNT1); when(account2.getUuid()).thenReturn(ACCOUNT2); - when(accounts.getAllFrom(anyInt())).thenReturn(new AccountCrawlChunk(Arrays.asList(account1, account2), ACCOUNT2)); - when(accounts.getAllFrom(eq(ACCOUNT1), anyInt())).thenReturn( - new AccountCrawlChunk(Arrays.asList(account2), ACCOUNT2)); - when(accounts.getAllFrom(eq(ACCOUNT2), anyInt())).thenReturn(new AccountCrawlChunk(Collections.emptyList(), null)); - when(accounts.getAllFromDynamo(anyInt())).thenReturn( new AccountCrawlChunk(Arrays.asList(account1, account2), ACCOUNT2)); when(accounts.getAllFromDynamo(eq(ACCOUNT1), anyInt())).thenReturn( @@ -81,17 +68,10 @@ class AccountDatabaseCrawlerTest { when(cache.claimActiveWork(any(), anyLong())).thenReturn(true); when(cache.isAccelerated()).thenReturn(false); - final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class); - when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration); - dynamicAccountsDynamoDbMigrationConfiguration = mock(DynamicAccountsDynamoDbMigrationConfiguration.class); - when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn( - dynamicAccountsDynamoDbMigrationConfiguration); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testCrawlStart(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException { - when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo); + @Test + void testCrawlStart() throws AccountDatabaseCrawlerRestartException { when(cache.getLastUuid()).thenReturn(Optional.empty()); when(cache.getLastUuidDynamo()).thenReturn(Optional.empty()); @@ -99,20 +79,15 @@ class AccountDatabaseCrawlerTest { assertThat(accelerated).isFalse(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(useDynamo ? 0 : 1)).getLastUuid(); - verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo(); + verify(cache, times(0)).getLastUuid(); + verify(cache, times(1)).getLastUuidDynamo(); verify(listener, times(1)).onCrawlStart(); - if (useDynamo) { - verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE)); - verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE)); - } else { - verify(accounts, times(1)).getAllFrom(eq(CHUNK_SIZE)); - verify(accounts, times(0)).getAllFrom(any(UUID.class), eq(CHUNK_SIZE)); - } + verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE)); + verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE)); verify(account1, times(0)).getUuid(); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.empty()), eq(Arrays.asList(account1, account2))); - verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2))); - verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); + verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2))); + verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); @@ -123,10 +98,8 @@ class AccountDatabaseCrawlerTest { verifyNoMoreInteractions(cache); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testCrawlChunk(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException { - when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo); + @Test + void testCrawlChunk() throws AccountDatabaseCrawlerRestartException { when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1)); @@ -134,18 +107,13 @@ class AccountDatabaseCrawlerTest { assertThat(accelerated).isFalse(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(useDynamo ? 0: 1)).getLastUuid(); - verify(cache, times(useDynamo ? 1: 0)).getLastUuidDynamo(); - if (useDynamo) { - verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); - } else { - verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE)); - } + verify(cache, times(0)).getLastUuid(); + verify(cache, times(1)).getLastUuidDynamo(); + verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); + verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); - verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2))); - verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); + verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2))); + verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); @@ -157,46 +125,8 @@ class AccountDatabaseCrawlerTest { verifyNoMoreInteractions(cache); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testCrawlChunk_useDynamoDedicatedMigrationCrawler(final boolean dedicatedMigrationCrawler) throws Exception { - crawler.setDedicatedDynamoMigrationCrawler(dedicatedMigrationCrawler); - - when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(true); - when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); - when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1)); - - boolean accelerated = crawler.doPeriodicWork(); - assertThat(accelerated).isFalse(); - - verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(dedicatedMigrationCrawler ? 1 : 0)).getLastUuid(); - verify(cache, times(dedicatedMigrationCrawler ? 0 : 1)).getLastUuidDynamo(); - if (dedicatedMigrationCrawler) { - verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE)); - } else { - verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); - } - verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); - verify(cache, times(dedicatedMigrationCrawler ? 1 : 0)).setLastUuid(eq(Optional.of(ACCOUNT2))); - verify(cache, times(dedicatedMigrationCrawler ? 0 : 1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); - verify(cache, times(1)).isAccelerated(); - verify(cache, times(1)).releaseActiveWork(any(String.class)); - - verifyNoInteractions(account1); - - verifyNoMoreInteractions(account2); - verifyNoMoreInteractions(accounts); - verifyNoMoreInteractions(listener); - verifyNoMoreInteractions(cache); - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testCrawlChunkAccelerated(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException { - when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo); + @Test + void testCrawlChunkAccelerated() throws AccountDatabaseCrawlerRestartException { when(cache.isAccelerated()).thenReturn(true); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1)); @@ -205,22 +135,17 @@ class AccountDatabaseCrawlerTest { assertThat(accelerated).isTrue(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(useDynamo ? 0 : 1)).getLastUuid(); - verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo(); - if (useDynamo) { - verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); - } else { - verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE)); - } + verify(cache, times(0)).getLastUuid(); + verify(cache, times(1)).getLastUuidDynamo(); + verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); + verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); - verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2))); - verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); + verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2))); + verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); - verifyZeroInteractions(account1); + verifyNoInteractions(account1); verifyNoMoreInteractions(account2); verifyNoMoreInteractions(accounts); @@ -228,36 +153,30 @@ class AccountDatabaseCrawlerTest { verifyNoMoreInteractions(cache); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testCrawlChunkRestart(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException { - when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo); + @Test + void testCrawlChunkRestart() throws AccountDatabaseCrawlerRestartException { when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1)); - doThrow(AccountDatabaseCrawlerRestartException.class).when(listener).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); + doThrow(AccountDatabaseCrawlerRestartException.class).when(listener) + .timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); boolean accelerated = crawler.doPeriodicWork(); assertThat(accelerated).isFalse(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(useDynamo ? 0 : 1)).getLastUuid(); - verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo(); - if (useDynamo) { - verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); - } else { - verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE)); - } + verify(cache, times(0)).getLastUuid(); + verify(cache, times(1)).getLastUuidDynamo(); + verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); + verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); verify(account2, times(0)).getNumber(); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); - verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.empty())); - verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.empty())); + verify(cache, times(0)).setLastUuid(eq(Optional.empty())); + verify(cache, times(1)).setLastUuidDynamo(eq(Optional.empty())); verify(cache, times(1)).setAccelerated(false); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); - verifyZeroInteractions(account1); + verifyNoInteractions(account1); verifyNoMoreInteractions(account2); verifyNoMoreInteractions(accounts); @@ -265,10 +184,8 @@ class AccountDatabaseCrawlerTest { verifyNoMoreInteractions(cache); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testCrawlEnd(final boolean useDynamo) { - when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo); + @Test + void testCrawlEnd() { when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2)); when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT2)); @@ -276,26 +193,21 @@ class AccountDatabaseCrawlerTest { assertThat(accelerated).isFalse(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(useDynamo ? 0 : 1)).getLastUuid(); - verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo(); - if (useDynamo) { - verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE)); - } else { - verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFrom(eq(ACCOUNT2), eq(CHUNK_SIZE)); - } + verify(cache, times(0)).getLastUuid(); + verify(cache, times(1)).getLastUuidDynamo(); + verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); + verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE)); verify(account1, times(0)).getNumber(); verify(account2, times(0)).getNumber(); verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2))); - verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.empty())); - verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.empty())); + verify(cache, times(0)).setLastUuid(eq(Optional.empty())); + verify(cache, times(1)).setLastUuidDynamo(eq(Optional.empty())); verify(cache, times(1)).setAccelerated(false); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); - verifyZeroInteractions(account1); - verifyZeroInteractions(account2); + verifyNoInteractions(account1); + verifyNoInteractions(account2); verifyNoMoreInteractions(accounts); verifyNoMoreInteractions(listener); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsManagerTest.java index d6845a114..6053411ad 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsManagerTest.java @@ -33,6 +33,7 @@ import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -40,16 +41,13 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.stubbing.Answer; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.entities.AccountAttributes; import org.whispersystems.textsecuregcm.entities.SignedPreKey; -import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; import org.whispersystems.textsecuregcm.storage.Account; -import org.whispersystems.textsecuregcm.storage.Accounts; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException; @@ -59,7 +57,6 @@ import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.storage.KeysDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesManager; -import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts; import org.whispersystems.textsecuregcm.storage.ProfilesManager; import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager; import org.whispersystems.textsecuregcm.storage.UsernamesManager; @@ -68,12 +65,10 @@ import org.whispersystems.textsecuregcm.tests.util.RedisClusterHelper; class AccountsManagerTest { - private Accounts accounts; private AccountsDynamoDb accountsDynamoDb; private DeletedAccountsManager deletedAccountsManager; private DirectoryQueue directoryQueue; private DynamicConfigurationManager dynamicConfigurationManager; - private ExperimentEnrollmentManager experimentEnrollmentManager; private KeysDynamoDb keys; private MessagesManager messagesManager; private ProfilesManager profilesManager; @@ -91,12 +86,10 @@ class AccountsManagerTest { @BeforeEach void setup() throws InterruptedException { - accounts = mock(Accounts.class); accountsDynamoDb = mock(AccountsDynamoDb.class); deletedAccountsManager = mock(DeletedAccountsManager.class); directoryQueue = mock(DirectoryQueue.class); dynamicConfigurationManager = mock(DynamicConfigurationManager.class); - experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class); keys = mock(KeysDynamoDb.class); messagesManager = mock(MessagesManager.class); profilesManager = mock(ProfilesManager.class); @@ -114,30 +107,24 @@ class AccountsManagerTest { }).when(deletedAccountsManager).lockAndTake(anyString(), any()); accountsManager = new AccountsManager( - accounts, accountsDynamoDb, RedisClusterHelper.buildMockRedisCluster(commands), deletedAccountsManager, directoryQueue, keys, messagesManager, - mock(MigrationMismatchedAccounts.class), mock(UsernamesManager.class), profilesManager, mock(StoredVerificationCodeManager.class), mock(SecureStorageClient.class), mock(SecureBackupClient.class), - experimentEnrollmentManager, dynamicConfigurationManager); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testGetAccountByNumberInCache(final boolean dynamoEnabled) { + @Test + void testGetAccountByNumberInCache() { UUID uuid = UUID.randomUUID(); - enableDynamo(dynamoEnabled); - when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(uuid.toString()); when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}"); @@ -150,31 +137,14 @@ class AccountsManagerTest { verify(commands, times(1)).get(eq("AccountMap::+14152222222")); verify(commands, times(1)).get(eq("Account3::" + uuid)); verifyNoMoreInteractions(commands); - verifyNoMoreInteractions(accounts); verifyNoInteractions(accountsDynamoDb); } - private void enableDynamo(boolean dynamoEnabled) { - final DynamicAccountsDynamoDbMigrationConfiguration config = dynamicConfigurationManager.getConfiguration() - .getAccountsDynamoDbMigrationConfiguration(); - - config.setDeleteEnabled(dynamoEnabled); - config.setReadEnabled(dynamoEnabled); - config.setWriteEnabled(dynamoEnabled); - - when(experimentEnrollmentManager.isEnrolled(any(UUID.class), anyString())) - .thenReturn(dynamoEnabled); - - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testGetAccountByUuidInCache(boolean dynamoEnabled) { + @Test + void testGetAccountByUuidInCache() { UUID uuid = UUID.randomUUID(); - enableDynamo(dynamoEnabled); - when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}"); Optional account = accountsManager.get(uuid); @@ -186,22 +156,19 @@ class AccountsManagerTest { verify(commands, times(1)).get(eq("Account3::" + uuid)); verifyNoMoreInteractions(commands); - verifyNoMoreInteractions(accounts); verifyNoInteractions(accountsDynamoDb); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testGetAccountByNumberNotInCache(boolean dynamoEnabled) { + @Test + void testGetAccountByNumberNotInCache() { + final boolean dynamoEnabled = true; UUID uuid = UUID.randomUUID(); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - enableDynamo(dynamoEnabled); - when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(null); - when(accounts.get(eq("+14152222222"))).thenReturn(Optional.of(account)); + when(accountsDynamoDb.get(eq("+14152222222"))).thenReturn(Optional.of(account)); Optional retrieved = accountsManager.get("+14152222222"); @@ -213,24 +180,18 @@ class AccountsManagerTest { verify(commands, times(1)).set(eq("Account3::" + uuid), anyString()); verifyNoMoreInteractions(commands); - verify(accounts, times(1)).get(eq("+14152222222")); - verifyNoMoreInteractions(accounts); - - verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()) - .get(eq("+14152222222")); + verify(accountsDynamoDb, times(1)).get(eq("+14152222222")); verifyNoMoreInteractions(accountsDynamoDb); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testGetAccountByUuidNotInCache(boolean dynamoEnabled) { + @Test + void testGetAccountByUuidNotInCache() { + final boolean dynamoEnabled = true; UUID uuid = UUID.randomUUID(); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - enableDynamo(dynamoEnabled); - when(commands.get(eq("Account3::" + uuid))).thenReturn(null); - when(accounts.get(eq(uuid))).thenReturn(Optional.of(account)); + when(accountsDynamoDb.get(eq(uuid))).thenReturn(Optional.of(account)); Optional retrieved = accountsManager.get(uuid); @@ -242,25 +203,19 @@ class AccountsManagerTest { verify(commands, times(1)).set(eq("Account3::" + uuid), anyString()); verifyNoMoreInteractions(commands); - verify(accounts, times(1)).get(eq(uuid)); - verifyNoMoreInteractions(accounts); - - verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq(uuid)); + verify(accountsDynamoDb, times(1)).get(eq(uuid)); verifyNoMoreInteractions(accountsDynamoDb); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testGetAccountByNumberBrokenCache(boolean dynamoEnabled) { - UUID uuid = UUID.randomUUID(); - Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - - enableDynamo(dynamoEnabled); + @Test + void testGetAccountByNumberBrokenCache() { + UUID uuid = UUID.randomUUID(); + Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); when(commands.get(eq("AccountMap::+14152222222"))).thenThrow(new RedisException("Connection lost!")); - when(accounts.get(eq("+14152222222"))).thenReturn(Optional.of(account)); + when(accountsDynamoDb.get(eq("+14152222222"))).thenReturn(Optional.of(account)); - Optional retrieved = accountsManager.get("+14152222222"); + Optional retrieved = accountsManager.get("+14152222222"); assertTrue(retrieved.isPresent()); assertSame(retrieved.get(), account); @@ -270,25 +225,20 @@ class AccountsManagerTest { verify(commands, times(1)).set(eq("Account3::" + uuid), anyString()); verifyNoMoreInteractions(commands); - verify(accounts, times(1)).get(eq("+14152222222")); - verifyNoMoreInteractions(accounts); - - verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq("+14152222222")); + verify(accountsDynamoDb, times(1)).get(eq("+14152222222")); verifyNoMoreInteractions(accountsDynamoDb); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testGetAccountByUuidBrokenCache(boolean dynamoEnabled) { - UUID uuid = UUID.randomUUID(); - Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - - enableDynamo(dynamoEnabled); + @Test + void testGetAccountByUuidBrokenCache() { + final boolean dynamoEnabled = true; + UUID uuid = UUID.randomUUID(); + Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); when(commands.get(eq("Account3::" + uuid))).thenThrow(new RedisException("Connection lost!")); - when(accounts.get(eq(uuid))).thenReturn(Optional.of(account)); + when(accountsDynamoDb.get(eq(uuid))).thenReturn(Optional.of(account)); - Optional retrieved = accountsManager.get(uuid); + Optional retrieved = accountsManager.get(uuid); assertTrue(retrieved.isPresent()); assertSame(retrieved.get(), account); @@ -298,26 +248,25 @@ class AccountsManagerTest { verify(commands, times(1)).set(eq("Account3::" + uuid), anyString()); verifyNoMoreInteractions(commands); - verify(accounts, times(1)).get(eq(uuid)); - verifyNoMoreInteractions(accounts); - - verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq(uuid)); + verify(accountsDynamoDb, times(1)).get(eq(uuid)); verifyNoMoreInteractions(accountsDynamoDb); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testUpdate_dynamoDbMigration(boolean dynamoEnabled) throws IOException { + // TODO delete + @Disabled("migration specific") + @Test + void testUpdate_dynamoDbMigration() throws IOException { + UUID uuid = UUID.randomUUID(); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - enableDynamo(dynamoEnabled); - when(commands.get(eq("Account3::" + uuid))).thenReturn(null); // database fetches should always return new instances - when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); - when(accountsDynamoDb.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); - doAnswer(ACCOUNT_UPDATE_ANSWER).when(accounts).update(any(Account.class)); + when(accountsDynamoDb.get(uuid)).thenReturn( + Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); + when(accountsDynamoDb.get(uuid)).thenReturn( + Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); + doAnswer(ACCOUNT_UPDATE_ANSWER).when(accountsDynamoDb).update(any(Account.class)); Account updatedAccount = accountsManager.update(account, a -> a.setProfileName("name")); @@ -325,17 +274,13 @@ class AccountsManagerTest { assertNotSame(updatedAccount, account); - verify(accounts, times(1)).update(account); - verifyNoMoreInteractions(accounts); + verify(accountsDynamoDb, times(1)).update(account); + verifyNoMoreInteractions(accountsDynamoDb); - if (dynamoEnabled) { - ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Account.class); - verify(accountsDynamoDb, times(1)).update(argumentCaptor.capture()); - assertEquals(uuid, argumentCaptor.getValue().getUuid()); - } else { - verify(accountsDynamoDb, never()).update(any()); - } - verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(uuid); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Account.class); + verify(accountsDynamoDb, times(1)).update(argumentCaptor.capture()); + assertEquals(uuid, argumentCaptor.getValue().getUuid()); + verify(accountsDynamoDb, times(1)).get(uuid); verifyNoMoreInteractions(accountsDynamoDb); ArgumentCaptor redisSetArgumentCapture = ArgumentCaptor.forClass(String.class); @@ -347,25 +292,26 @@ class AccountsManagerTest { // uuid is @JsonIgnore, so we need to set it for compareAccounts to work accountCached.setUuid(uuid); - assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(updatedAccount), Optional.of(accountCached))); + assertEquals(Optional.empty(), + accountsManager.compareAccounts(Optional.of(updatedAccount), Optional.of(accountCached))); } + // TODO delete + @Disabled("migration specific") @Test void testUpdate_dynamoMissing() { - UUID uuid = UUID.randomUUID(); - Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - - enableDynamo(true); + UUID uuid = UUID.randomUUID(); + Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); when(commands.get(eq("Account3::" + uuid))).thenReturn(null); when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty()); - doAnswer(ACCOUNT_UPDATE_ANSWER).when(accounts).update(any()); doAnswer(ACCOUNT_UPDATE_ANSWER).when(accountsDynamoDb).update(any()); - Account updatedAccount = accountsManager.update(account, a -> {}); + Account updatedAccount = accountsManager.update(account, a -> { + }); - verify(accounts, times(1)).update(account); - verifyNoMoreInteractions(accounts); + verify(accountsDynamoDb, times(1)).update(account); + verifyNoMoreInteractions(accountsDynamoDb); verify(accountsDynamoDb, never()).update(account); verify(accountsDynamoDb, times(1)).get(uuid); @@ -376,19 +322,19 @@ class AccountsManagerTest { @Test void testUpdate_optimisticLockingFailure() { - UUID uuid = UUID.randomUUID(); - Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - - enableDynamo(true); + UUID uuid = UUID.randomUUID(); + Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); when(commands.get(eq("Account3::" + uuid))).thenReturn(null); - when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); + when(accountsDynamoDb.get(uuid)).thenReturn( + Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); doThrow(ContestedOptimisticLockException.class) .doAnswer(ACCOUNT_UPDATE_ANSWER) - .when(accounts).update(any()); + .when(accountsDynamoDb).update(any()); - when(accountsDynamoDb.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); + when(accountsDynamoDb.get(uuid)).thenReturn( + Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); doThrow(ContestedOptimisticLockException.class) .doAnswer(ACCOUNT_UPDATE_ANSWER) .when(accountsDynamoDb).update(any()); @@ -398,12 +344,7 @@ class AccountsManagerTest { assertEquals(1, account.getVersion()); assertEquals("name", account.getProfileName()); - verify(accounts, times(1)).get(uuid); - verify(accounts, times(2)).update(any()); - verifyNoMoreInteractions(accounts); - - // dynamo has an extra get() because the account is fetched before every update - verify(accountsDynamoDb, times(2)).get(uuid); + verify(accountsDynamoDb, times(1)).get(uuid); verify(accountsDynamoDb, times(2)).update(any()); verifyNoMoreInteractions(accountsDynamoDb); } @@ -413,8 +354,6 @@ class AccountsManagerTest { UUID uuid = UUID.randomUUID(); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - enableDynamo(true); - when(commands.get(eq("Account3::" + uuid))).thenReturn(null); when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty()) .thenReturn(Optional.of(account)); @@ -422,10 +361,7 @@ class AccountsManagerTest { accountsManager.update(account, a -> {}); - verify(accounts, times(1)).update(account); - verifyNoMoreInteractions(accounts); - - verify(accountsDynamoDb, times(1)).get(uuid); + verify(accountsDynamoDb, times(1)).update(account); verifyNoMoreInteractions(accountsDynamoDb); } @@ -436,7 +372,8 @@ class AccountsManagerTest { final UUID uuid = UUID.randomUUID(); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); - when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); + when(accountsDynamoDb.get(uuid)).thenReturn( + Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); assertTrue(account.getDevices().isEmpty()); @@ -463,6 +400,8 @@ class AccountsManagerTest { verify(unknownDeviceUpdater, never()).accept(any(Device.class)); } + // TODO delete + @Disabled("migration specific") @Test void testCompareAccounts() throws Exception { assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.empty(), Optional.empty())); @@ -538,13 +477,13 @@ class AccountsManagerTest { @Test void testCreateFreshAccount() throws InterruptedException { - when(accounts.create(any())).thenReturn(true); + when(accountsDynamoDb.create(any())).thenReturn(true); final String e164 = "+18005550123"; final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null); accountsManager.create(e164, "password", null, attributes); - verify(accounts).create(argThat(account -> e164.equals(account.getNumber()))); + verify(accountsDynamoDb).create(argThat(account -> e164.equals(account.getNumber()))); verifyNoInteractions(keys); verifyNoInteractions(messagesManager); verifyNoInteractions(profilesManager); @@ -554,7 +493,7 @@ class AccountsManagerTest { void testReregisterAccount() throws InterruptedException { final UUID existingUuid = UUID.randomUUID(); - when(accounts.create(any())).thenAnswer(invocation -> { + when(accountsDynamoDb.create(any())).thenAnswer(invocation -> { invocation.getArgument(0, Account.class).setUuid(existingUuid); return false; }); @@ -563,7 +502,8 @@ class AccountsManagerTest { final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null); accountsManager.create(e164, "password", null, attributes); - verify(accounts).create(argThat(account -> e164.equals(account.getNumber()) && existingUuid.equals(account.getUuid()))); + verify(accountsDynamoDb).create( + argThat(account -> e164.equals(account.getNumber()) && existingUuid.equals(account.getUuid()))); verify(keys).delete(existingUuid); verify(messagesManager).clear(existingUuid); verify(profilesManager).deleteAll(existingUuid); @@ -579,13 +519,14 @@ class AccountsManagerTest { return null; }).when(deletedAccountsManager).lockAndTake(anyString(), any()); - when(accounts.create(any())).thenReturn(true); + when(accountsDynamoDb.create(any())).thenReturn(true); final String e164 = "+18005550123"; final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null); accountsManager.create(e164, "password", null, attributes); - verify(accounts).create(argThat(account -> e164.equals(account.getNumber()) && recentlyDeletedUuid.equals(account.getUuid()))); + verify(accountsDynamoDb).create( + argThat(account -> e164.equals(account.getNumber()) && recentlyDeletedUuid.equals(account.getUuid()))); verifyNoInteractions(keys); verifyNoInteractions(messagesManager); verifyNoInteractions(profilesManager); @@ -634,6 +575,7 @@ class AccountsManagerTest { verify(directoryQueue, times(expectRefresh ? 1 : 0)).refreshAccount(updatedAccount); } + @SuppressWarnings("unused") private static Stream testUpdateDirectoryQueue() { return Stream.of( Arguments.of(false, false, false), @@ -654,7 +596,7 @@ class AccountsManagerTest { accountsManager.updateDeviceLastSeen(account, device, updatedLastSeen); assertEquals(expectUpdate ? updatedLastSeen : initialLastSeen, device.getLastSeen()); - verify(accounts, expectUpdate ? times(1) : never()).update(account); + verify(accountsDynamoDb, expectUpdate ? times(1) : never()).update(account); } @SuppressWarnings("unused") diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsTest.java deleted file mode 100644 index ef75a88b1..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsTest.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.tests.storage; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; - -import com.fasterxml.uuid.UUIDComparator; -import com.opentable.db.postgres.embedded.LiquibasePreparer; -import com.opentable.db.postgres.junit.EmbeddedPostgresRules; -import com.opentable.db.postgres.junit.PreparedDbRule; -import io.github.resilience4j.circuitbreaker.CallNotPermittedException; -import java.io.IOException; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Random; -import java.util.Set; -import java.util.UUID; -import org.jdbi.v3.core.HandleConsumer; -import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.transaction.TransactionException; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; -import org.whispersystems.textsecuregcm.entities.SignedPreKey; -import org.whispersystems.textsecuregcm.storage.Account; -import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk; -import org.whispersystems.textsecuregcm.storage.Accounts; -import org.whispersystems.textsecuregcm.storage.Device; -import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; -import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper; - -public class AccountsTest { - - @Rule - public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml")); - - private Accounts accounts; - - @Before - public void setupAccountsDao() { - FaultTolerantDatabase faultTolerantDatabase = new FaultTolerantDatabase("accountsTest", - Jdbi.create(db.getTestDatabase()), - new CircuitBreakerConfiguration()); - - this.accounts = new Accounts(faultTolerantDatabase); - } - - @Test - public void testStore() throws SQLException, IOException { - Device device = generateDevice (1 ); - Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device)); - - boolean freshUser = accounts.create(account); - assertThat(freshUser).isTrue(); - - PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?"); - verifyStoredState(statement, "+14151112222", account.getUuid(), account); - - freshUser = accounts.create(account); - assertThat(freshUser).isTrue(); - - statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?"); - verifyStoredState(statement, "+14151112222", account.getUuid(), account); - } - - @Test - public void testStoreMulti() throws SQLException, IOException { - Set devices = new HashSet<>(); - devices.add(generateDevice(1)); - devices.add(generateDevice(2)); - - Account account = generateAccount("+14151112222", UUID.randomUUID(), devices); - - accounts.create(account); - - PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?"); - verifyStoredState(statement, "+14151112222", account.getUuid(), account); - } - - @Test - public void testRetrieve() { - Set devicesFirst = new HashSet<>(); - devicesFirst.add(generateDevice(1)); - devicesFirst.add(generateDevice(2)); - - UUID uuidFirst = UUID.randomUUID(); - Account accountFirst = generateAccount("+14151112222", uuidFirst, devicesFirst); - - Set devicesSecond = new HashSet<>(); - devicesSecond.add(generateDevice(1)); - devicesSecond.add(generateDevice(2)); - - UUID uuidSecond = UUID.randomUUID(); - Account accountSecond = generateAccount("+14152221111", uuidSecond, devicesSecond); - - accounts.create(accountFirst); - accounts.create(accountSecond); - - Optional retrievedFirst = accounts.get("+14151112222"); - Optional retrievedSecond = accounts.get("+14152221111"); - - assertThat(retrievedFirst.isPresent()).isTrue(); - assertThat(retrievedSecond.isPresent()).isTrue(); - - verifyStoredState("+14151112222", uuidFirst, retrievedFirst.get(), accountFirst); - verifyStoredState("+14152221111", uuidSecond, retrievedSecond.get(), accountSecond); - - retrievedFirst = accounts.get(uuidFirst); - retrievedSecond = accounts.get(uuidSecond); - - assertThat(retrievedFirst.isPresent()).isTrue(); - assertThat(retrievedSecond.isPresent()).isTrue(); - - verifyStoredState("+14151112222", uuidFirst, retrievedFirst.get(), accountFirst); - verifyStoredState("+14152221111", uuidSecond, retrievedSecond.get(), accountSecond); - } - - @Test - public void testOverwrite() throws Exception { - Device device = generateDevice (1 ); - UUID firstUuid = UUID.randomUUID(); - Account account = generateAccount("+14151112222", firstUuid, Collections.singleton(device)); - - accounts.create(account); - - PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?"); - verifyStoredState(statement, "+14151112222", account.getUuid(), account); - - UUID secondUuid = UUID.randomUUID(); - - device = generateDevice(1); - account = generateAccount("+14151112222", secondUuid, Collections.singleton(device)); - - final boolean freshUser = accounts.create(account); - assertThat(freshUser).isFalse(); - verifyStoredState(statement, "+14151112222", firstUuid, account); - - device = generateDevice(1); - Account invalidAccount = generateAccount("+14151113333", firstUuid, Collections.singleton(device)); - - assertThatThrownBy(() -> accounts.create(invalidAccount)); - } - - @Test - public void testUpdate() { - Device device = generateDevice (1 ); - Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device)); - - accounts.create(account); - - device.setName("foobar"); - - accounts.update(account); - - account.setProfileName("profileName"); - - accounts.update(account); - - assertThat(account.getVersion()).isEqualTo(2); - - Optional retrieved = accounts.get("+14151112222"); - - assertThat(retrieved.isPresent()).isTrue(); - verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account); - - retrieved = accounts.get(account.getUuid()); - - assertThat(retrieved.isPresent()).isTrue(); - verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account); - } - - @Test - public void testRetrieveFrom() { - List users = new ArrayList<>(); - - for (int i=1;i<=100;i++) { - Account account = generateAccount("+1" + String.format("%03d", i), UUID.randomUUID()); - users.add(account); - accounts.create(account); - } - - users.sort((account, t1) -> UUIDComparator.staticCompare(account.getUuid(), t1.getUuid())); - - AccountCrawlChunk retrieved = accounts.getAllFrom(10); - assertThat(retrieved.getAccounts().size()).isEqualTo(10); - - for (int i=0;i retrieved = accounts.get("+14151112222"); - assertThat(retrieved.isPresent()).isTrue(); - - verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account); - } - - @Test - public void testMissing() { - Device device = generateDevice (1 ); - Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device)); - - accounts.create(account); - - Optional retrieved = accounts.get("+11111111"); - assertThat(retrieved.isPresent()).isFalse(); - - retrieved = accounts.get(UUID.randomUUID()); - assertThat(retrieved.isPresent()).isFalse(); - } - - @Test - public void testBreaker() throws InterruptedException { - Jdbi jdbi = mock(Jdbi.class); - doThrow(new TransactionException("Database error!")).when(jdbi).useHandle(any(HandleConsumer.class)); - - CircuitBreakerConfiguration configuration = new CircuitBreakerConfiguration(); - configuration.setWaitDurationInOpenStateInSeconds(1); - configuration.setRingBufferSizeInHalfOpenState(1); - configuration.setRingBufferSizeInClosedState(2); - configuration.setFailureRateThreshold(50); - - Accounts accounts = new Accounts(new FaultTolerantDatabase("testAccountBreaker", jdbi, configuration)); - Account account = generateAccount("+14151112222", UUID.randomUUID()); - - try { - accounts.update(account); - throw new AssertionError(); - } catch (TransactionException e) { - // good - } - - try { - accounts.update(account); - throw new AssertionError(); - } catch (TransactionException e) { - // good - } - - try { - accounts.update(account); - throw new AssertionError(); - } catch (CallNotPermittedException e) { - // good - } - - Thread.sleep(1100); - - try { - accounts.update(account); - throw new AssertionError(); - } catch (TransactionException e) { - // good - } - - } - - private Device generateDevice(long id) { - Random random = new Random(System.currentTimeMillis()); - SignedPreKey signedPreKey = new SignedPreKey(random.nextInt(), "testPublicKey-" + random.nextInt(), "testSignature-" + random.nextInt()); - return new Device(id, "testName-" + random.nextInt(), "testAuthToken-" + random.nextInt(), "testSalt-" + random.nextInt(), - "testGcmId-" + random.nextInt(), "testApnId-" + random.nextInt(), "testVoipApnId-" + random.nextInt(), random.nextBoolean(), random.nextInt(), signedPreKey, random.nextInt(), random.nextInt(), "testUserAgent-" + random.nextInt() , 0, new Device.DeviceCapabilities(random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), - random.nextBoolean(), random.nextBoolean(), random.nextBoolean())); - } - - private Account generateAccount(String number, UUID uuid) { - Device device = generateDevice(1); - return generateAccount(number, uuid, Collections.singleton(device)); - } - - private Account generateAccount(String number, UUID uuid, Set devices) { - byte[] unidentifiedAccessKey = new byte[16]; - Random random = new Random(System.currentTimeMillis()); - Arrays.fill(unidentifiedAccessKey, (byte)random.nextInt(255)); - - return new Account(number, uuid, devices, unidentifiedAccessKey); - } - - private void verifyStoredState(PreparedStatement statement, String number, UUID uuid, Account expecting) - throws SQLException, IOException - { - statement.setString(1, number); - - ResultSet resultSet = statement.executeQuery(); - - if (resultSet.next()) { - String data = resultSet.getString("data"); - assertThat(data).isNotEmpty(); - - Account result = new AccountRowMapper().map(resultSet, null); - verifyStoredState(number, uuid, result, expecting); - } else { - throw new AssertionError("No data"); - } - - assertThat(resultSet.next()).isFalse(); - } - - private void verifyStoredState(String number, UUID uuid, Account result, Account expecting) { - assertThat(result.getNumber()).isEqualTo(number); - assertThat(result.getLastSeen()).isEqualTo(expecting.getLastSeen()); - assertThat(result.getUuid()).isEqualTo(uuid); - assertThat(result.getVersion()).isEqualTo(expecting.getVersion()); - assertThat(Arrays.equals(result.getUnidentifiedAccessKey().get(), expecting.getUnidentifiedAccessKey().get())).isTrue(); - - for (Device expectingDevice : expecting.getDevices()) { - Device resultDevice = result.getDevice(expectingDevice.getId()).get(); - assertThat(resultDevice.getApnId()).isEqualTo(expectingDevice.getApnId()); - assertThat(resultDevice.getGcmId()).isEqualTo(expectingDevice.getGcmId()); - assertThat(resultDevice.getLastSeen()).isEqualTo(expectingDevice.getLastSeen()); - assertThat(resultDevice.getSignedPreKey().getPublicKey()).isEqualTo(expectingDevice.getSignedPreKey().getPublicKey()); - assertThat(resultDevice.getSignedPreKey().getKeyId()).isEqualTo(expectingDevice.getSignedPreKey().getKeyId()); - assertThat(resultDevice.getSignedPreKey().getSignature()).isEqualTo(expectingDevice.getSignedPreKey().getSignature()); - assertThat(resultDevice.getFetchesMessages()).isEqualTo(expectingDevice.getFetchesMessages()); - assertThat(resultDevice.getUserAgent()).isEqualTo(expectingDevice.getUserAgent()); - assertThat(resultDevice.getName()).isEqualTo(expectingDevice.getName()); - assertThat(resultDevice.getCreated()).isEqualTo(expectingDevice.getCreated()); - } - } - -}