Remove Accounts Postgres

This commit is contained in:
Chris Eager 2021-09-20 11:10:24 -07:00 committed by GitHub
parent 8161f55a82
commit 2a67b2e610
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 253 additions and 2906 deletions

View File

@ -46,9 +46,7 @@ import java.util.Optional;
import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import javax.servlet.DispatcherType; import javax.servlet.DispatcherType;
import javax.servlet.FilterRegistration; import javax.servlet.FilterRegistration;
@ -62,13 +60,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.whispersystems.dispatch.DispatchManager; import org.whispersystems.dispatch.DispatchManager;
import org.whispersystems.textsecuregcm.auth.AccountAuthenticator; import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
import org.whispersystems.textsecuregcm.auth.WebsocketRefreshApplicationEventListener;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount; import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
import org.whispersystems.textsecuregcm.auth.CertificateGenerator; import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccountAuthenticator; import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccountAuthenticator;
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount; import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator; import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator; import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
import org.whispersystems.textsecuregcm.auth.WebsocketRefreshApplicationEventListener;
import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter; import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter;
import org.whispersystems.textsecuregcm.badges.ProfileBadgeConverter; import org.whispersystems.textsecuregcm.badges.ProfileBadgeConverter;
import org.whispersystems.textsecuregcm.configuration.DirectoryServerConfiguration; import org.whispersystems.textsecuregcm.configuration.DirectoryServerConfiguration;
@ -156,9 +154,7 @@ import org.whispersystems.textsecuregcm.storage.AccountCleaner;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDbMigrator;
import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.ContactDiscoveryWriter; import org.whispersystems.textsecuregcm.storage.ContactDiscoveryWriter;
import org.whispersystems.textsecuregcm.storage.DeletedAccounts; import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
@ -174,11 +170,6 @@ import org.whispersystems.textsecuregcm.storage.MessagePersister;
import org.whispersystems.textsecuregcm.storage.MessagesCache; import org.whispersystems.textsecuregcm.storage.MessagesCache;
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager; import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts;
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts;
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccountsTableCrawler;
import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts;
import org.whispersystems.textsecuregcm.storage.MigrationRetryAccountsTableCrawler;
import org.whispersystems.textsecuregcm.storage.Profiles; import org.whispersystems.textsecuregcm.storage.Profiles;
import org.whispersystems.textsecuregcm.storage.ProfilesManager; import org.whispersystems.textsecuregcm.storage.ProfilesManager;
import org.whispersystems.textsecuregcm.storage.PubSubManager; import org.whispersystems.textsecuregcm.storage.PubSubManager;
@ -211,14 +202,12 @@ import org.whispersystems.textsecuregcm.workers.ServerVersionCommand;
import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask; import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask;
import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask; import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask;
import org.whispersystems.textsecuregcm.workers.SetUserDiscoverabilityCommand; import org.whispersystems.textsecuregcm.workers.SetUserDiscoverabilityCommand;
import org.whispersystems.textsecuregcm.workers.VacuumCommand;
import org.whispersystems.textsecuregcm.workers.ZkParamsCommand; import org.whispersystems.textsecuregcm.workers.ZkParamsCommand;
import org.whispersystems.websocket.WebSocketResourceProviderFactory; import org.whispersystems.websocket.WebSocketResourceProviderFactory;
import org.whispersystems.websocket.setup.WebSocketEnvironment; import org.whispersystems.websocket.setup.WebSocketEnvironment;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Client;
@ -228,7 +217,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
@Override @Override
public void initialize(Bootstrap<WhisperServerConfiguration> bootstrap) { public void initialize(Bootstrap<WhisperServerConfiguration> bootstrap) {
bootstrap.addCommand(new VacuumCommand());
bootstrap.addCommand(new DeleteUserCommand()); bootstrap.addCommand(new DeleteUserCommand());
bootstrap.addCommand(new CertificateCommand()); bootstrap.addCommand(new CertificateCommand());
bootstrap.addCommand(new ZkParamsCommand()); bootstrap.addCommand(new ZkParamsCommand());
@ -243,7 +231,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
} }
}); });
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("abusedb", "abusedb.xml") { bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("abusedb", "abusedb.xml") {
@Override @Override
public PooledDataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) { public PooledDataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) {
@ -316,20 +303,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(config.getAccountsDynamoDbConfiguration(), DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(config.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
// The thread pool core & max sizes are set via dynamic configuration within AccountsDynamoDb
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
new LinkedBlockingDeque<>());
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(config.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
accountsDynamoDbMigrationThreadPool);
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(config.getDeletedAccountsDynamoDbConfiguration(), DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(config.getDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient recentlyDeletedAccountsDynamoDb = DynamoDbFromConfig.client(config.getMigrationDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient pushChallengeDynamoDbClient = DynamoDbFromConfig.client( DynamoDbClient pushChallengeDynamoDbClient = DynamoDbFromConfig.client(
config.getPushChallengeDynamoDbConfiguration(), config.getPushChallengeDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@ -338,14 +314,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
config.getReportMessageDynamoDbConfiguration(), config.getReportMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(
config.getMigrationRetryAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient migrationMismatchedAccountsDynamoDb = DynamoDbFromConfig.client(
config.getMigrationMismatchedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig.client( DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig.client(
config.getPendingAccountsDynamoDbConfiguration(), config.getPendingAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@ -366,19 +334,11 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
config.getDeletedAccountsDynamoDbConfiguration().getTableName(), config.getDeletedAccountsDynamoDbConfiguration().getTableName(),
config.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName()); config.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(recentlyDeletedAccountsDynamoDb,
config.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb,
config.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
MigrationMismatchedAccounts mismatchedAccounts = new MigrationMismatchedAccounts(
migrationMismatchedAccountsDynamoDb,
config.getMigrationMismatchedAccountsDynamoDbConfiguration().getTableName());
Accounts accounts = new Accounts(accountDatabase); AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient,
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, config.getAccountsDynamoDbConfiguration().getTableName(),
accountsDynamoDbMigrationThreadPool, config.getAccountsDynamoDbConfiguration().getTableName(), config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName()
config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, );
migrationRetryAccounts);
Usernames usernames = new Usernames(accountDatabase); Usernames usernames = new Usernames(accountDatabase);
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase); ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
Profiles profiles = new Profiles(accountDatabase); Profiles profiles = new Profiles(accountDatabase);
@ -431,8 +391,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
ExecutorService donationExecutor = environment.lifecycle().executorService(name(getClass(), "donation-%d")).maxThreads(1).minThreads(1).build(); ExecutorService donationExecutor = environment.lifecycle().executorService(name(getClass(), "donation-%d")).maxThreads(1).minThreads(1).build();
ExecutorService multiRecipientMessageExecutor = environment.lifecycle() ExecutorService multiRecipientMessageExecutor = environment.lifecycle()
.executorService(name(getClass(), "multiRecipientMessage-%d")).minThreads(64).maxThreads(64).build(); .executorService(name(getClass(), "multiRecipientMessage-%d")).minThreads(64).maxThreads(64).build();
ExecutorService accountsCrawlerChunkPreReadExecutor = environment.lifecycle()
.executorService(name(getClass(), "accountsCrawler-%d")).maxThreads(2).minThreads(2).build();
ExternalServiceCredentialGenerator directoryCredentialsGenerator = new ExternalServiceCredentialGenerator(config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenSharedSecret(), ExternalServiceCredentialGenerator directoryCredentialsGenerator = new ExternalServiceCredentialGenerator(config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenSharedSecret(),
config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenUserIdSecret(), config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenUserIdSecret(),
@ -464,9 +422,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager, reportMessageManager); MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager, reportMessageManager);
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts, DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
deletedAccountsLockDynamoDbClient, config.getDeletedAccountsLockDynamoDbConfiguration().getTableName()); deletedAccountsLockDynamoDbClient, config.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, AccountsManager accountsManager = new AccountsManager(accountsDynamoDb, cacheCluster,
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, mismatchedAccounts, usernamesManager, deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager,
profilesManager, pendingAccountsManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, profilesManager, pendingAccountsManager, secureStorageClient, secureBackupClient,
dynamicConfigurationManager); dynamicConfigurationManager);
RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs); RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs);
DeadLetterHandler deadLetterHandler = new DeadLetterHandler(accountsManager, messagesManager); DeadLetterHandler deadLetterHandler = new DeadLetterHandler(accountsManager, messagesManager);
@ -539,27 +497,10 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
AccountDatabaseCrawler accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, AccountDatabaseCrawler accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager,
accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners, accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
config.getAccountDatabaseCrawlerConfiguration().getChunkSize(), config.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
config.getAccountDatabaseCrawlerConfiguration().getChunkIntervalMs(), config.getAccountDatabaseCrawlerConfiguration().getChunkIntervalMs()
accountsCrawlerChunkPreReadExecutor, );
dynamicConfigurationManager);
AccountDatabaseCrawlerCache dynamoDbMigrationCrawlerCache = new AccountDatabaseCrawlerCache(cacheCluster);
dynamoDbMigrationCrawlerCache.setPrefix("DynamoMigration");
AccountDatabaseCrawler accountDynamoDbMigrationCrawler = new AccountDatabaseCrawler(accountsManager,
dynamoDbMigrationCrawlerCache,
List.of(new AccountsDynamoDbMigrator(accountsDynamoDb, dynamicConfigurationManager)),
config.getDynamoDbMigrationCrawlerConfiguration().getChunkSize(),
config.getDynamoDbMigrationCrawlerConfiguration().getChunkIntervalMs(),
accountsCrawlerChunkPreReadExecutor,
dynamicConfigurationManager);
accountDynamoDbMigrationCrawler.setDedicatedDynamoMigrationCrawler(true);
DeletedAccountsTableCrawler deletedAccountsTableCrawler = new DeletedAccountsTableCrawler(deletedAccountsManager, deletedAccountsDirectoryReconcilers, cacheCluster, recurringJobExecutor); DeletedAccountsTableCrawler deletedAccountsTableCrawler = new DeletedAccountsTableCrawler(deletedAccountsManager, deletedAccountsDirectoryReconcilers, cacheCluster, recurringJobExecutor);
MigrationRetryAccountsTableCrawler migrationRetryAccountsTableCrawler = new MigrationRetryAccountsTableCrawler(
migrationRetryAccounts, accountsManager, accountsDynamoDb, cacheCluster, recurringJobExecutor);
MigrationMismatchedAccountsTableCrawler migrationMismatchedAccountsTableCrawler = new MigrationMismatchedAccountsTableCrawler(
mismatchedAccounts, accountsManager, accounts, accountsDynamoDb, dynamicConfigurationManager, cacheCluster,
recurringJobExecutor);
apnSender.setApnFallbackManager(apnFallbackManager); apnSender.setApnFallbackManager(apnFallbackManager);
environment.lifecycle().manage(new ApplicationShutdownMonitor()); environment.lifecycle().manage(new ApplicationShutdownMonitor());
@ -567,10 +508,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
environment.lifecycle().manage(pubSubManager); environment.lifecycle().manage(pubSubManager);
environment.lifecycle().manage(messageSender); environment.lifecycle().manage(messageSender);
environment.lifecycle().manage(accountDatabaseCrawler); environment.lifecycle().manage(accountDatabaseCrawler);
environment.lifecycle().manage(accountDynamoDbMigrationCrawler);
environment.lifecycle().manage(deletedAccountsTableCrawler); environment.lifecycle().manage(deletedAccountsTableCrawler);
environment.lifecycle().manage(migrationRetryAccountsTableCrawler);
environment.lifecycle().manage(migrationMismatchedAccountsTableCrawler);
environment.lifecycle().manage(remoteConfigsManager); environment.lifecycle().manage(remoteConfigsManager);
environment.lifecycle().manage(messagesCache); environment.lifecycle().manage(messagesCache);
environment.lifecycle().manage(messagePersister); environment.lifecycle().manage(messagePersister);

View File

@ -1,112 +1,15 @@
package org.whispersystems.textsecuregcm.configuration.dynamic; package org.whispersystems.textsecuregcm.configuration.dynamic;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
public class DynamicAccountsDynamoDbMigrationConfiguration { public class DynamicAccountsDynamoDbMigrationConfiguration {
@JsonProperty
boolean dynamoPrimary;
@JsonProperty
boolean backgroundMigrationEnabled;
@JsonProperty
int backgroundMigrationExecutorThreads = 1;
@JsonProperty
boolean deleteEnabled;
@JsonProperty
boolean writeEnabled;
@JsonProperty
boolean readEnabled;
@JsonProperty
boolean postCheckMismatches;
@JsonProperty
boolean logMismatches;
@JsonProperty
boolean crawlerPreReadNextChunkEnabled;
@JsonProperty
boolean dynamoCrawlerEnabled;
@JsonProperty @JsonProperty
int dynamoCrawlerScanPageSize = 10; int dynamoCrawlerScanPageSize = 10;
public boolean isDynamoPrimary() { // TODO move out of "migration" configuration
return dynamoPrimary;
}
public boolean isBackgroundMigrationEnabled() {
return backgroundMigrationEnabled;
}
public int getBackgroundMigrationExecutorThreads() {
return backgroundMigrationExecutorThreads;
}
@VisibleForTesting
public void setBackgroundMigrationEnabled(boolean backgroundMigrationEnabled) {
this.backgroundMigrationEnabled = backgroundMigrationEnabled;
}
public void setDeleteEnabled(boolean deleteEnabled) {
this.deleteEnabled = deleteEnabled;
}
public boolean isDeleteEnabled() {
return deleteEnabled;
}
public void setWriteEnabled(boolean writeEnabled) {
this.writeEnabled = writeEnabled;
}
public boolean isWriteEnabled() {
return writeEnabled;
}
@VisibleForTesting
public void setReadEnabled(boolean readEnabled) {
this.readEnabled = readEnabled;
}
public boolean isReadEnabled() {
return readEnabled;
}
public boolean isPostCheckMismatches() {
return postCheckMismatches;
}
public boolean isLogMismatches() {
return logMismatches;
}
public boolean isCrawlerPreReadNextChunkEnabled() {
return crawlerPreReadNextChunkEnabled;
}
public boolean isDynamoCrawlerEnabled() {
return dynamoCrawlerEnabled;
}
public int getDynamoCrawlerScanPageSize() { public int getDynamoCrawlerScanPageSize() {
return dynamoCrawlerScanPageSize; return dynamoCrawlerScanPageSize;
} }
@VisibleForTesting
public void setLogMismatches(boolean logMismatches) {
this.logMismatches = logMismatches;
}
@VisibleForTesting
public void setBackgroundMigrationExecutorThreads(int threads) {
this.backgroundMigrationExecutorThreads = threads;
}
} }

View File

@ -14,7 +14,6 @@ import io.dropwizard.lifecycle.Managed;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -41,32 +40,22 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
private final String workerId; private final String workerId;
private final AccountDatabaseCrawlerCache cache; private final AccountDatabaseCrawlerCache cache;
private final List<AccountDatabaseCrawlerListener> listeners; private final List<AccountDatabaseCrawlerListener> listeners;
private final ExecutorService chunkPreReadExecutorService;
private final DynamicConfigurationManager dynamicConfigurationManager;
private AtomicBoolean running = new AtomicBoolean(false); private AtomicBoolean running = new AtomicBoolean(false);
private boolean finished; private boolean finished;
// temporary to control behavior during the Postgres Dynamo transition
private boolean dedicatedDynamoMigrationCrawler;
public AccountDatabaseCrawler(AccountsManager accounts, public AccountDatabaseCrawler(AccountsManager accounts,
AccountDatabaseCrawlerCache cache, AccountDatabaseCrawlerCache cache,
List<AccountDatabaseCrawlerListener> listeners, List<AccountDatabaseCrawlerListener> listeners,
int chunkSize, int chunkSize,
long chunkIntervalMs, long chunkIntervalMs) {
ExecutorService chunkPreReadExecutorService,
DynamicConfigurationManager dynamicConfigurationManager) {
this.accounts = accounts; this.accounts = accounts;
this.chunkSize = chunkSize; this.chunkSize = chunkSize;
this.chunkIntervalMs = chunkIntervalMs; this.chunkIntervalMs = chunkIntervalMs;
this.workerId = UUID.randomUUID().toString(); this.workerId = UUID.randomUUID().toString();
this.cache = cache; this.cache = cache;
this.listeners = listeners; this.listeners = listeners;
this.chunkPreReadExecutorService = chunkPreReadExecutorService;
this.dynamicConfigurationManager = dynamicConfigurationManager;
} }
@Override @Override
@ -131,25 +120,19 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
try (Timer.Context timer = processChunkTimer.time()) { try (Timer.Context timer = processChunkTimer.time()) {
final boolean useDynamo = !dedicatedDynamoMigrationCrawler && dynamicConfigurationManager.getConfiguration() final Optional<UUID> fromUuid = getLastUuid();
.getAccountsDynamoDbMigrationConfiguration()
.isDynamoCrawlerEnabled();
final Optional<UUID> fromUuid = getLastUuid(useDynamo);
if (fromUuid.isEmpty()) { if (fromUuid.isEmpty()) {
logger.info("Started crawl"); logger.info("Started crawl");
listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart); listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart);
} }
final AccountCrawlChunk chunkAccounts = readChunk(fromUuid, chunkSize, useDynamo); final AccountCrawlChunk chunkAccounts = readChunk(fromUuid, chunkSize);
primeDatabaseForNextChunkAsync(chunkAccounts.getLastUuid(), chunkSize, useDynamo);
if (chunkAccounts.getAccounts().isEmpty()) { if (chunkAccounts.getAccounts().isEmpty()) {
logger.info("Finished crawl"); logger.info("Finished crawl");
listeners.forEach(listener -> listener.onCrawlEnd(fromUuid)); listeners.forEach(listener -> listener.onCrawlEnd(fromUuid));
cacheLastUuid(Optional.empty(), useDynamo); cacheLastUuid(Optional.empty());
cache.setAccelerated(false); cache.setAccelerated(false);
} else { } else {
logger.info("Processing chunk"); logger.info("Processing chunk");
@ -157,70 +140,42 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
for (AccountDatabaseCrawlerListener listener : listeners) { for (AccountDatabaseCrawlerListener listener : listeners) {
listener.timeAndProcessCrawlChunk(fromUuid, chunkAccounts.getAccounts()); listener.timeAndProcessCrawlChunk(fromUuid, chunkAccounts.getAccounts());
} }
cacheLastUuid(chunkAccounts.getLastUuid(), useDynamo); cacheLastUuid(chunkAccounts.getLastUuid());
} catch (AccountDatabaseCrawlerRestartException e) { } catch (AccountDatabaseCrawlerRestartException e) {
cacheLastUuid(Optional.empty(), useDynamo); cacheLastUuid(Optional.empty());
cache.setAccelerated(false); cache.setAccelerated(false);
} }
} }
} }
} }
/** private AccountCrawlChunk readChunk(Optional<UUID> fromUuid, int chunkSize) {
* This is an optimization based on the observation that cold reads of chunks are slow, but subsequent reads of the return readChunk(fromUuid, chunkSize, readChunkTimer);
* same chunk (within a few minutes) are fast. We cant easily store the actual result data, since the next chunk
* might be processed elsewhere, but the time savings are still substantial.
*/
private void primeDatabaseForNextChunkAsync(Optional<UUID> fromUuid, int chunkSize, boolean useDynamo) {
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
.isCrawlerPreReadNextChunkEnabled()) {
if (!useDynamo && fromUuid.isPresent()) {
chunkPreReadExecutorService.submit(() -> readChunk(fromUuid, chunkSize, false, preReadChunkTimer));
}
}
} }
private AccountCrawlChunk readChunk(Optional<UUID> fromUuid, int chunkSize, boolean useDynamo) { private AccountCrawlChunk readChunk(Optional<UUID> fromUuid, int chunkSize, Timer readTimer) {
return readChunk(fromUuid, chunkSize, useDynamo, readChunkTimer);
}
private AccountCrawlChunk readChunk(Optional<UUID> fromUuid, int chunkSize, boolean useDynamo, Timer readTimer) {
try (Timer.Context timer = readTimer.time()) { try (Timer.Context timer = readTimer.time()) {
if (fromUuid.isPresent()) { if (fromUuid.isPresent()) {
return useDynamo return accounts.getAllFromDynamo(fromUuid.get(), chunkSize);
? accounts.getAllFromDynamo(fromUuid.get(), chunkSize)
: accounts.getAllFrom(fromUuid.get(), chunkSize);
} }
return useDynamo return accounts.getAllFromDynamo(chunkSize);
? accounts.getAllFromDynamo(chunkSize)
: accounts.getAllFrom(chunkSize);
} }
} }
private Optional<UUID> getLastUuid(final boolean useDynamo) { private Optional<UUID> getLastUuid() {
if (useDynamo) { return cache.getLastUuidDynamo();
return cache.getLastUuidDynamo();
} else {
return cache.getLastUuid();
}
} }
private void cacheLastUuid(final Optional<UUID> lastUuid, final boolean useDynamo) { private void cacheLastUuid(final Optional<UUID> lastUuid) {
if (useDynamo) { cache.setLastUuidDynamo(lastUuid);
cache.setLastUuidDynamo(lastUuid);
} else {
cache.setLastUuid(lastUuid);
}
}
public void setDedicatedDynamoMigrationCrawler(final boolean dedicatedDynamoMigrationCrawler) {
this.dedicatedDynamoMigrationCrawler = dedicatedDynamoMigrationCrawler;
} }
private synchronized void sleepWhileRunning(long delayMs) { private synchronized void sleepWhileRunning(long delayMs) {
if (running.get()) Util.wait(this, delayMs); if (running.get()) {
Util.wait(this, delayMs);
}
} }
} }

View File

@ -27,8 +27,6 @@ public class AccountDatabaseCrawlerCache {
private final FaultTolerantRedisCluster cacheCluster; private final FaultTolerantRedisCluster cacheCluster;
private final ClusterLuaScript unlockClusterScript; private final ClusterLuaScript unlockClusterScript;
private String prefix = "";
public AccountDatabaseCrawlerCache(FaultTolerantRedisCluster cacheCluster) throws IOException { public AccountDatabaseCrawlerCache(FaultTolerantRedisCluster cacheCluster) throws IOException {
this.cacheCluster = cacheCluster; this.cacheCluster = cacheCluster;
this.unlockClusterScript = ClusterLuaScript.fromResource(cacheCluster, "lua/account_database_crawler/unlock.lua", this.unlockClusterScript = ClusterLuaScript.fromResource(cacheCluster, "lua/account_database_crawler/unlock.lua",
@ -37,9 +35,9 @@ public class AccountDatabaseCrawlerCache {
public void setAccelerated(final boolean accelerated) { public void setAccelerated(final boolean accelerated) {
if (accelerated) { if (accelerated) {
cacheCluster.useCluster(connection -> connection.sync().set(getPrefixedKey(ACCELERATE_KEY), "1")); cacheCluster.useCluster(connection -> connection.sync().set(ACCELERATE_KEY, "1"));
} else { } else {
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(ACCELERATE_KEY))); cacheCluster.useCluster(connection -> connection.sync().del(ACCELERATE_KEY));
} }
} }
@ -49,16 +47,16 @@ public class AccountDatabaseCrawlerCache {
public boolean claimActiveWork(String workerId, long ttlMs) { public boolean claimActiveWork(String workerId, long ttlMs) {
return "OK".equals(cacheCluster.withCluster(connection -> connection.sync() return "OK".equals(cacheCluster.withCluster(connection -> connection.sync()
.set(getPrefixedKey(ACTIVE_WORKER_KEY), workerId, SetArgs.Builder.nx().px(ttlMs)))); .set(ACTIVE_WORKER_KEY, workerId, SetArgs.Builder.nx().px(ttlMs))));
} }
public void releaseActiveWork(String workerId) { public void releaseActiveWork(String workerId) {
unlockClusterScript.execute(List.of(getPrefixedKey(ACTIVE_WORKER_KEY)), List.of(workerId)); unlockClusterScript.execute(List.of(ACTIVE_WORKER_KEY), List.of(workerId));
} }
public Optional<UUID> getLastUuid() { public Optional<UUID> getLastUuid() {
final String lastUuidString = cacheCluster.withCluster( final String lastUuidString = cacheCluster.withCluster(
connection -> connection.sync().get(getPrefixedKey(LAST_UUID_KEY))); connection -> connection.sync().get(LAST_UUID_KEY));
if (lastUuidString == null) { if (lastUuidString == null) {
return Optional.empty(); return Optional.empty();
@ -70,15 +68,15 @@ public class AccountDatabaseCrawlerCache {
public void setLastUuid(Optional<UUID> lastUuid) { public void setLastUuid(Optional<UUID> lastUuid) {
if (lastUuid.isPresent()) { if (lastUuid.isPresent()) {
cacheCluster.useCluster(connection -> connection.sync() cacheCluster.useCluster(connection -> connection.sync()
.psetex(getPrefixedKey(LAST_UUID_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString())); .psetex(LAST_UUID_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
} else { } else {
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_KEY))); cacheCluster.useCluster(connection -> connection.sync().del(LAST_UUID_KEY));
} }
} }
public Optional<UUID> getLastUuidDynamo() { public Optional<UUID> getLastUuidDynamo() {
final String lastUuidString = cacheCluster.withCluster( final String lastUuidString = cacheCluster.withCluster(
connection -> connection.sync().get(getPrefixedKey(LAST_UUID_DYNAMO_KEY))); connection -> connection.sync().get(LAST_UUID_DYNAMO_KEY));
if (lastUuidString == null) { if (lastUuidString == null) {
return Optional.empty(); return Optional.empty();
@ -91,21 +89,9 @@ public class AccountDatabaseCrawlerCache {
if (lastUuid.isPresent()) { if (lastUuid.isPresent()) {
cacheCluster.useCluster( cacheCluster.useCluster(
connection -> connection.sync() connection -> connection.sync()
.psetex(getPrefixedKey(LAST_UUID_DYNAMO_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString())); .psetex(LAST_UUID_DYNAMO_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
} else { } else {
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_DYNAMO_KEY))); cacheCluster.useCluster(connection -> connection.sync().del(LAST_UUID_DYNAMO_KEY));
} }
} }
private String getPrefixedKey(final String key) {
return prefix + key;
}
/**
* Set a cache key prefix, allowing for uses beyond the canonical crawler
*/
public void setPrefix(final String prefix) {
this.prefix = prefix + "::";
}
} }

View File

@ -1,172 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import com.codahale.metrics.Timer.Context;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import org.jdbi.v3.core.transaction.TransactionIsolationLevel;
import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.SystemMapper;
public class Accounts implements AccountStore {
public static final String ID = "id";
public static final String UID = "uuid";
public static final String NUMBER = "number";
public static final String DATA = "data";
public static final String VERSION = "version";
private static final ObjectMapper mapper = SystemMapper.getMapper();
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Timer createTimer = metricRegistry.timer(name(Accounts.class, "create" ));
private final Timer updateTimer = metricRegistry.timer(name(Accounts.class, "update" ));
private final Timer getByNumberTimer = metricRegistry.timer(name(Accounts.class, "getByNumber" ));
private final Timer getByUuidTimer = metricRegistry.timer(name(Accounts.class, "getByUuid" ));
private final Timer getAllFromTimer = metricRegistry.timer(name(Accounts.class, "getAllFrom" ));
private final Timer getAllFromOffsetTimer = metricRegistry.timer(name(Accounts.class, "getAllFromOffset"));
private final Timer deleteTimer = metricRegistry.timer(name(Accounts.class, "delete" ));
private final Timer vacuumTimer = metricRegistry.timer(name(Accounts.class, "vacuum" ));
private final FaultTolerantDatabase database;
public Accounts(FaultTolerantDatabase database) {
this.database = database;
this.database.getDatabase().registerRowMapper(new AccountRowMapper());
}
@Override
public boolean create(Account account) {
return database.with(jdbi -> jdbi.inTransaction(TransactionIsolationLevel.SERIALIZABLE, handle -> {
try (Timer.Context ignored = createTimer.time()) {
final Map<String, Object> resultMap = handle.createQuery("INSERT INTO accounts (" + NUMBER + ", " + UID + ", " + DATA + ") VALUES (:number, :uuid, CAST(:data AS json)) ON CONFLICT(number) DO UPDATE SET " + DATA + " = EXCLUDED.data, " + VERSION + " = accounts.version + 1 RETURNING uuid, version")
.bind("number", account.getNumber())
.bind("uuid", account.getUuid())
.bind("data", mapper.writeValueAsString(account))
.mapToMap()
.findOnly();
final UUID uuid = (UUID) resultMap.get(UID);
final int version = (int) resultMap.get(VERSION);
boolean isNew = uuid.equals(account.getUuid());
account.setUuid(uuid);
account.setVersion(version);
return isNew;
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(e);
}
}));
}
@Override
public void update(Account account) throws ContestedOptimisticLockException {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = updateTimer.time()) {
final int newVersion = account.getVersion() + 1;
int rowsModified = handle.createUpdate("UPDATE accounts SET " + DATA + " = CAST(:data AS json), " + VERSION + " = :newVersion WHERE " + UID + " = :uuid AND " + VERSION + " = :version")
.bind("uuid", account.getUuid())
.bind("data", mapper.writeValueAsString(account))
.bind("version", account.getVersion())
.bind("newVersion", newVersion)
.execute();
if (rowsModified == 0) {
throw new ContestedOptimisticLockException();
}
account.setVersion(newVersion);
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(e);
}
}));
}
@Override
public Optional<Account> get(String number) {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = getByNumberTimer.time()) {
return handle.createQuery("SELECT * FROM accounts WHERE " + NUMBER + " = :number")
.bind("number", number)
.mapTo(Account.class)
.findFirst();
}
}));
}
@Override
public Optional<Account> get(UUID uuid) {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = getByUuidTimer.time()) {
return handle.createQuery("SELECT * FROM accounts WHERE " + UID + " = :uuid")
.bind("uuid", uuid)
.mapTo(Account.class)
.findFirst();
}
}));
}
public AccountCrawlChunk getAllFrom(UUID from, int length) {
final List<Account> accounts = database.with(jdbi -> jdbi.withHandle(handle -> {
try (Context ignored = getAllFromOffsetTimer.time()) {
return handle.createQuery("SELECT * FROM accounts WHERE " + UID + " > :from ORDER BY " + UID + " LIMIT :limit")
.bind("from", from)
.bind("limit", length)
.mapTo(Account.class)
.list();
}
}));
return buildChunkForAccounts(accounts);
}
public AccountCrawlChunk getAllFrom(int length) {
final List<Account> accounts = database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = getAllFromTimer.time()) {
return handle.createQuery("SELECT * FROM accounts ORDER BY " + UID + " LIMIT :limit")
.bind("limit", length)
.mapTo(Account.class)
.list();
}
}));
return buildChunkForAccounts(accounts);
}
private AccountCrawlChunk buildChunkForAccounts(final List<Account> accounts) {
return new AccountCrawlChunk(accounts, accounts.isEmpty() ? null : accounts.get(accounts.size() - 1).getUuid());
}
@Override
public void delete(final UUID uuid) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = deleteTimer.time()) {
handle.createUpdate("DELETE FROM accounts WHERE " + UID + " = :uuid")
.bind("uuid", uuid)
.execute();
}
}));
}
public void vacuum() {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = vacuumTimer.time()) {
handle.execute("VACUUM accounts");
}
}));
}
}

View File

@ -8,7 +8,6 @@ import static com.codahale.metrics.MetricRegistry.name;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.Timer;
import java.io.IOException; import java.io.IOException;
@ -17,16 +16,10 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.AttributeValues; import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.SystemMapper; import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.UUIDUtil; import org.whispersystems.textsecuregcm.util.UUIDUtil;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.CancellationReason; import software.amazon.awssdk.services.dynamodb.model.CancellationReason;
@ -59,12 +52,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
static final String ATTR_CANONICALLY_DISCOVERABLE = "C"; static final String ATTR_CANONICALLY_DISCOVERABLE = "C";
private final DynamoDbClient client; private final DynamoDbClient client;
private final DynamoDbAsyncClient asyncClient;
private final ThreadPoolExecutor migrationThreadPool;
private final MigrationDeletedAccounts migrationDeletedAccounts;
private final MigrationRetryAccounts migrationRetryAccounts;
private final String phoneNumbersTableName; private final String phoneNumbersTableName;
private final String accountsTableName; private final String accountsTableName;
@ -76,26 +63,15 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFrom")); private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFrom"));
private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFromOffset")); private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFromOffset"));
private static final Timer DELETE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "delete")); private static final Timer DELETE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "delete"));
private static final Timer DELETE_RECENTLY_DELETED_UUIDS_TIMER = Metrics.timer(
name(AccountsDynamoDb.class, "deleteRecentlyDeletedUuids"));
private final Logger logger = LoggerFactory.getLogger(AccountsDynamoDb.class);
public AccountsDynamoDb(DynamoDbClient client, DynamoDbAsyncClient asyncClient, public AccountsDynamoDb(DynamoDbClient client, String accountsTableName, String phoneNumbersTableName) {
ThreadPoolExecutor migrationThreadPool, String accountsTableName, String phoneNumbersTableName,
MigrationDeletedAccounts migrationDeletedAccounts,
MigrationRetryAccounts accountsMigrationErrors) {
super(client); super(client);
this.client = client; this.client = client;
this.asyncClient = asyncClient;
this.phoneNumbersTableName = phoneNumbersTableName; this.phoneNumbersTableName = phoneNumbersTableName;
this.accountsTableName = accountsTableName; this.accountsTableName = accountsTableName;
this.migrationThreadPool = migrationThreadPool;
this.migrationDeletedAccounts = migrationDeletedAccounts;
this.migrationRetryAccounts = accountsMigrationErrors;
} }
@Override @Override
@ -215,29 +191,19 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
} }
try { try {
try { UpdateItemResponse response = client.updateItem(updateItemRequest);
UpdateItemResponse response = client.updateItem(updateItemRequest);
account.setVersion(AttributeValues.getInt(response.attributes(), "V", account.getVersion() + 1)); account.setVersion(AttributeValues.getInt(response.attributes(), "V", account.getVersion() + 1));
} catch (final TransactionConflictException e) { } catch (final TransactionConflictException e) {
throw new ContestedOptimisticLockException(); throw new ContestedOptimisticLockException();
} catch (final ConditionalCheckFailedException e) { } catch (final ConditionalCheckFailedException e) {
// the exception doesnt give details about which condition failed, // the exception doesnt give details about which condition failed,
// but we can infer it was an optimistic locking failure if the UUID is known // but we can infer it was an optimistic locking failure if the UUID is known
throw get(account.getUuid()).isPresent() ? new ContestedOptimisticLockException() : e; throw get(account.getUuid()).isPresent() ? new ContestedOptimisticLockException() : e;
}
} catch (final Exception e) {
if (!(e instanceof ContestedOptimisticLockException)) {
// the Dynamo account now lags the Postgres account version. Put it in the migration retry table so that it will
// get updated fasterotherwise it will be stale until the accounts crawler runs again
migrationRetryAccounts.put(account.getUuid());
}
throw e;
} }
}); });
} }
@ -279,10 +245,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
DELETE_TIMER.record(() -> delete(uuid, true)); DELETE_TIMER.record(() -> delete(uuid, true));
} }
public void deleteInvalidMigration(UUID uuid) {
DELETE_TIMER.record(() -> delete(uuid, false));
}
public AccountCrawlChunk getAllFrom(final UUID from, final int maxCount, final int pageSize) { public AccountCrawlChunk getAllFrom(final UUID from, final int maxCount, final int pageSize) {
final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder() final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder()
.limit(pageSize) .limit(pageSize)
@ -312,10 +274,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
private void delete(UUID uuid, boolean saveInDeletedAccountsTable) { private void delete(UUID uuid, boolean saveInDeletedAccountsTable) {
if (saveInDeletedAccountsTable) {
migrationDeletedAccounts.put(uuid);
}
Optional<Account> maybeAccount = get(uuid); Optional<Account> maybeAccount = get(uuid);
maybeAccount.ifPresent(account -> { maybeAccount.ifPresent(account -> {
@ -341,105 +299,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
}); });
} }
private static final Counter MIGRATED_COUNTER = Metrics.counter(name(AccountsDynamoDb.class, "migration", "count"));
private static final Counter ERROR_COUNTER = Metrics.counter(name(AccountsDynamoDb.class, "migration", "error"));
public CompletableFuture<Void> migrate(List<Account> accounts, int threads) {
if (threads > migrationThreadPool.getMaximumPoolSize()) {
migrationThreadPool.setMaximumPoolSize(threads);
migrationThreadPool.setCorePoolSize(threads);
} else {
migrationThreadPool.setCorePoolSize(threads);
migrationThreadPool.setMaximumPoolSize(threads);
}
final List<CompletableFuture<?>> futures = accounts.stream()
.map(this::migrate)
.map(f -> f.whenCompleteAsync((migrated, e) -> {
if (e == null) {
MIGRATED_COUNTER.increment(migrated ? 1 : 0);
} else {
ERROR_COUNTER.increment();
}
}, migrationThreadPool))
.collect(Collectors.toList());
CompletableFuture<Void> migrationBatch = CompletableFuture.allOf(futures.toArray(new CompletableFuture[]{}));
return migrationBatch.whenCompleteAsync((result, exception) -> {
if (exception != null) {
logger.warn("Exception migrating batch", exception);
}
deleteRecentlyDeletedUuids();
}, migrationThreadPool);
}
public void deleteRecentlyDeletedUuids() {
DELETE_RECENTLY_DELETED_UUIDS_TIMER.record(() -> {
final List<UUID> recentlyDeletedUuids = migrationDeletedAccounts.getRecentlyDeletedUuids();
for (UUID recentlyDeletedUuid : recentlyDeletedUuids) {
delete(recentlyDeletedUuid, false);
}
migrationDeletedAccounts.delete(recentlyDeletedUuids);
});
}
public CompletableFuture<Boolean> migrate(Account account) {
try {
TransactWriteItem phoneNumberConstraintPut = buildPutWriteItemForPhoneNumberConstraint(account, account.getUuid());
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid(), Put.builder()
.conditionExpression("attribute_not_exists(#uuid) OR (attribute_exists(#uuid) AND #version < :version)")
.expressionAttributeNames(Map.of(
"#uuid", KEY_ACCOUNT_UUID,
"#version", ATTR_VERSION))
.expressionAttributeValues(Map.of(
":version", AttributeValues.fromInt(account.getVersion()))));
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(phoneNumberConstraintPut, accountPut).build();
final CompletableFuture<Boolean> resultFuture = new CompletableFuture<>();
asyncClient.transactWriteItems(request).whenCompleteAsync((result, exception) -> {
if (result != null) {
resultFuture.complete(true);
return;
}
if (exception instanceof CompletionException) {
// whenCompleteAsync can wrap exceptions in a CompletionException; unwrap it to get to the root cause.
exception = exception.getCause();
}
if (exception instanceof TransactionCanceledException) {
// account is already migrated
resultFuture.complete(false);
return;
}
try {
migrationRetryAccounts.put(account.getUuid());
} catch (final Exception e) {
logger.error("Could not store account {}", account.getUuid());
}
resultFuture.completeExceptionally(exception);
}, migrationThreadPool);
return resultFuture;
} catch (Exception e) {
return CompletableFuture.failedFuture(e);
}
}
void putUuidForMigrationRetry(final UUID uuid) {
try {
migrationRetryAccounts.put(uuid);
} catch (final Exception e) {
logger.error("Failed to store for retry: {}", uuid, e);
}
}
private static String extractCancellationReasonCodes(final TransactionCanceledException exception) { private static String extractCancellationReasonCodes(final TransactionCanceledException exception) {
return exception.cancellationReasons().stream() return exception.cancellationReasons().stream()
.map(CancellationReason::code) .map(CancellationReason::code)

View File

@ -1,41 +0,0 @@
package org.whispersystems.textsecuregcm.storage;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
public class AccountsDynamoDbMigrator extends AccountDatabaseCrawlerListener {
private final AccountsDynamoDb accountsDynamoDb;
private final DynamicConfigurationManager dynamicConfigurationManager;
public AccountsDynamoDbMigrator(final AccountsDynamoDb accountsDynamoDb, final DynamicConfigurationManager dynamicConfigurationManager) {
this.accountsDynamoDb = accountsDynamoDb;
this.dynamicConfigurationManager = dynamicConfigurationManager;
}
@Override
public void onCrawlStart() {
}
@Override
public void onCrawlEnd(Optional<UUID> fromUuid) {
}
@Override
protected void onCrawlChunk(Optional<UUID> fromUuid, List<Account> chunkAccounts) {
if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
.isBackgroundMigrationEnabled()) {
return;
}
final CompletableFuture<Void> migrationBatch = accountsDynamoDb.migrate(chunkAccounts,
dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().getBackgroundMigrationExecutorThreads());
migrationBatch.join();
}
}

View File

@ -11,22 +11,17 @@ import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries; import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import io.lettuce.core.RedisException; import io.lettuce.core.RedisException;
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands; import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.Tags;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.function.BiFunction;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Supplier; import java.util.function.Supplier;
@ -37,7 +32,6 @@ import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials; import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
import org.whispersystems.textsecuregcm.controllers.AccountController; import org.whispersystems.textsecuregcm.controllers.AccountController;
import org.whispersystems.textsecuregcm.entities.AccountAttributes; import org.whispersystems.textsecuregcm.entities.AccountAttributes;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
@ -69,20 +63,14 @@ public class AccountsManager {
private static final String COUNTRY_CODE_TAG_NAME = "country"; private static final String COUNTRY_CODE_TAG_NAME = "country";
private static final String DELETION_REASON_TAG_NAME = "reason"; private static final String DELETION_REASON_TAG_NAME = "reason";
private static final String DYNAMO_MIGRATION_ERROR_COUNTER_NAME = name(AccountsManager.class, "migration", "error");
private static final Counter DYNAMO_MIGRATION_COMPARISON_COUNTER = Metrics.counter(name(AccountsManager.class, "migration", "comparisons"));
private static final String DYNAMO_MIGRATION_MISMATCH_COUNTER_NAME = name(AccountsManager.class, "migration", "mismatches");
private final Logger logger = LoggerFactory.getLogger(AccountsManager.class); private final Logger logger = LoggerFactory.getLogger(AccountsManager.class);
private final Accounts accounts;
private final AccountsDynamoDb accountsDynamoDb; private final AccountsDynamoDb accountsDynamoDb;
private final FaultTolerantRedisCluster cacheCluster; private final FaultTolerantRedisCluster cacheCluster;
private final DeletedAccountsManager deletedAccountsManager; private final DeletedAccountsManager deletedAccountsManager;
private final DirectoryQueue directoryQueue; private final DirectoryQueue directoryQueue;
private final KeysDynamoDb keysDynamoDb; private final KeysDynamoDb keysDynamoDb;
private final MessagesManager messagesManager; private final MessagesManager messagesManager;
private final MigrationMismatchedAccounts mismatchedAccounts;
private final UsernamesManager usernamesManager; private final UsernamesManager usernamesManager;
private final ProfilesManager profilesManager; private final ProfilesManager profilesManager;
private final StoredVerificationCodeManager pendingAccounts; private final StoredVerificationCodeManager pendingAccounts;
@ -90,10 +78,7 @@ public class AccountsManager {
private final SecureBackupClient secureBackupClient; private final SecureBackupClient secureBackupClient;
private final ObjectMapper mapper; private final ObjectMapper mapper;
private final ObjectMapper migrationComparisonMapper;
private final DynamicConfigurationManager dynamicConfigurationManager; private final DynamicConfigurationManager dynamicConfigurationManager;
private final ExperimentEnrollmentManager experimentEnrollmentManager;
public enum DeletionReason { public enum DeletionReason {
ADMIN_DELETED("admin"), ADMIN_DELETED("admin"),
@ -107,25 +92,22 @@ public class AccountsManager {
} }
} }
public AccountsManager(Accounts accounts, AccountsDynamoDb accountsDynamoDb, FaultTolerantRedisCluster cacheCluster, public AccountsManager(AccountsDynamoDb accountsDynamoDb, FaultTolerantRedisCluster cacheCluster,
final DeletedAccountsManager deletedAccountsManager, final DeletedAccountsManager deletedAccountsManager,
final DirectoryQueue directoryQueue, final DirectoryQueue directoryQueue,
final KeysDynamoDb keysDynamoDb, final MessagesManager messagesManager, final KeysDynamoDb keysDynamoDb, final MessagesManager messagesManager,
final MigrationMismatchedAccounts mismatchedAccounts, final UsernamesManager usernamesManager, final UsernamesManager usernamesManager,
final ProfilesManager profilesManager, final ProfilesManager profilesManager,
final StoredVerificationCodeManager pendingAccounts, final StoredVerificationCodeManager pendingAccounts,
final SecureStorageClient secureStorageClient, final SecureStorageClient secureStorageClient,
final SecureBackupClient secureBackupClient, final SecureBackupClient secureBackupClient,
final ExperimentEnrollmentManager experimentEnrollmentManager,
final DynamicConfigurationManager dynamicConfigurationManager) { final DynamicConfigurationManager dynamicConfigurationManager) {
this.accounts = accounts; this.accountsDynamoDb = accountsDynamoDb;
this.accountsDynamoDb = accountsDynamoDb;
this.cacheCluster = cacheCluster; this.cacheCluster = cacheCluster;
this.deletedAccountsManager = deletedAccountsManager; this.deletedAccountsManager = deletedAccountsManager;
this.directoryQueue = directoryQueue; this.directoryQueue = directoryQueue;
this.keysDynamoDb = keysDynamoDb; this.keysDynamoDb = keysDynamoDb;
this.messagesManager = messagesManager; this.messagesManager = messagesManager;
this.mismatchedAccounts = mismatchedAccounts;
this.usernamesManager = usernamesManager; this.usernamesManager = usernamesManager;
this.profilesManager = profilesManager; this.profilesManager = profilesManager;
this.pendingAccounts = pendingAccounts; this.pendingAccounts = pendingAccounts;
@ -133,11 +115,7 @@ public class AccountsManager {
this.secureBackupClient = secureBackupClient; this.secureBackupClient = secureBackupClient;
this.mapper = SystemMapper.getMapper(); this.mapper = SystemMapper.getMapper();
this.migrationComparisonMapper = mapper.copy();
migrationComparisonMapper.addMixIn(Device.class, DeviceComparisonMixin.class);
this.dynamicConfigurationManager = dynamicConfigurationManager; this.dynamicConfigurationManager = dynamicConfigurationManager;
this.experimentEnrollmentManager = experimentEnrollmentManager;
} }
public Account create(final String number, public Account create(final String number,
@ -170,36 +148,12 @@ public class AccountsManager {
final UUID originalUuid = account.getUuid(); final UUID originalUuid = account.getUuid();
boolean freshUser = primaryCreate(account); boolean freshUser = dynamoCreate(account);
// create() sometimes updates the UUID, if there was a number conflict. // create() sometimes updates the UUID, if there was a number conflict.
// for metrics, we want secondary to run with the same original UUID // for metrics, we want secondary to run with the same original UUID
final UUID actualUuid = account.getUuid(); final UUID actualUuid = account.getUuid();
try {
if (secondaryWriteEnabled()) {
account.setUuid(originalUuid);
runSafelyAndRecordMetrics(() -> secondaryCreate(account), Optional.of(account.getUuid()), freshUser,
(primaryResult, secondaryResult) -> {
if (primaryResult.equals(secondaryResult)) {
return Optional.empty();
}
if (secondaryResult) {
return Optional.of("secondaryFreshUser");
}
return Optional.of("primaryFreshUser");
},
"create");
}
} finally {
account.setUuid(actualUuid);
}
redisSet(account); redisSet(account);
pendingAccounts.remove(number); pendingAccounts.remove(number);
@ -293,26 +247,7 @@ public class AccountsManager {
final UUID uuid = account.getUuid(); final UUID uuid = account.getUuid();
updatedAccount = updateWithRetries(account, updater, this::primaryUpdate, () -> primaryGet(uuid).get()); updatedAccount = updateWithRetries(account, updater, this::dynamoUpdate, () -> dynamoGet(uuid).get());
if (secondaryWriteEnabled()) {
runSafelyAndRecordMetrics(() -> secondaryGet(uuid).map(secondaryAccount -> {
try {
return updateWithRetries(secondaryAccount, updater, this::secondaryUpdate, () -> secondaryGet(uuid).get());
} catch (final OptimisticLockRetryLimitExceededException e) {
if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
.isDynamoPrimary()) {
accountsDynamoDb.putUuidForMigrationRetry(uuid);
}
throw e;
}
}),
Optional.of(uuid),
Optional.of(updatedAccount),
this::compareAccounts,
"update");
}
redisSet(updatedAccount); redisSet(updatedAccount);
} }
@ -378,14 +313,9 @@ public class AccountsManager {
try (Timer.Context ignored = getByNumberTimer.time()) { try (Timer.Context ignored = getByNumberTimer.time()) {
Optional<Account> account = redisGet(number); Optional<Account> account = redisGet(number);
if (!account.isPresent()) { if (account.isEmpty()) {
account = primaryGet(number); account = dynamoGet(number);
account.ifPresent(value -> redisSet(value)); account.ifPresent(this::redisSet);
if (secondaryReadEnabled()) {
runSafelyAndRecordMetrics(() -> secondaryGet(number), Optional.empty(), account, this::compareAccounts,
"getByNumber");
}
} }
return account; return account;
@ -396,29 +326,15 @@ public class AccountsManager {
try (Timer.Context ignored = getByUuidTimer.time()) { try (Timer.Context ignored = getByUuidTimer.time()) {
Optional<Account> account = redisGet(uuid); Optional<Account> account = redisGet(uuid);
if (!account.isPresent()) { if (account.isEmpty()) {
account = primaryGet(uuid); account = dynamoGet(uuid);
account.ifPresent(value -> redisSet(value)); account.ifPresent(this::redisSet);
if (secondaryReadEnabled()) {
runSafelyAndRecordMetrics(() -> secondaryGet(uuid), Optional.of(uuid), account, this::compareAccounts,
"getByUuid");
}
} }
return account; return account;
} }
} }
public AccountCrawlChunk getAllFrom(int length) {
return accounts.getAllFrom(length);
}
public AccountCrawlChunk getAllFrom(UUID uuid, int length) {
return accounts.getAllFrom(uuid, length);
}
public AccountCrawlChunk getAllFromDynamo(int length) { public AccountCrawlChunk getAllFromDynamo(int length) {
final int maxPageSize = dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration() final int maxPageSize = dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
.getDynamoCrawlerScanPageSize(); .getDynamoCrawlerScanPageSize();
@ -447,16 +363,7 @@ public class AccountsManager {
deleteBackupServiceDataFuture.join(); deleteBackupServiceDataFuture.join();
redisDelete(account); redisDelete(account);
primaryDelete(account); dynamoDelete(account);
if (secondaryDeleteEnabled()) {
try {
secondaryDelete(account);
} catch (final Exception e) {
logger.error("Could not delete account {} from secondary", account.getUuid().toString());
Metrics.counter(DYNAMO_MIGRATION_ERROR_COUNTER_NAME, "action", "delete").increment();
}
}
return account.getUuid(); return account.getUuid();
}); });
@ -537,100 +444,6 @@ public class AccountsManager {
} }
} }
private Optional<Account> primaryGet(String number) {
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
?
dynamoGet(number) :
databaseGet(number);
}
private Optional<Account> secondaryGet(String number) {
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
?
databaseGet(number) :
dynamoGet(number);
}
private Optional<Account> primaryGet(UUID uuid) {
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
?
dynamoGet(uuid) :
databaseGet(uuid);
}
private Optional<Account> secondaryGet(UUID uuid) {
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
?
databaseGet(uuid) :
dynamoGet(uuid);
}
private boolean primaryCreate(Account account) {
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
?
dynamoCreate(account) :
databaseCreate(account);
}
private boolean secondaryCreate(Account account) {
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
?
databaseCreate(account) :
dynamoCreate(account);
}
private void primaryUpdate(Account account) {
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) {
dynamoUpdate(account);
} else {
databaseUpdate(account);
}
}
private void secondaryUpdate(Account account) {
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) {
databaseUpdate(account);
} else {
dynamoUpdate(account);
}
}
private void primaryDelete(Account account) {
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) {
dynamoDelete(account);
} else {
databaseDelete(account);
}
}
private void secondaryDelete(Account account) {
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) {
databaseDelete(account);
} else {
dynamoDelete(account);
}
}
private Optional<Account> databaseGet(String number) {
return accounts.get(number);
}
private Optional<Account> databaseGet(UUID uuid) {
return accounts.get(uuid);
}
private boolean databaseCreate(Account account) {
return accounts.create(account);
}
private void databaseUpdate(Account account) {
accounts.update(account);
}
private void databaseDelete(final Account account) {
accounts.delete(account.getUuid());
}
private Optional<Account> dynamoGet(String number) { private Optional<Account> dynamoGet(String number) {
return accountsDynamoDb.get(number); return accountsDynamoDb.get(number);
} }
@ -651,175 +464,14 @@ public class AccountsManager {
accountsDynamoDb.delete(account.getUuid()); accountsDynamoDb.delete(account.getUuid());
} }
private boolean secondaryDeleteEnabled() { // TODO delete
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDeleteEnabled();
}
private boolean secondaryReadEnabled() {
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isReadEnabled();
}
private boolean secondaryWriteEnabled() {
return secondaryDeleteEnabled()
&& dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isWriteEnabled();
}
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") @SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Deprecated
public Optional<String> compareAccounts(final Optional<Account> maybePrimaryAccount, public Optional<String> compareAccounts(final Optional<Account> maybePrimaryAccount,
final Optional<Account> maybeSecondaryAccount) { final Optional<Account> maybeSecondaryAccount) {
if (maybePrimaryAccount.isEmpty() && maybeSecondaryAccount.isEmpty()) {
return Optional.empty();
}
if (maybePrimaryAccount.isEmpty()) {
return Optional.of("primaryMissing");
}
if (maybeSecondaryAccount.isEmpty()) {
return Optional.of("secondaryMissing");
}
final Account primaryAccount = maybePrimaryAccount.get();
final Account secondaryAccount = maybeSecondaryAccount.get();
final int uuidCompare = primaryAccount.getUuid().compareTo(secondaryAccount.getUuid());
if (uuidCompare != 0) {
return Optional.of("uuid");
}
final int numberCompare = primaryAccount.getNumber().compareTo(secondaryAccount.getNumber());
if (numberCompare != 0) {
return Optional.of("number");
}
if (!Objects.equals(primaryAccount.getIdentityKey(), secondaryAccount.getIdentityKey())) {
return Optional.of("identityKey");
}
if (!Objects.equals(primaryAccount.getCurrentProfileVersion(), secondaryAccount.getCurrentProfileVersion())) {
return Optional.of("currentProfileVersion");
}
if (!Objects.equals(primaryAccount.getProfileName(), secondaryAccount.getProfileName())) {
return Optional.of("profileName");
}
if (!Objects.equals(primaryAccount.getAvatar(), secondaryAccount.getAvatar())) {
return Optional.of("avatar");
}
if (!Objects.equals(primaryAccount.getUnidentifiedAccessKey(), secondaryAccount.getUnidentifiedAccessKey())) {
if (primaryAccount.getUnidentifiedAccessKey().isPresent() && secondaryAccount.getUnidentifiedAccessKey()
.isPresent()) {
if (Arrays.compare(primaryAccount.getUnidentifiedAccessKey().get(),
secondaryAccount.getUnidentifiedAccessKey().get()) != 0) {
return Optional.of("unidentifiedAccessKey");
}
} else {
return Optional.of("unidentifiedAccessKey");
}
}
if (!Objects.equals(primaryAccount.isUnrestrictedUnidentifiedAccess(),
secondaryAccount.isUnrestrictedUnidentifiedAccess())) {
return Optional.of("unrestrictedUnidentifiedAccess");
}
if (!Objects.equals(primaryAccount.isDiscoverableByPhoneNumber(), secondaryAccount.isDiscoverableByPhoneNumber())) {
return Optional.of("discoverableByPhoneNumber");
}
if (primaryAccount.getMasterDevice().isPresent() && secondaryAccount.getMasterDevice().isPresent()) {
if (!Objects.equals(primaryAccount.getMasterDevice().get().getSignedPreKey(),
secondaryAccount.getMasterDevice().get().getSignedPreKey())) {
return Optional.of("masterDeviceSignedPreKey");
}
}
try {
if (!serializedEquals(primaryAccount.getDevices(), secondaryAccount.getDevices())) {
return Optional.of("devices");
}
if (primaryAccount.getVersion() != secondaryAccount.getVersion()) {
return Optional.of("version");
}
if (primaryAccount.getMasterDevice().isPresent() && secondaryAccount.getMasterDevice().isPresent()) {
if (Math.abs(primaryAccount.getMasterDevice().get().getPushTimestamp() -
secondaryAccount.getMasterDevice().get().getPushTimestamp()) > 60 * 1_000L) {
// These are generally few milliseconds off, because the setter uses System.currentTimeMillis() internally,
// but we can be more relaxed
return Optional.of("masterDevicePushTimestamp");
}
}
if (!serializedEquals(primaryAccount, secondaryAccount)) {
return Optional.of("serialization");
}
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
return Optional.empty(); return Optional.empty();
} }
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private <T> void runSafelyAndRecordMetrics(Callable<T> callable, Optional<UUID> maybeUuid, final T primaryResult,
final BiFunction<T, T, Optional<String>> mismatchClassifier, final String action) {
if (maybeUuid.isPresent()) {
// the only time we dont have a UUID is in getByNumber, which is sufficiently low volume to not be a concern, and
// it will also be gated by the global readEnabled configuration
final boolean enrolled = experimentEnrollmentManager.isEnrolled(maybeUuid.get(), "accountsDynamoDbMigration");
if (!enrolled) {
return;
}
}
try {
final T secondaryResult = callable.call();
compare(primaryResult, secondaryResult, mismatchClassifier, action, maybeUuid);
} catch (final Exception e) {
logger.error("Error running " + action + " in Dynamo", e);
Metrics.counter(DYNAMO_MIGRATION_ERROR_COUNTER_NAME, "action", action).increment();
}
}
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private <T> void compare(final T primaryResult, final T secondaryResult,
final BiFunction<T, T, Optional<String>> mismatchClassifier, final String action,
final Optional<UUID> maybeUUid) {
DYNAMO_MIGRATION_COMPARISON_COUNTER.increment();
mismatchClassifier.apply(primaryResult, secondaryResult)
.ifPresent(mismatchType -> {
final String mismatchDescription = action + ":" + mismatchType;
Metrics.counter(DYNAMO_MIGRATION_MISMATCH_COUNTER_NAME,
"mismatchType", mismatchDescription)
.increment();
maybeUUid.ifPresent(uuid -> {
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
.isPostCheckMismatches()) {
mismatchedAccounts.put(uuid);
}
});
});
}
private String getAbbreviatedCallChain(final StackTraceElement[] stackTrace) { private String getAbbreviatedCallChain(final StackTraceElement[] stackTrace) {
return Arrays.stream(stackTrace) return Arrays.stream(stackTrace)
.filter(stackTraceElement -> stackTraceElement.getClassName().contains("org.whispersystems")) .filter(stackTraceElement -> stackTraceElement.getClassName().contains("org.whispersystems"))
@ -827,22 +479,4 @@ public class AccountsManager {
.map(stackTraceElement -> StringUtils.substringAfterLast(stackTraceElement.getClassName(), ".") + ":" + stackTraceElement.getMethodName()) .map(stackTraceElement -> StringUtils.substringAfterLast(stackTraceElement.getClassName(), ".") + ":" + stackTraceElement.getMethodName())
.collect(Collectors.joining(" -> ")); .collect(Collectors.joining(" -> "));
} }
private static abstract class DeviceComparisonMixin extends Device {
@JsonIgnore
private long lastSeen;
@JsonIgnore
private long pushTimestamp;
}
private boolean serializedEquals(final Object primary, final Object secondary) throws JsonProcessingException {
final byte[] primarySerialized = migrationComparisonMapper.writeValueAsBytes(primary);
final byte[] secondarySerialized = migrationComparisonMapper.writeValueAsBytes(secondary);
final int serializeCompare = Arrays.compare(primarySerialized, secondarySerialized);
return serializeCompare == 0;
}
} }

View File

@ -1,71 +0,0 @@
package org.whispersystems.textsecuregcm.storage;
import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
public class MigrationDeletedAccounts extends AbstractDynamoDbStore {
private final String tableName;
static final String KEY_UUID = "U";
public MigrationDeletedAccounts(DynamoDbClient dynamoDb, String tableName) {
super(dynamoDb);
this.tableName = tableName;
}
public void put(UUID uuid) {
db().putItem(PutItemRequest.builder()
.tableName(tableName)
.item(primaryKey(uuid))
.build());
}
public List<UUID> getRecentlyDeletedUuids() {
final List<UUID> uuids = new ArrayList<>();
Optional<ScanResponse> firstPage = db().scanPaginator(ScanRequest.builder()
.tableName(tableName)
.build()).stream().findAny(); // get the first available response
if (firstPage.isPresent()) {
for (Map<String, AttributeValue> item : firstPage.get().items()) {
// only process one page each time. If we have a significant backlog at the end of the migration
// we can handle it separately
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
}
}
return uuids;
}
public void delete(List<UUID> uuids) {
writeInBatches(uuids, (batch) -> {
List<WriteRequest> deletes = batch.stream().map((uuid) -> WriteRequest.builder().deleteRequest(DeleteRequest.builder()
.key(primaryKey(uuid))
.build()).build()).collect(Collectors.toList());
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
});
}
@VisibleForTesting
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid));
}
}

View File

@ -1,111 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import com.google.common.annotations.VisibleForTesting;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
import software.amazon.awssdk.services.dynamodb.paginators.ScanIterable;
public class MigrationMismatchedAccounts extends AbstractDynamoDbStore {
static final String KEY_UUID = "U";
static final String ATTR_TIMESTAMP = "T";
@VisibleForTesting
static final long MISMATCH_CHECK_DELAY_MILLIS = Duration.ofMinutes(1).toMillis();
private final String tableName;
private final Clock clock;
public void put(UUID uuid) {
final Map<String, AttributeValue> item = primaryKey(uuid);
item.put("T", AttributeValues.fromLong(clock.millis()));
db().putItem(PutItemRequest.builder()
.tableName(tableName)
.item(item)
.build());
}
public MigrationMismatchedAccounts(DynamoDbClient dynamoDb, String tableName) {
this(dynamoDb, tableName, Clock.systemUTC());
}
@VisibleForTesting
MigrationMismatchedAccounts(DynamoDbClient dynamoDb, String tableName, final Clock clock) {
super(dynamoDb);
this.tableName = tableName;
this.clock = clock;
}
/**
* returns a list of UUIDs stored in the table that have passed {@link #MISMATCH_CHECK_DELAY_MILLIS}
*/
public List<UUID> getUuids(int max) {
final List<UUID> uuids = new ArrayList<>();
final ScanIterable scanPaginator = db().scanPaginator(ScanRequest.builder()
.tableName(tableName)
.filterExpression("#timestamp <= :timestamp")
.expressionAttributeNames(Map.of("#timestamp", ATTR_TIMESTAMP))
.expressionAttributeValues(Map.of(":timestamp",
AttributeValues.fromLong(clock.millis() - MISMATCH_CHECK_DELAY_MILLIS)))
.build());
for (ScanResponse response : scanPaginator) {
for (Map<String, AttributeValue> item : response.items()) {
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
if (uuids.size() >= max) {
break;
}
}
if (uuids.size() >= max) {
break;
}
}
return uuids;
}
@VisibleForTesting
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
final HashMap<String, AttributeValue> item = new HashMap<>();
item.put(KEY_UUID, AttributeValues.fromUUID(uuid));
return item;
}
public void delete(final List<UUID> uuidsToDelete) {
writeInBatches(uuidsToDelete, (uuids -> {
final List<WriteRequest> deletes = uuids.stream()
.map(uuid -> WriteRequest.builder().deleteRequest(
DeleteRequest.builder().key(Map.of(KEY_UUID, AttributeValues.fromUUID(uuid))).build()).build())
.collect(Collectors.toList());
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
}));
}
}

View File

@ -1,112 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import net.logstash.logback.argument.StructuredArguments;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
public class MigrationMismatchedAccountsTableCrawler extends ManagedPeriodicWork {
private static final Logger logger = LoggerFactory.getLogger(MigrationMismatchedAccountsTableCrawler.class);
private static final Duration WORKER_TTL = Duration.ofMinutes(2);
private static final Duration RUN_INTERVAL = Duration.ofMinutes(1);
private static final String ACTIVE_WORKER_KEY = "migration_mismatched_accounts_crawler_cache_active_worker";
private static final int MAX_BATCH_SIZE = 5_000;
private static final Counter COMPARISONS_COUNTER = Metrics.counter(
name(MigrationMismatchedAccountsTableCrawler.class, "comparisons"));
private static final String MISMATCH_COUNTER_NAME = name(MigrationMismatchedAccountsTableCrawler.class, "mismatches");
private static final Counter ERRORS_COUNTER = Metrics.counter(
name(MigrationMismatchedAccountsTableCrawler.class, "errors"));
private final MigrationMismatchedAccounts mismatchedAccounts;
private final AccountsManager accountsManager;
private final Accounts accountsDb;
private final AccountsDynamoDb accountsDynamoDb;
private final DynamicConfigurationManager dynamicConfigurationManager;
public MigrationMismatchedAccountsTableCrawler(
final MigrationMismatchedAccounts mismatchedAccounts,
final AccountsManager accountsManager,
final Accounts accountsDb,
final AccountsDynamoDb accountsDynamoDb,
final DynamicConfigurationManager dynamicConfigurationManager,
final FaultTolerantRedisCluster cluster,
final ScheduledExecutorService executorService) throws IOException {
super(new ManagedPeriodicWorkLock(ACTIVE_WORKER_KEY, cluster), WORKER_TTL, RUN_INTERVAL, executorService);
this.mismatchedAccounts = mismatchedAccounts;
this.accountsManager = accountsManager;
this.accountsDb = accountsDb;
this.accountsDynamoDb = accountsDynamoDb;
this.dynamicConfigurationManager = dynamicConfigurationManager;
}
@Override
public void doPeriodicWork() {
if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
.isPostCheckMismatches()) {
return;
}
final List<UUID> uuids = this.mismatchedAccounts.getUuids(MAX_BATCH_SIZE);
final List<UUID> processedUuids = new ArrayList<>(uuids.size());
try {
for (UUID uuid : uuids) {
try {
final Optional<String> result = accountsManager.compareAccounts(accountsDb.get(uuid),
accountsDynamoDb.get(uuid));
COMPARISONS_COUNTER.increment();
result.ifPresent(mismatchType -> {
Metrics.counter(MISMATCH_COUNTER_NAME, "type", mismatchType)
.increment();
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
.isLogMismatches()) {
logger.info("Mismatch: {}", StructuredArguments.entries(Map.of(
"type", mismatchType,
"uuid", uuid)));
}
});
processedUuids.add(uuid);
} catch (final Exception e) {
ERRORS_COUNTER.increment();
logger.warn("Failed to check account mismatch", e);
}
}
} finally {
this.mismatchedAccounts.delete(processedUuids);
}
}
}

View File

@ -1,76 +0,0 @@
package org.whispersystems.textsecuregcm.storage;
import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
public class MigrationRetryAccounts extends AbstractDynamoDbStore {
private final String tableName;
static final String KEY_UUID = "U";
public MigrationRetryAccounts(DynamoDbClient dynamoDb, String tableName) {
super(dynamoDb);
this.tableName = tableName;
}
public void put(UUID uuid) {
db().putItem(PutItemRequest.builder()
.tableName(tableName)
.item(primaryKey(uuid))
.build());
}
public List<UUID> getUuids(int max) {
final List<UUID> uuids = new ArrayList<>();
for (ScanResponse response : db().scanPaginator(ScanRequest.builder().tableName(tableName).build())) {
for (Map<String, AttributeValue> item : response.items()) {
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
if (uuids.size() >= max) {
break;
}
}
if (uuids.size() >= max) {
break;
}
}
return uuids;
}
@VisibleForTesting
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid));
}
public void delete(final List<UUID> uuidsToDelete) {
writeInBatches(uuidsToDelete, (uuids -> {
final List<WriteRequest> deletes = uuids.stream()
.map(uuid -> WriteRequest.builder().deleteRequest(
DeleteRequest.builder().key(Map.of(KEY_UUID, AttributeValues.fromUUID(uuid))).build()).build())
.collect(Collectors.toList());
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
}));
}
}

View File

@ -1,88 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
public class MigrationRetryAccountsTableCrawler extends ManagedPeriodicWork {
private static final Logger logger = LoggerFactory.getLogger(MigrationRetryAccountsTableCrawler.class);
private static final Duration WORKER_TTL = Duration.ofMinutes(2);
private static final Duration RUN_INTERVAL = Duration.ofMinutes(15);
private static final String ACTIVE_WORKER_KEY = "migration_retry_accounts_crawler_cache_active_worker";
private static final int MAX_BATCH_SIZE = 5_000;
private static final Counter MIGRATED_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "migrated"));
private static final Counter ERROR_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "error"));
private static final Counter TOTAL_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "total"));
private final MigrationRetryAccounts retryAccounts;
private final AccountsManager accountsManager;
private final AccountsDynamoDb accountsDynamoDb;
public MigrationRetryAccountsTableCrawler(
final MigrationRetryAccounts retryAccounts,
final AccountsManager accountsManager,
final AccountsDynamoDb accountsDynamoDb,
final FaultTolerantRedisCluster cluster,
final ScheduledExecutorService executorService) throws IOException {
super(new ManagedPeriodicWorkLock(ACTIVE_WORKER_KEY, cluster), WORKER_TTL, RUN_INTERVAL, executorService);
this.retryAccounts = retryAccounts;
this.accountsManager = accountsManager;
this.accountsDynamoDb = accountsDynamoDb;
}
@Override
public void doPeriodicWork() {
final List<UUID> uuids = this.retryAccounts.getUuids(MAX_BATCH_SIZE);
final List<UUID> processedUuids = new ArrayList<>(uuids.size());
try {
for (UUID uuid : uuids) {
try {
final Optional<Account> maybeDynamoAccount = accountsDynamoDb.get(uuid);
if (maybeDynamoAccount.isEmpty()) {
accountsManager.get(uuid).ifPresent(account -> {
accountsDynamoDb.migrate(account);
MIGRATED_COUNTER.increment();
});
}
processedUuids.add(uuid);
TOTAL_COUNTER.increment();
} catch (final Exception e) {
ERROR_COUNTER.increment();
logger.warn("Failed to migrate account");
}
}
} finally {
this.retryAccounts.delete(processedUuids);
}
}
}

View File

@ -5,18 +5,16 @@
package org.whispersystems.textsecuregcm.storage; package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries; import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import org.jdbi.v3.core.JdbiException;
import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper;
import org.whispersystems.textsecuregcm.util.Constants;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import org.jdbi.v3.core.JdbiException;
import static com.codahale.metrics.MetricRegistry.name; import org.whispersystems.textsecuregcm.util.Constants;
public class Usernames { public class Usernames {
@ -34,7 +32,6 @@ public class Usernames {
public Usernames(FaultTolerantDatabase database) { public Usernames(FaultTolerantDatabase database) {
this.database = database; this.database = database;
this.database.getDatabase().registerRowMapper(new AccountRowMapper());
} }
public boolean put(UUID uuid, String username) { public boolean put(UUID uuid, String username) {

View File

@ -1,36 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage.mappers;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.core.statement.StatementContext;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.UUID;
public class AccountRowMapper implements RowMapper<Account> {
private static ObjectMapper mapper = SystemMapper.getMapper();
@Override
public Account map(ResultSet resultSet, StatementContext ctx) throws SQLException {
try {
Account account = mapper.readValue(resultSet.getString(Accounts.DATA), Account.class);
account.setNumber(resultSet.getString(Accounts.NUMBER));
account.setUuid(UUID.fromString(resultSet.getString(Accounts.UID)));
account.setVersion(resultSet.getInt(Accounts.VERSION));
return account;
} catch (IOException e) {
throw new SQLException(e);
}
}
}

View File

@ -20,9 +20,6 @@ import io.lettuce.core.resource.ClientResources;
import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Metrics;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import net.sourceforge.argparse4j.inf.Namespace; import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser; import net.sourceforge.argparse4j.inf.Subparser;
import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.core.Jdbi;
@ -30,14 +27,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.WhisperServerConfiguration; import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator; import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager; import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.AccountsManager.DeletionReason; import org.whispersystems.textsecuregcm.storage.AccountsManager.DeletionReason;
@ -49,9 +44,6 @@ import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesCache; import org.whispersystems.textsecuregcm.storage.MessagesCache;
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager; import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts;
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts;
import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts;
import org.whispersystems.textsecuregcm.storage.Profiles; import org.whispersystems.textsecuregcm.storage.Profiles;
import org.whispersystems.textsecuregcm.storage.ProfilesManager; import org.whispersystems.textsecuregcm.storage.ProfilesManager;
import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb; import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb;
@ -62,7 +54,6 @@ import org.whispersystems.textsecuregcm.storage.Usernames;
import org.whispersystems.textsecuregcm.storage.UsernamesManager; import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore; import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig; import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfiguration> { public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfiguration> {
@ -70,11 +61,9 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
private final Logger logger = LoggerFactory.getLogger(DeleteUserCommand.class); private final Logger logger = LoggerFactory.getLogger(DeleteUserCommand.class);
public DeleteUserCommand() { public DeleteUserCommand() {
super(new Application<WhisperServerConfiguration>() { super(new Application<>() {
@Override @Override
public void run(WhisperServerConfiguration configuration, Environment environment) public void run(WhisperServerConfiguration configuration, Environment environment) {
throws Exception
{
} }
}, "rmuser", "remove user"); }, "rmuser", "remove user");
@ -105,9 +94,6 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_delete_user", accountJdbi, configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration()); FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_delete_user", accountJdbi, configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
ClientResources redisClusterClientResources = ClientResources.builder().build(); ClientResources redisClusterClientResources = ClientResources.builder().build();
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
new LinkedBlockingDeque<>());
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig.client( DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig.client(
configuration.getReportMessageDynamoDbConfiguration(), configuration.getReportMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@ -118,16 +104,9 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client( DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(
configuration.getAccountsDynamoDbConfiguration(), configuration.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(
configuration.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
accountsDynamoDbMigrationThreadPool);
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client( DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(
configuration.getDeletedAccountsDynamoDbConfiguration(), configuration.getDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient migrationMismatchedAccountsDynamoDb = DynamoDbFromConfig.client(
configuration.getMigrationMismatchedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster",
configuration.getCacheClusterConfiguration(), redisClusterClientResources); configuration.getCacheClusterConfiguration(), redisClusterClientResources);
@ -151,39 +130,41 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
configuration.getAppConfig().getConfigurationName()); configuration.getAppConfig().getConfigurationName());
dynamicConfigurationManager.start(); dynamicConfigurationManager.start();
ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(
dynamicConfigurationManager);
DynamoDbClient migrationDeletedAccountsDynamoDb = DynamoDbFromConfig.client(
configuration.getMigrationDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(
configuration.getMigrationRetryAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig.client(configuration.getPendingAccountsDynamoDbConfiguration(), DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig.client(configuration.getPendingAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
AmazonDynamoDB deletedAccountsLockDynamoDbClient = AmazonDynamoDBClientBuilder.standard() AmazonDynamoDB deletedAccountsLockDynamoDbClient = AmazonDynamoDBClientBuilder.standard()
.withRegion(configuration.getDeletedAccountsLockDynamoDbConfiguration().getRegion()) .withRegion(configuration.getDeletedAccountsLockDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientExecutionTimeout().toMillis())) .withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(
.withRequestTimeout((int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientRequestTimeout().toMillis())) ((int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientExecutionTimeout()
.toMillis()))
.withRequestTimeout(
(int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientRequestTimeout()
.toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance()) .withCredentials(InstanceProfileCredentialsProvider.getInstance())
.build(); .build();
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(), configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName()); DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(migrationDeletedAccountsDynamoDb, configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName()); configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(),
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb, configuration.getMigrationRetryAccountsDynamoDbConfiguration().getTableName()); configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient, configuration.getPendingAccountsDynamoDbConfiguration().getTableName()); VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient,
configuration.getPendingAccountsDynamoDbConfiguration().getTableName());
Accounts accounts = new Accounts(accountDatabase); AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient,
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, accountsDynamoDbMigrationThreadPool, configuration.getAccountsDynamoDbConfiguration().getTableName(), configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts); configuration.getAccountsDynamoDbConfiguration().getTableName(),
Usernames usernames = new Usernames(accountDatabase); configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName());
Profiles profiles = new Profiles(accountDatabase); Usernames usernames = new Usernames(accountDatabase);
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase); Profiles profiles = new Profiles(accountDatabase);
KeysDynamoDb keysDynamoDb = new KeysDynamoDb(preKeysDynamoDb, configuration.getKeysDynamoDbConfiguration().getTableName()); ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb, configuration.getMessageDynamoDbConfiguration().getTableName(), configuration.getMessageDynamoDbConfiguration().getTimeToLive()); KeysDynamoDb keysDynamoDb = new KeysDynamoDb(preKeysDynamoDb,
FaultTolerantRedisCluster messageInsertCacheCluster = new FaultTolerantRedisCluster("message_insert_cluster", configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources); configuration.getKeysDynamoDbConfiguration().getTableName());
FaultTolerantRedisCluster messageReadDeleteCluster = new FaultTolerantRedisCluster("message_read_delete_cluster", configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources); MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb,
configuration.getMessageDynamoDbConfiguration().getTableName(),
configuration.getMessageDynamoDbConfiguration().getTimeToLive());
FaultTolerantRedisCluster messageInsertCacheCluster = new FaultTolerantRedisCluster("message_insert_cluster",
configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
FaultTolerantRedisCluster messageReadDeleteCluster = new FaultTolerantRedisCluster("message_read_delete_cluster",
configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster",
configuration.getMetricsClusterConfiguration(), redisClusterClientResources); configuration.getMetricsClusterConfiguration(), redisClusterClientResources);
SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor, SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor,
@ -203,16 +184,13 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
Metrics.globalRegistry); Metrics.globalRegistry);
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager, MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager,
reportMessageManager); reportMessageManager);
MigrationMismatchedAccounts mismatchedAccounts = new MigrationMismatchedAccounts(
migrationMismatchedAccountsDynamoDb,
configuration.getMigrationMismatchedAccountsDynamoDbConfiguration().getTableName());
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts, DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
deletedAccountsLockDynamoDbClient, deletedAccountsLockDynamoDbClient,
configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName()); configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts); StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, AccountsManager accountsManager = new AccountsManager(accountsDynamoDb, cacheCluster,
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, mismatchedAccounts, usernamesManager, deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager,
profilesManager, pendingAccountsManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, profilesManager, pendingAccountsManager, secureStorageClient, secureBackupClient,
dynamicConfigurationManager); dynamicConfigurationManager);
for (String user : users) { for (String user : users) {

View File

@ -21,22 +21,17 @@ import io.micrometer.core.instrument.Metrics;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import net.sourceforge.argparse4j.inf.Namespace; import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser; import net.sourceforge.argparse4j.inf.Subparser;
import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.core.Jdbi;
import org.whispersystems.textsecuregcm.WhisperServerConfiguration; import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator; import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager; import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.DeletedAccounts; import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
@ -47,9 +42,6 @@ import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesCache; import org.whispersystems.textsecuregcm.storage.MessagesCache;
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager; import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts;
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts;
import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts;
import org.whispersystems.textsecuregcm.storage.Profiles; import org.whispersystems.textsecuregcm.storage.Profiles;
import org.whispersystems.textsecuregcm.storage.ProfilesManager; import org.whispersystems.textsecuregcm.storage.ProfilesManager;
import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb; import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb;
@ -60,7 +52,6 @@ import org.whispersystems.textsecuregcm.storage.Usernames;
import org.whispersystems.textsecuregcm.storage.UsernamesManager; import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore; import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig; import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperServerConfiguration> { public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperServerConfiguration> {
@ -105,9 +96,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration()); configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
ClientResources redisClusterClientResources = ClientResources.builder().build(); ClientResources redisClusterClientResources = ClientResources.builder().build();
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
new LinkedBlockingDeque<>());
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig
.client(configuration.getReportMessageDynamoDbConfiguration(), .client(configuration.getReportMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@ -118,10 +106,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig
.client(configuration.getAccountsDynamoDbConfiguration(), .client(configuration.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig
.asyncClient(configuration.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
accountsDynamoDbMigrationThreadPool);
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig
.client(configuration.getDeletedAccountsDynamoDbConfiguration(), .client(configuration.getDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@ -148,18 +132,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
configuration.getAppConfig().getConfigurationName()); configuration.getAppConfig().getConfigurationName());
dynamicConfigurationManager.start(); dynamicConfigurationManager.start();
ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(
dynamicConfigurationManager);
DynamoDbClient migrationDeletedAccountsDynamoDb = DynamoDbFromConfig
.client(configuration.getMigrationDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient migrationMismatchedAccountsDynamoDb = DynamoDbFromConfig
.client(configuration.getMigrationMismatchedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig
.client(configuration.getMigrationRetryAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig
.client(configuration.getPendingAccountsDynamoDbConfiguration(), .client(configuration.getPendingAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@ -178,18 +150,13 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(), configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(),
configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName()); configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(migrationDeletedAccountsDynamoDb,
configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb,
configuration.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient, VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient,
configuration.getPendingAccountsDynamoDbConfiguration().getTableName()); configuration.getPendingAccountsDynamoDbConfiguration().getTableName());
Accounts accounts = new Accounts(accountDatabase); AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient,
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, configuration.getAccountsDynamoDbConfiguration().getTableName(),
accountsDynamoDbMigrationThreadPool, configuration.getAccountsDynamoDbConfiguration().getTableName(), configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName()
configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, );
migrationRetryAccounts);
Usernames usernames = new Usernames(accountDatabase); Usernames usernames = new Usernames(accountDatabase);
Profiles profiles = new Profiles(accountDatabase); Profiles profiles = new Profiles(accountDatabase);
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase); ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
@ -221,17 +188,14 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
Metrics.globalRegistry); Metrics.globalRegistry);
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager, MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager,
reportMessageManager); reportMessageManager);
MigrationMismatchedAccounts mismatchedAccounts = new MigrationMismatchedAccounts(
migrationMismatchedAccountsDynamoDb,
configuration.getMigrationMismatchedAccountsDynamoDbConfiguration().getTableName());
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts, DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
deletedAccountsLockDynamoDbClient, deletedAccountsLockDynamoDbClient,
configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName()); configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts); StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, AccountsManager accountsManager = new AccountsManager(accountsDynamoDb, cacheCluster,
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, mismatchedAccounts, usernamesManager, deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager,
profilesManager, profilesManager,
pendingAccountsManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, pendingAccountsManager, secureStorageClient, secureBackupClient,
dynamicConfigurationManager); dynamicConfigurationManager);
Optional<Account> maybeAccount; Optional<Account> maybeAccount;

View File

@ -1,47 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.workers;
import net.sourceforge.argparse4j.inf.Namespace;
import org.jdbi.v3.core.Jdbi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
import io.dropwizard.cli.ConfiguredCommand;
import io.dropwizard.setup.Bootstrap;
public class VacuumCommand extends ConfiguredCommand<WhisperServerConfiguration> {
private final Logger logger = LoggerFactory.getLogger(VacuumCommand.class);
public VacuumCommand() {
super("vacuum", "Vacuum Postgres Tables");
}
@Override
protected void run(Bootstrap<WhisperServerConfiguration> bootstrap,
Namespace namespace,
WhisperServerConfiguration config)
throws Exception
{
DatabaseConfiguration accountDbConfig = config.getAbuseDatabaseConfiguration();
Jdbi accountJdbi = Jdbi.create(accountDbConfig.getUrl(), accountDbConfig.getUser(), accountDbConfig.getPassword());
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_vacuum", accountJdbi, accountDbConfig.getCircuitBreakerConfiguration());
Accounts accounts = new Accounts(accountDatabase);
logger.info("Vacuuming accounts...");
accounts.vacuum();
Thread.sleep(3000);
System.exit(0);
}
}

View File

@ -320,30 +320,19 @@ class DynamicConfigurationTest {
final DynamicConfiguration emptyConfig = final DynamicConfiguration emptyConfig =
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow(); DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isBackgroundMigrationEnabled()); assertEquals(10, emptyConfig.getAccountsDynamoDbMigrationConfiguration().getDynamoCrawlerScanPageSize());
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isDeleteEnabled());
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isWriteEnabled());
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isReadEnabled());
} }
{ {
final String accountsDynamoDbMigrationConfig = final String accountsDynamoDbMigrationConfig =
"accountsDynamoDbMigration:\n" "accountsDynamoDbMigration:\n"
+ " backgroundMigrationEnabled: true\n" + " dynamoCrawlerScanPageSize: 5000";
+ " backgroundMigrationExecutorThreads: 100\n"
+ " deleteEnabled: true\n"
+ " readEnabled: true\n"
+ " writeEnabled: true";
final DynamicAccountsDynamoDbMigrationConfiguration config = final DynamicAccountsDynamoDbMigrationConfiguration config =
DynamicConfigurationManager.parseConfiguration(accountsDynamoDbMigrationConfig).orElseThrow() DynamicConfigurationManager.parseConfiguration(accountsDynamoDbMigrationConfig).orElseThrow()
.getAccountsDynamoDbMigrationConfiguration(); .getAccountsDynamoDbMigrationConfiguration();
assertTrue(config.isBackgroundMigrationEnabled()); assertEquals(5000, config.getDynamoCrawlerScanPageSize());
assertEquals(100, config.getBackgroundMigrationExecutorThreads());
assertTrue(config.isDeleteEnabled());
assertTrue(config.isWriteEnabled());
assertTrue(config.isReadEnabled());
} }
} }

View File

@ -18,11 +18,8 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutorService;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest; import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterTest { public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterTest {
@ -58,18 +55,15 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT
when(firstAccount.getUuid()).thenReturn(FIRST_UUID); when(firstAccount.getUuid()).thenReturn(FIRST_UUID);
when(secondAccount.getUuid()).thenReturn(SECOND_UUID); when(secondAccount.getUuid()).thenReturn(SECOND_UUID);
when(accountsManager.getAllFrom(CHUNK_SIZE)).thenReturn(new AccountCrawlChunk(List.of(firstAccount), FIRST_UUID)); when(accountsManager.getAllFromDynamo(CHUNK_SIZE)).thenReturn(
when(accountsManager.getAllFrom(any(UUID.class), eq(CHUNK_SIZE))) new AccountCrawlChunk(List.of(firstAccount), FIRST_UUID));
when(accountsManager.getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE)))
.thenReturn(new AccountCrawlChunk(List.of(secondAccount), SECOND_UUID)) .thenReturn(new AccountCrawlChunk(List.of(secondAccount), SECOND_UUID))
.thenReturn(new AccountCrawlChunk(Collections.emptyList(), null)); .thenReturn(new AccountCrawlChunk(Collections.emptyList(), null));
final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn(mock(DynamicAccountsDynamoDbMigrationConfiguration.class));
final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache(getRedisCluster()); final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache(getRedisCluster());
accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, crawlerCache, List.of(listener), CHUNK_SIZE, accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, crawlerCache, List.of(listener), CHUNK_SIZE,
CHUNK_INTERVAL_MS, mock(ExecutorService.class), dynamicConfigurationManager); CHUNK_INTERVAL_MS);
} }
@Test @Test
@ -78,9 +72,9 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT
assertFalse(accountDatabaseCrawler.doPeriodicWork()); assertFalse(accountDatabaseCrawler.doPeriodicWork());
assertFalse(accountDatabaseCrawler.doPeriodicWork()); assertFalse(accountDatabaseCrawler.doPeriodicWork());
verify(accountsManager).getAllFrom(CHUNK_SIZE); verify(accountsManager).getAllFromDynamo(CHUNK_SIZE);
verify(accountsManager).getAllFrom(FIRST_UUID, CHUNK_SIZE); verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE);
verify(accountsManager).getAllFrom(SECOND_UUID, CHUNK_SIZE); verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE);
verify(listener).onCrawlStart(); verify(listener).onCrawlStart();
verify(listener).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount)); verify(listener).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));
@ -98,9 +92,9 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT
assertFalse(accountDatabaseCrawler.doPeriodicWork()); assertFalse(accountDatabaseCrawler.doPeriodicWork());
assertFalse(accountDatabaseCrawler.doPeriodicWork()); assertFalse(accountDatabaseCrawler.doPeriodicWork());
verify(accountsManager, times(2)).getAllFrom(CHUNK_SIZE); verify(accountsManager, times(2)).getAllFromDynamo(CHUNK_SIZE);
verify(accountsManager).getAllFrom(FIRST_UUID, CHUNK_SIZE); verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE);
verify(accountsManager).getAllFrom(SECOND_UUID, CHUNK_SIZE); verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE);
verify(listener, times(2)).onCrawlStart(); verify(listener, times(2)).onCrawlStart();
verify(listener, times(2)).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount)); verify(listener, times(2)).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));

View File

@ -1,271 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.opentable.db.postgres.embedded.LiquibasePreparer;
import com.opentable.db.postgres.junit5.EmbeddedPostgresExtension;
import com.opentable.db.postgres.junit5.PreparedDbExtension;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.jdbi.v3.core.Jdbi;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.limits.RateLimiter;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.tests.util.SynchronousExecutorService;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.Projection;
import software.amazon.awssdk.services.dynamodb.model.ProjectionType;
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class AccountsDynamoDbMigrationCrawlerIntegrationTest {
private static final int CHUNK_SIZE = 20;
private static final long CHUNK_INTERVAL_MS = 0;
private static final String ACCOUNTS_TABLE_NAME = "accounts_test";
private static final String KEYS_TABLE_NAME = "keys_test";
private static final String MIGRATION_DELETED_ACCOUNTS_TABLE_NAME = "migration_deleted_accounts_test";
private static final String MIGRATION_RETRY_ACCOUNTS_TABLE_NAME = "migration_retry_accounts_test";
private static final String NUMBERS_TABLE_NAME = "numbers_test";
private static final String VERIFICATION_CODE_TABLE_NAME = "verification_code_test";
@RegisterExtension
static final RedisClusterExtension REDIS_CLUSTER_EXTENSION = RedisClusterExtension.builder().build();
@RegisterExtension
static final DynamoDbExtension KEYS_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
.tableName(KEYS_TABLE_NAME)
.hashKey("U")
.rangeKey("DK")
.attributeDefinition(AttributeDefinition.builder()
.attributeName("U")
.attributeType(ScalarAttributeType.B)
.build())
.attributeDefinition(AttributeDefinition.builder()
.attributeName("DK")
.attributeType(ScalarAttributeType.B)
.build())
.build();
@RegisterExtension
static final DynamoDbExtension VERIFICATION_CODE_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
.tableName(VERIFICATION_CODE_TABLE_NAME)
.hashKey("P")
.attributeDefinition(AttributeDefinition.builder()
.attributeName("P")
.attributeType(ScalarAttributeType.S)
.build())
.build();
@RegisterExtension
static PreparedDbExtension db = EmbeddedPostgresExtension
.preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml"));
@RegisterExtension
static DynamoDbExtension ACCOUNTS_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
.tableName(ACCOUNTS_TABLE_NAME)
.hashKey("U")
.attributeDefinition(AttributeDefinition.builder()
.attributeName("U")
.attributeType(ScalarAttributeType.B)
.build())
.build();
private static final String NEEDS_RECONCILIATION_INDEX_NAME = "needs_reconciliation_test";
@RegisterExtension
static final DynamoDbExtension DELETED_ACCOUNTS_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
.tableName("deleted_accounts_test")
.hashKey(DeletedAccounts.KEY_ACCOUNT_E164)
.attributeDefinition(AttributeDefinition.builder()
.attributeName(DeletedAccounts.KEY_ACCOUNT_E164)
.attributeType(ScalarAttributeType.S).build())
.attributeDefinition(AttributeDefinition.builder()
.attributeName(DeletedAccounts.ATTR_NEEDS_CDS_RECONCILIATION)
.attributeType(ScalarAttributeType.N)
.build())
.globalSecondaryIndex(GlobalSecondaryIndex.builder()
.indexName(NEEDS_RECONCILIATION_INDEX_NAME)
.keySchema(
KeySchemaElement.builder().attributeName(DeletedAccounts.KEY_ACCOUNT_E164).keyType(KeyType.HASH).build(),
KeySchemaElement.builder().attributeName(DeletedAccounts.ATTR_NEEDS_CDS_RECONCILIATION)
.keyType(KeyType.RANGE).build())
.projection(Projection.builder().projectionType(ProjectionType.INCLUDE)
.nonKeyAttributes(DeletedAccounts.ATTR_ACCOUNT_UUID).build())
.provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(10L).writeCapacityUnits(10L).build())
.build())
.build();
@RegisterExtension
static DynamoDbExtension DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
.tableName("deleted_accounts_lock_test")
.hashKey(DeletedAccounts.KEY_ACCOUNT_E164)
.attributeDefinition(AttributeDefinition.builder()
.attributeName(DeletedAccounts.KEY_ACCOUNT_E164)
.attributeType(ScalarAttributeType.S).build())
.build();
private DynamicAccountsDynamoDbMigrationConfiguration accountMigrationConfiguration;
private AccountsManager accountsManager;
private AccountDatabaseCrawler accountDatabaseCrawler;
private Accounts accounts;
private AccountsDynamoDb accountsDynamoDb;
@BeforeEach
void setUp() throws Exception {
createAdditionalDynamoDbTables();
final DeletedAccounts deletedAccounts = new DeletedAccounts(DELETED_ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(),
DELETED_ACCOUNTS_DYNAMODB_EXTENSION.getTableName(),
NEEDS_RECONCILIATION_INDEX_NAME);
final DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION.getLegacyDynamoClient(),
DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION.getTableName());
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(), MIGRATION_DELETED_ACCOUNTS_TABLE_NAME);
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(
(ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient()),
MIGRATION_RETRY_ACCOUNTS_TABLE_NAME);
accountsDynamoDb = new AccountsDynamoDb(
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(),
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbAsyncClient(),
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
ACCOUNTS_DYNAMODB_EXTENSION.getTableName(),
NUMBERS_TABLE_NAME,
migrationDeletedAccounts,
migrationRetryAccounts);
final KeysDynamoDb keysDynamoDb = new KeysDynamoDb(KEYS_DYNAMODB_EXTENSION.getDynamoDbClient(), KEYS_TABLE_NAME);
accounts = new Accounts(new FaultTolerantDatabase("accountsTest",
Jdbi.create(db.getTestDatabase()),
new CircuitBreakerConfiguration()));
final DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
final RateLimiters rateLimiters = mock(RateLimiters.class);
when(rateLimiters.getVerifyLimiter()).thenReturn(mock(RateLimiter.class));
final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
accountMigrationConfiguration = new DynamicAccountsDynamoDbMigrationConfiguration();
accountMigrationConfiguration.setBackgroundMigrationEnabled(true);
accountMigrationConfiguration.setLogMismatches(true);
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn(accountMigrationConfiguration);
final ExperimentEnrollmentManager experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class);
when(experimentEnrollmentManager.isEnrolled(any(UUID.class), eq("accountsDynamoDbMigration"))).thenReturn(true);
accountsManager = new AccountsManager(
accounts,
accountsDynamoDb,
REDIS_CLUSTER_EXTENSION.getRedisCluster(),
deletedAccountsManager,
directoryQueue,
keysDynamoDb,
mock(MessagesManager.class),
mock(MigrationMismatchedAccounts.class),
mock(UsernamesManager.class),
mock(ProfilesManager.class),
mock(StoredVerificationCodeManager.class),
mock(SecureStorageClient.class),
mock(SecureBackupClient.class),
experimentEnrollmentManager,
dynamicConfigurationManager);
final AccountsDynamoDbMigrator dynamoDbMigrator = new AccountsDynamoDbMigrator(accountsDynamoDb,
dynamicConfigurationManager);
final PushFeedbackProcessor pushFeedbackProcessor = new PushFeedbackProcessor(accountsManager);
final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache(
REDIS_CLUSTER_EXTENSION.getRedisCluster());
// Using a synchronous service doesnt meaningfully impact the test
final ExecutorService chunkPreReadExecutorService = new SynchronousExecutorService();
accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, crawlerCache,
List.of(dynamoDbMigrator, pushFeedbackProcessor), CHUNK_SIZE,
CHUNK_INTERVAL_MS, chunkPreReadExecutorService, dynamicConfigurationManager);
}
void createAdditionalDynamoDbTables() {
CreateTableRequest createNumbersTableRequest = CreateTableRequest.builder()
.tableName(NUMBERS_TABLE_NAME)
.keySchema(KeySchemaElement.builder()
.attributeName("P")
.keyType(KeyType.HASH)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName("P")
.attributeType(ScalarAttributeType.S)
.build())
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
.build();
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createNumbersTableRequest);
final CreateTableRequest createMigrationDeletedAccountsTableRequest = CreateTableRequest.builder()
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
.keySchema(KeySchemaElement.builder()
.attributeName("U")
.keyType(KeyType.HASH)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName("U")
.attributeType(ScalarAttributeType.B)
.build())
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
.build();
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createMigrationDeletedAccountsTableRequest);
final CreateTableRequest createMigrationRetryAccountsTableRequest = CreateTableRequest.builder()
.tableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME)
.keySchema(KeySchemaElement.builder()
.attributeName("U")
.keyType(KeyType.HASH)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName("U")
.attributeType(ScalarAttributeType.B)
.build())
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
.build();
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createMigrationRetryAccountsTableRequest);
}
}

View File

@ -24,10 +24,6 @@ import java.util.Optional;
import java.util.Random; import java.util.Random;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.jdbi.v3.core.transaction.TransactionException; import org.jdbi.v3.core.transaction.TransactionException;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Disabled;
@ -36,21 +32,15 @@ import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.entities.SignedPreKey; import org.whispersystems.textsecuregcm.entities.SignedPreKey;
import org.whispersystems.textsecuregcm.util.AttributeValues; import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException; import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest; import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
import software.amazon.awssdk.services.dynamodb.model.KeyType; import software.amazon.awssdk.services.dynamodb.model.KeyType;
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest; import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest;
import software.amazon.awssdk.services.dynamodb.model.TransactionConflictException; import software.amazon.awssdk.services.dynamodb.model.TransactionConflictException;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest; import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
@ -59,8 +49,6 @@ class AccountsDynamoDbTest {
private static final String ACCOUNTS_TABLE_NAME = "accounts_test"; private static final String ACCOUNTS_TABLE_NAME = "accounts_test";
private static final String NUMBERS_TABLE_NAME = "numbers_test"; private static final String NUMBERS_TABLE_NAME = "numbers_test";
private static final String MIGRATION_DELETED_ACCOUNTS_TABLE_NAME = "migration_deleted_accounts_test";
private static final String MIGRATION_RETRY_ACCOUNTS_TABLE_NAME = "migration_retry_accounts_test";
@RegisterExtension @RegisterExtension
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder() static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
@ -91,50 +79,11 @@ class AccountsDynamoDbTest {
dynamoDbExtension.getDynamoDbClient().createTable(createNumbersTableRequest); dynamoDbExtension.getDynamoDbClient().createTable(createNumbersTableRequest);
final CreateTableRequest createMigrationDeletedAccountsTableRequest = CreateTableRequest.builder()
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
.keySchema(KeySchemaElement.builder()
.attributeName(MigrationDeletedAccounts.KEY_UUID)
.keyType(KeyType.HASH)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName(MigrationDeletedAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
.build();
dynamoDbExtension.getDynamoDbClient().createTable(createMigrationDeletedAccountsTableRequest);
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(
dynamoDbExtension.getDynamoDbClient(), MIGRATION_DELETED_ACCOUNTS_TABLE_NAME);
final CreateTableRequest createMigrationRetryAccountsTableRequest = CreateTableRequest.builder()
.tableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME)
.keySchema(KeySchemaElement.builder()
.attributeName(MigrationRetryAccounts.KEY_UUID)
.keyType(KeyType.HASH)
.build())
.attributeDefinitions(AttributeDefinition.builder()
.attributeName(MigrationRetryAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
.build();
dynamoDbExtension.getDynamoDbClient().createTable(createMigrationRetryAccountsTableRequest);
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts((dynamoDbExtension.getDynamoDbClient()),
MIGRATION_RETRY_ACCOUNTS_TABLE_NAME);
this.accountsDynamoDb = new AccountsDynamoDb( this.accountsDynamoDb = new AccountsDynamoDb(
dynamoDbExtension.getDynamoDbClient(), dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getDynamoDbAsyncClient(),
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
dynamoDbExtension.getTableName(), dynamoDbExtension.getTableName(),
NUMBERS_TABLE_NAME, NUMBERS_TABLE_NAME
migrationDeletedAccounts, );
migrationRetryAccounts);
} }
@Test @Test
@ -283,15 +232,13 @@ class AccountsDynamoDbTest {
void testUpdateWithMockTransactionConflictException() { void testUpdateWithMockTransactionConflictException() {
final DynamoDbClient dynamoDbClient = mock(DynamoDbClient.class); final DynamoDbClient dynamoDbClient = mock(DynamoDbClient.class);
accountsDynamoDb = new AccountsDynamoDb(dynamoDbClient, mock(DynamoDbAsyncClient.class), accountsDynamoDb = new AccountsDynamoDb(dynamoDbClient,
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()), dynamoDbExtension.getTableName(), NUMBERS_TABLE_NAME);
dynamoDbExtension.getTableName(), NUMBERS_TABLE_NAME, mock(MigrationDeletedAccounts.class),
mock(MigrationRetryAccounts.class));
when(dynamoDbClient.updateItem(any(UpdateItemRequest.class))) when(dynamoDbClient.updateItem(any(UpdateItemRequest.class)))
.thenThrow(TransactionConflictException.class); .thenThrow(TransactionConflictException.class);
Device device = generateDevice (1 ); Device device = generateDevice(1);
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device)); Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
assertThatThrownBy(() -> accountsDynamoDb.update(account)).isInstanceOfAny(ContestedOptimisticLockException.class); assertThatThrownBy(() -> accountsDynamoDb.update(account)).isInstanceOfAny(ContestedOptimisticLockException.class);
@ -376,33 +323,6 @@ class AccountsDynamoDbTest {
verifyStoredState(recreatedAccount.getNumber(), recreatedAccount.getUuid(), verifyStoredState(recreatedAccount.getNumber(), recreatedAccount.getUuid(),
accountsDynamoDb.get(recreatedAccount.getUuid()).get(), recreatedAccount); accountsDynamoDb.get(recreatedAccount.getUuid()).get(), recreatedAccount);
} }
verifyRecentlyDeletedAccountsTableItemCount(1);
Map<String, AttributeValue> primaryKey = MigrationDeletedAccounts.primaryKey(deletedAccount.getUuid());
assertThat(dynamoDbExtension.getDynamoDbClient().getItem(GetItemRequest.builder()
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
.key(Map.of(MigrationDeletedAccounts.KEY_UUID, primaryKey.get(MigrationDeletedAccounts.KEY_UUID)))
.build()))
.isNotNull();
accountsDynamoDb.deleteRecentlyDeletedUuids();
verifyRecentlyDeletedAccountsTableItemCount(0);
}
private void verifyRecentlyDeletedAccountsTableItemCount(int expectedItemCount) {
int totalItems = 0;
for (ScanResponse page : dynamoDbExtension.getDynamoDbClient().scanPaginator(ScanRequest.builder()
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
.build())) {
for (Map<String, AttributeValue> item : page.items()) {
totalItems++;
}
}
assertThat(totalItems).isEqualTo(expectedItemCount);
} }
@Test @Test
@ -437,9 +357,8 @@ class AccountsDynamoDbTest {
when(client.updateItem(any(UpdateItemRequest.class))) when(client.updateItem(any(UpdateItemRequest.class)))
.thenThrow(RuntimeException.class); .thenThrow(RuntimeException.class);
AccountsDynamoDb accounts = new AccountsDynamoDb(client, mock(DynamoDbAsyncClient.class), mock(ThreadPoolExecutor.class), ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME, mock( AccountsDynamoDb accounts = new AccountsDynamoDb(client, ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME);
MigrationDeletedAccounts.class), mock(MigrationRetryAccounts.class)); Account account = generateAccount("+14151112222", UUID.randomUUID());
Account account = generateAccount("+14151112222", UUID.randomUUID());
try { try {
accounts.update(account); accounts.update(account);
@ -472,42 +391,6 @@ class AccountsDynamoDbTest {
} }
} }
@Test
void testMigrate() throws ExecutionException, InterruptedException {
Device device = generateDevice (1 );
UUID firstUuid = UUID.randomUUID();
Account account = generateAccount("+14151112222", firstUuid, Collections.singleton(device));
boolean migrated = accountsDynamoDb.migrate(account).get();
assertThat(migrated).isTrue();
verifyStoredState("+14151112222", account.getUuid(), account, true);
migrated = accountsDynamoDb.migrate(account).get();
assertThat(migrated).isFalse();
verifyStoredState("+14151112222", account.getUuid(), account, true);
UUID secondUuid = UUID.randomUUID();
device = generateDevice(1);
Account accountRemigrationWithDifferentUuid = generateAccount("+14151112222", secondUuid, Collections.singleton(device));
migrated = accountsDynamoDb.migrate(accountRemigrationWithDifferentUuid).get();
assertThat(migrated).isFalse();
verifyStoredState("+14151112222", firstUuid, account, true);
account.setVersion(account.getVersion() + 1);
migrated = accountsDynamoDb.migrate(account).get();
assertThat(migrated).isTrue();
}
@Test @Test
void testCanonicallyDiscoverableSet() { void testCanonicallyDiscoverableSet() {
Device device = generateDevice(1); Device device = generateDevice(1);

View File

@ -23,7 +23,6 @@ import com.opentable.db.postgres.junit5.PreparedDbExtension;
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands; import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
import java.io.IOException; import java.io.IOException;
import java.time.Instant; import java.time.Instant;
import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.Random; import java.util.Random;
import java.util.UUID; import java.util.UUID;
@ -34,18 +33,14 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.jdbi.v3.core.Jdbi;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials; import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.entities.AccountAttributes; import org.whispersystems.textsecuregcm.entities.AccountAttributes;
import org.whispersystems.textsecuregcm.entities.SignedPreKey; import org.whispersystems.textsecuregcm.entities.SignedPreKey;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
@ -76,8 +71,6 @@ class AccountsManagerConcurrentModificationIntegrationTest {
.build()) .build())
.build(); .build();
private Accounts accounts;
private AccountsDynamoDb accountsDynamoDb; private AccountsDynamoDb accountsDynamoDb;
private AccountsManager accountsManager; private AccountsManager accountsManager;
@ -108,22 +101,9 @@ class AccountsManagerConcurrentModificationIntegrationTest {
accountsDynamoDb = new AccountsDynamoDb( accountsDynamoDb = new AccountsDynamoDb(
dynamoDbExtension.getDynamoDbClient(), dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getDynamoDbAsyncClient(),
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
dynamoDbExtension.getTableName(), dynamoDbExtension.getTableName(),
NUMBERS_TABLE_NAME, NUMBERS_TABLE_NAME
mock(MigrationDeletedAccounts.class), );
mock(MigrationRetryAccounts.class));
{
final CircuitBreakerConfiguration circuitBreakerConfiguration = new CircuitBreakerConfiguration();
circuitBreakerConfiguration.setIgnoredExceptions(List.of("org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException"));
FaultTolerantDatabase faultTolerantDatabase = new FaultTolerantDatabase("accountsTest",
Jdbi.create(db.getTestDatabase()),
circuitBreakerConfiguration);
accounts = new Accounts(faultTolerantDatabase);
}
{ {
final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class); final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
@ -131,17 +111,6 @@ class AccountsManagerConcurrentModificationIntegrationTest {
DynamicConfiguration dynamicConfiguration = new DynamicConfiguration(); DynamicConfiguration dynamicConfiguration = new DynamicConfiguration();
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration); when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
final ExperimentEnrollmentManager experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class);
final DynamicAccountsDynamoDbMigrationConfiguration config = dynamicConfiguration
.getAccountsDynamoDbMigrationConfiguration();
config.setDeleteEnabled(true);
config.setReadEnabled(true);
config.setWriteEnabled(true);
when(experimentEnrollmentManager.isEnrolled(any(UUID.class), anyString())).thenReturn(true);
commands = mock(RedisAdvancedClusterCommands.class); commands = mock(RedisAdvancedClusterCommands.class);
final DeletedAccountsManager deletedAccountsManager = mock(DeletedAccountsManager.class); final DeletedAccountsManager deletedAccountsManager = mock(DeletedAccountsManager.class);
@ -153,20 +122,17 @@ class AccountsManagerConcurrentModificationIntegrationTest {
}).when(deletedAccountsManager).lockAndTake(anyString(), any()); }).when(deletedAccountsManager).lockAndTake(anyString(), any());
accountsManager = new AccountsManager( accountsManager = new AccountsManager(
accounts,
accountsDynamoDb, accountsDynamoDb,
RedisClusterHelper.buildMockRedisCluster(commands), RedisClusterHelper.buildMockRedisCluster(commands),
deletedAccountsManager, deletedAccountsManager,
mock(DirectoryQueue.class), mock(DirectoryQueue.class),
mock(KeysDynamoDb.class), mock(KeysDynamoDb.class),
mock(MessagesManager.class), mock(MessagesManager.class),
mock(MigrationMismatchedAccounts.class),
mock(UsernamesManager.class), mock(UsernamesManager.class),
mock(ProfilesManager.class), mock(ProfilesManager.class),
mock(StoredVerificationCodeManager.class), mock(StoredVerificationCodeManager.class),
mock(SecureStorageClient.class), mock(SecureStorageClient.class),
mock(SecureBackupClient.class), mock(SecureBackupClient.class),
experimentEnrollmentManager,
dynamicConfigurationManager); dynamicConfigurationManager);
} }
} }
@ -225,14 +191,12 @@ class AccountsManagerConcurrentModificationIntegrationTest {
).join(); ).join();
final Account managerAccount = accountsManager.get(uuid).get(); final Account managerAccount = accountsManager.get(uuid).get();
final Account dbAccount = accounts.get(uuid).get();
final Account dynamoAccount = accountsDynamoDb.get(uuid).get(); final Account dynamoAccount = accountsDynamoDb.get(uuid).get();
final Account redisAccount = getLastAccountFromRedisMock(commands); final Account redisAccount = getLastAccountFromRedisMock(commands);
Stream.of( Stream.of(
new Pair<>("manager", managerAccount), new Pair<>("manager", managerAccount),
new Pair<>("db", dbAccount),
new Pair<>("dynamo", dynamoAccount), new Pair<>("dynamo", dynamoAccount),
new Pair<>("redis", redisAccount) new Pair<>("redis", redisAccount)
).forEach(pair -> ).forEach(pair ->

View File

@ -1,44 +0,0 @@
package org.whispersystems.textsecuregcm.storage;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class MigrationDeletedAccountsTest {
@RegisterExtension
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName("deleted_accounts_test")
.hashKey(MigrationDeletedAccounts.KEY_UUID)
.attributeDefinition(AttributeDefinition.builder()
.attributeName(MigrationDeletedAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.build();
@Test
void test() {
final MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getTableName());
UUID firstUuid = UUID.randomUUID();
UUID secondUuid = UUID.randomUUID();
assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().isEmpty());
migrationDeletedAccounts.put(firstUuid);
migrationDeletedAccounts.put(secondUuid);
assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().containsAll(List.of(firstUuid, secondUuid)));
migrationDeletedAccounts.delete(List.of(firstUuid, secondUuid));
assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().isEmpty());
}
}

View File

@ -1,56 +0,0 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.time.Clock;
import java.util.List;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class MigrationMismatchAccountsTest {
@RegisterExtension
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName("account_migration_mismatches_test")
.hashKey(MigrationRetryAccounts.KEY_UUID)
.attributeDefinition(AttributeDefinition.builder()
.attributeName(MigrationRetryAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.build();
@Test
void test() {
final Clock clock = mock(Clock.class);
when(clock.millis()).thenReturn(0L);
final MigrationMismatchedAccounts migrationMismatchedAccounts = new MigrationMismatchedAccounts(
dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getTableName(), clock);
UUID firstUuid = UUID.randomUUID();
UUID secondUuid = UUID.randomUUID();
assertTrue(migrationMismatchedAccounts.getUuids(10).isEmpty());
migrationMismatchedAccounts.put(firstUuid);
migrationMismatchedAccounts.put(secondUuid);
assertTrue(migrationMismatchedAccounts.getUuids(10).isEmpty());
when(clock.millis()).thenReturn(MigrationMismatchedAccounts.MISMATCH_CHECK_DELAY_MILLIS);
assertTrue(migrationMismatchedAccounts.getUuids(10).containsAll(List.of(firstUuid, secondUuid)));
}
}

View File

@ -1,40 +0,0 @@
package org.whispersystems.textsecuregcm.storage;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import java.util.UUID;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class MigrationRetryAccountsTest {
@RegisterExtension
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName("account_migration_errors_test")
.hashKey(MigrationRetryAccounts.KEY_UUID)
.attributeDefinition(AttributeDefinition.builder()
.attributeName(MigrationRetryAccounts.KEY_UUID)
.attributeType(ScalarAttributeType.B)
.build())
.build();
@Test
void test() {
final MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(dynamoDbExtension.getDynamoDbClient(),
dynamoDbExtension.getTableName());
UUID firstUuid = UUID.randomUUID();
UUID secondUuid = UUID.randomUUID();
assertTrue(migrationRetryAccounts.getUuids(10).isEmpty());
migrationRetryAccounts.put(firstUuid);
migrationRetryAccounts.put(secondUuid);
assertTrue(migrationRetryAccounts.getUuids(10).containsAll(List.of(firstUuid, secondUuid)));
}
}

View File

@ -16,7 +16,6 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.util.Arrays; import java.util.Arrays;
@ -25,10 +24,7 @@ import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.provider.ValueSource;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk; import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
@ -36,7 +32,6 @@ import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerRestartException; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerRestartException;
import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
class AccountDatabaseCrawlerTest { class AccountDatabaseCrawlerTest {
@ -55,22 +50,14 @@ class AccountDatabaseCrawlerTest {
private final ExecutorService chunkPreReadExecutorService = mock(ExecutorService.class); private final ExecutorService chunkPreReadExecutorService = mock(ExecutorService.class);
private final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
private final AccountDatabaseCrawler crawler = new AccountDatabaseCrawler(accounts, cache, Arrays.asList(listener), private final AccountDatabaseCrawler crawler = new AccountDatabaseCrawler(accounts, cache, Arrays.asList(listener),
CHUNK_SIZE, CHUNK_INTERVAL_MS, chunkPreReadExecutorService, dynamicConfigurationManager); CHUNK_SIZE, CHUNK_INTERVAL_MS);
private DynamicAccountsDynamoDbMigrationConfiguration dynamicAccountsDynamoDbMigrationConfiguration;
@BeforeEach @BeforeEach
void setup() { void setup() {
when(account1.getUuid()).thenReturn(ACCOUNT1); when(account1.getUuid()).thenReturn(ACCOUNT1);
when(account2.getUuid()).thenReturn(ACCOUNT2); when(account2.getUuid()).thenReturn(ACCOUNT2);
when(accounts.getAllFrom(anyInt())).thenReturn(new AccountCrawlChunk(Arrays.asList(account1, account2), ACCOUNT2));
when(accounts.getAllFrom(eq(ACCOUNT1), anyInt())).thenReturn(
new AccountCrawlChunk(Arrays.asList(account2), ACCOUNT2));
when(accounts.getAllFrom(eq(ACCOUNT2), anyInt())).thenReturn(new AccountCrawlChunk(Collections.emptyList(), null));
when(accounts.getAllFromDynamo(anyInt())).thenReturn( when(accounts.getAllFromDynamo(anyInt())).thenReturn(
new AccountCrawlChunk(Arrays.asList(account1, account2), ACCOUNT2)); new AccountCrawlChunk(Arrays.asList(account1, account2), ACCOUNT2));
when(accounts.getAllFromDynamo(eq(ACCOUNT1), anyInt())).thenReturn( when(accounts.getAllFromDynamo(eq(ACCOUNT1), anyInt())).thenReturn(
@ -81,17 +68,10 @@ class AccountDatabaseCrawlerTest {
when(cache.claimActiveWork(any(), anyLong())).thenReturn(true); when(cache.claimActiveWork(any(), anyLong())).thenReturn(true);
when(cache.isAccelerated()).thenReturn(false); when(cache.isAccelerated()).thenReturn(false);
final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
dynamicAccountsDynamoDbMigrationConfiguration = mock(DynamicAccountsDynamoDbMigrationConfiguration.class);
when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn(
dynamicAccountsDynamoDbMigrationConfiguration);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testCrawlStart() throws AccountDatabaseCrawlerRestartException {
void testCrawlStart(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException {
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
when(cache.getLastUuid()).thenReturn(Optional.empty()); when(cache.getLastUuid()).thenReturn(Optional.empty());
when(cache.getLastUuidDynamo()).thenReturn(Optional.empty()); when(cache.getLastUuidDynamo()).thenReturn(Optional.empty());
@ -99,20 +79,15 @@ class AccountDatabaseCrawlerTest {
assertThat(accelerated).isFalse(); assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(useDynamo ? 0 : 1)).getLastUuid(); verify(cache, times(0)).getLastUuid();
verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo(); verify(cache, times(1)).getLastUuidDynamo();
verify(listener, times(1)).onCrawlStart(); verify(listener, times(1)).onCrawlStart();
if (useDynamo) { verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE));
verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE));
} else {
verify(accounts, times(1)).getAllFrom(eq(CHUNK_SIZE));
verify(accounts, times(0)).getAllFrom(any(UUID.class), eq(CHUNK_SIZE));
}
verify(account1, times(0)).getUuid(); verify(account1, times(0)).getUuid();
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.empty()), eq(Arrays.asList(account1, account2))); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.empty()), eq(Arrays.asList(account1, account2)));
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
@ -123,10 +98,8 @@ class AccountDatabaseCrawlerTest {
verifyNoMoreInteractions(cache); verifyNoMoreInteractions(cache);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testCrawlChunk() throws AccountDatabaseCrawlerRestartException {
void testCrawlChunk(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException {
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
@ -134,18 +107,13 @@ class AccountDatabaseCrawlerTest {
assertThat(accelerated).isFalse(); assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(useDynamo ? 0: 1)).getLastUuid(); verify(cache, times(0)).getLastUuid();
verify(cache, times(useDynamo ? 1: 0)).getLastUuidDynamo(); verify(cache, times(1)).getLastUuidDynamo();
if (useDynamo) { verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
} else {
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
}
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
@ -157,46 +125,8 @@ class AccountDatabaseCrawlerTest {
verifyNoMoreInteractions(cache); verifyNoMoreInteractions(cache);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testCrawlChunkAccelerated() throws AccountDatabaseCrawlerRestartException {
void testCrawlChunk_useDynamoDedicatedMigrationCrawler(final boolean dedicatedMigrationCrawler) throws Exception {
crawler.setDedicatedDynamoMigrationCrawler(dedicatedMigrationCrawler);
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(true);
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
boolean accelerated = crawler.doPeriodicWork();
assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(dedicatedMigrationCrawler ? 1 : 0)).getLastUuid();
verify(cache, times(dedicatedMigrationCrawler ? 0 : 1)).getLastUuidDynamo();
if (dedicatedMigrationCrawler) {
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
} else {
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
}
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
verify(cache, times(dedicatedMigrationCrawler ? 1 : 0)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(dedicatedMigrationCrawler ? 0 : 1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyNoInteractions(account1);
verifyNoMoreInteractions(account2);
verifyNoMoreInteractions(accounts);
verifyNoMoreInteractions(listener);
verifyNoMoreInteractions(cache);
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testCrawlChunkAccelerated(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException {
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
when(cache.isAccelerated()).thenReturn(true); when(cache.isAccelerated()).thenReturn(true);
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
@ -205,22 +135,17 @@ class AccountDatabaseCrawlerTest {
assertThat(accelerated).isTrue(); assertThat(accelerated).isTrue();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(useDynamo ? 0 : 1)).getLastUuid(); verify(cache, times(0)).getLastUuid();
verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo(); verify(cache, times(1)).getLastUuidDynamo();
if (useDynamo) { verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
} else {
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
}
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyZeroInteractions(account1); verifyNoInteractions(account1);
verifyNoMoreInteractions(account2); verifyNoMoreInteractions(account2);
verifyNoMoreInteractions(accounts); verifyNoMoreInteractions(accounts);
@ -228,36 +153,30 @@ class AccountDatabaseCrawlerTest {
verifyNoMoreInteractions(cache); verifyNoMoreInteractions(cache);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testCrawlChunkRestart() throws AccountDatabaseCrawlerRestartException {
void testCrawlChunkRestart(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException {
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
doThrow(AccountDatabaseCrawlerRestartException.class).when(listener).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); doThrow(AccountDatabaseCrawlerRestartException.class).when(listener)
.timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
boolean accelerated = crawler.doPeriodicWork(); boolean accelerated = crawler.doPeriodicWork();
assertThat(accelerated).isFalse(); assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(useDynamo ? 0 : 1)).getLastUuid(); verify(cache, times(0)).getLastUuid();
verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo(); verify(cache, times(1)).getLastUuidDynamo();
if (useDynamo) { verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
} else {
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
}
verify(account2, times(0)).getNumber(); verify(account2, times(0)).getNumber();
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.empty())); verify(cache, times(0)).setLastUuid(eq(Optional.empty()));
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.empty())); verify(cache, times(1)).setLastUuidDynamo(eq(Optional.empty()));
verify(cache, times(1)).setAccelerated(false); verify(cache, times(1)).setAccelerated(false);
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyZeroInteractions(account1); verifyNoInteractions(account1);
verifyNoMoreInteractions(account2); verifyNoMoreInteractions(account2);
verifyNoMoreInteractions(accounts); verifyNoMoreInteractions(accounts);
@ -265,10 +184,8 @@ class AccountDatabaseCrawlerTest {
verifyNoMoreInteractions(cache); verifyNoMoreInteractions(cache);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testCrawlEnd() {
void testCrawlEnd(final boolean useDynamo) {
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2)); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT2)); when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT2));
@ -276,26 +193,21 @@ class AccountDatabaseCrawlerTest {
assertThat(accelerated).isFalse(); assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(useDynamo ? 0 : 1)).getLastUuid(); verify(cache, times(0)).getLastUuid();
verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo(); verify(cache, times(1)).getLastUuidDynamo();
if (useDynamo) { verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE));
} else {
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT2), eq(CHUNK_SIZE));
}
verify(account1, times(0)).getNumber(); verify(account1, times(0)).getNumber();
verify(account2, times(0)).getNumber(); verify(account2, times(0)).getNumber();
verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2))); verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2)));
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.empty())); verify(cache, times(0)).setLastUuid(eq(Optional.empty()));
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.empty())); verify(cache, times(1)).setLastUuidDynamo(eq(Optional.empty()));
verify(cache, times(1)).setAccelerated(false); verify(cache, times(1)).setAccelerated(false);
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
verifyZeroInteractions(account1); verifyNoInteractions(account1);
verifyZeroInteractions(account2); verifyNoInteractions(account2);
verifyNoMoreInteractions(accounts); verifyNoMoreInteractions(accounts);
verifyNoMoreInteractions(listener); verifyNoMoreInteractions(listener);

View File

@ -33,6 +33,7 @@ import java.util.UUID;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.Arguments;
@ -40,16 +41,13 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource; import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
import org.mockito.stubbing.Answer; import org.mockito.stubbing.Answer;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.entities.AccountAttributes; import org.whispersystems.textsecuregcm.entities.AccountAttributes;
import org.whispersystems.textsecuregcm.entities.SignedPreKey; import org.whispersystems.textsecuregcm.entities.SignedPreKey;
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException; import org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException;
@ -59,7 +57,6 @@ import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb; import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager; import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts;
import org.whispersystems.textsecuregcm.storage.ProfilesManager; import org.whispersystems.textsecuregcm.storage.ProfilesManager;
import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager; import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
import org.whispersystems.textsecuregcm.storage.UsernamesManager; import org.whispersystems.textsecuregcm.storage.UsernamesManager;
@ -68,12 +65,10 @@ import org.whispersystems.textsecuregcm.tests.util.RedisClusterHelper;
class AccountsManagerTest { class AccountsManagerTest {
private Accounts accounts;
private AccountsDynamoDb accountsDynamoDb; private AccountsDynamoDb accountsDynamoDb;
private DeletedAccountsManager deletedAccountsManager; private DeletedAccountsManager deletedAccountsManager;
private DirectoryQueue directoryQueue; private DirectoryQueue directoryQueue;
private DynamicConfigurationManager dynamicConfigurationManager; private DynamicConfigurationManager dynamicConfigurationManager;
private ExperimentEnrollmentManager experimentEnrollmentManager;
private KeysDynamoDb keys; private KeysDynamoDb keys;
private MessagesManager messagesManager; private MessagesManager messagesManager;
private ProfilesManager profilesManager; private ProfilesManager profilesManager;
@ -91,12 +86,10 @@ class AccountsManagerTest {
@BeforeEach @BeforeEach
void setup() throws InterruptedException { void setup() throws InterruptedException {
accounts = mock(Accounts.class);
accountsDynamoDb = mock(AccountsDynamoDb.class); accountsDynamoDb = mock(AccountsDynamoDb.class);
deletedAccountsManager = mock(DeletedAccountsManager.class); deletedAccountsManager = mock(DeletedAccountsManager.class);
directoryQueue = mock(DirectoryQueue.class); directoryQueue = mock(DirectoryQueue.class);
dynamicConfigurationManager = mock(DynamicConfigurationManager.class); dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class);
keys = mock(KeysDynamoDb.class); keys = mock(KeysDynamoDb.class);
messagesManager = mock(MessagesManager.class); messagesManager = mock(MessagesManager.class);
profilesManager = mock(ProfilesManager.class); profilesManager = mock(ProfilesManager.class);
@ -114,30 +107,24 @@ class AccountsManagerTest {
}).when(deletedAccountsManager).lockAndTake(anyString(), any()); }).when(deletedAccountsManager).lockAndTake(anyString(), any());
accountsManager = new AccountsManager( accountsManager = new AccountsManager(
accounts,
accountsDynamoDb, accountsDynamoDb,
RedisClusterHelper.buildMockRedisCluster(commands), RedisClusterHelper.buildMockRedisCluster(commands),
deletedAccountsManager, deletedAccountsManager,
directoryQueue, directoryQueue,
keys, keys,
messagesManager, messagesManager,
mock(MigrationMismatchedAccounts.class),
mock(UsernamesManager.class), mock(UsernamesManager.class),
profilesManager, profilesManager,
mock(StoredVerificationCodeManager.class), mock(StoredVerificationCodeManager.class),
mock(SecureStorageClient.class), mock(SecureStorageClient.class),
mock(SecureBackupClient.class), mock(SecureBackupClient.class),
experimentEnrollmentManager,
dynamicConfigurationManager); dynamicConfigurationManager);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testGetAccountByNumberInCache() {
void testGetAccountByNumberInCache(final boolean dynamoEnabled) {
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
enableDynamo(dynamoEnabled);
when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(uuid.toString()); when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(uuid.toString());
when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}"); when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}");
@ -150,31 +137,14 @@ class AccountsManagerTest {
verify(commands, times(1)).get(eq("AccountMap::+14152222222")); verify(commands, times(1)).get(eq("AccountMap::+14152222222"));
verify(commands, times(1)).get(eq("Account3::" + uuid)); verify(commands, times(1)).get(eq("Account3::" + uuid));
verifyNoMoreInteractions(commands); verifyNoMoreInteractions(commands);
verifyNoMoreInteractions(accounts);
verifyNoInteractions(accountsDynamoDb); verifyNoInteractions(accountsDynamoDb);
} }
private void enableDynamo(boolean dynamoEnabled) { @Test
final DynamicAccountsDynamoDbMigrationConfiguration config = dynamicConfigurationManager.getConfiguration() void testGetAccountByUuidInCache() {
.getAccountsDynamoDbMigrationConfiguration();
config.setDeleteEnabled(dynamoEnabled);
config.setReadEnabled(dynamoEnabled);
config.setWriteEnabled(dynamoEnabled);
when(experimentEnrollmentManager.isEnrolled(any(UUID.class), anyString()))
.thenReturn(dynamoEnabled);
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testGetAccountByUuidInCache(boolean dynamoEnabled) {
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
enableDynamo(dynamoEnabled);
when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}"); when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}");
Optional<Account> account = accountsManager.get(uuid); Optional<Account> account = accountsManager.get(uuid);
@ -186,22 +156,19 @@ class AccountsManagerTest {
verify(commands, times(1)).get(eq("Account3::" + uuid)); verify(commands, times(1)).get(eq("Account3::" + uuid));
verifyNoMoreInteractions(commands); verifyNoMoreInteractions(commands);
verifyNoMoreInteractions(accounts);
verifyNoInteractions(accountsDynamoDb); verifyNoInteractions(accountsDynamoDb);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testGetAccountByNumberNotInCache() {
void testGetAccountByNumberNotInCache(boolean dynamoEnabled) { final boolean dynamoEnabled = true;
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
enableDynamo(dynamoEnabled);
when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(null); when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(null);
when(accounts.get(eq("+14152222222"))).thenReturn(Optional.of(account)); when(accountsDynamoDb.get(eq("+14152222222"))).thenReturn(Optional.of(account));
Optional<Account> retrieved = accountsManager.get("+14152222222"); Optional<Account> retrieved = accountsManager.get("+14152222222");
@ -213,24 +180,18 @@ class AccountsManagerTest {
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString()); verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
verifyNoMoreInteractions(commands); verifyNoMoreInteractions(commands);
verify(accounts, times(1)).get(eq("+14152222222")); verify(accountsDynamoDb, times(1)).get(eq("+14152222222"));
verifyNoMoreInteractions(accounts);
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never())
.get(eq("+14152222222"));
verifyNoMoreInteractions(accountsDynamoDb); verifyNoMoreInteractions(accountsDynamoDb);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testGetAccountByUuidNotInCache() {
void testGetAccountByUuidNotInCache(boolean dynamoEnabled) { final boolean dynamoEnabled = true;
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
enableDynamo(dynamoEnabled);
when(commands.get(eq("Account3::" + uuid))).thenReturn(null); when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
when(accounts.get(eq(uuid))).thenReturn(Optional.of(account)); when(accountsDynamoDb.get(eq(uuid))).thenReturn(Optional.of(account));
Optional<Account> retrieved = accountsManager.get(uuid); Optional<Account> retrieved = accountsManager.get(uuid);
@ -242,25 +203,19 @@ class AccountsManagerTest {
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString()); verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
verifyNoMoreInteractions(commands); verifyNoMoreInteractions(commands);
verify(accounts, times(1)).get(eq(uuid)); verify(accountsDynamoDb, times(1)).get(eq(uuid));
verifyNoMoreInteractions(accounts);
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq(uuid));
verifyNoMoreInteractions(accountsDynamoDb); verifyNoMoreInteractions(accountsDynamoDb);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testGetAccountByNumberBrokenCache() {
void testGetAccountByNumberBrokenCache(boolean dynamoEnabled) { UUID uuid = UUID.randomUUID();
UUID uuid = UUID.randomUUID(); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
enableDynamo(dynamoEnabled);
when(commands.get(eq("AccountMap::+14152222222"))).thenThrow(new RedisException("Connection lost!")); when(commands.get(eq("AccountMap::+14152222222"))).thenThrow(new RedisException("Connection lost!"));
when(accounts.get(eq("+14152222222"))).thenReturn(Optional.of(account)); when(accountsDynamoDb.get(eq("+14152222222"))).thenReturn(Optional.of(account));
Optional<Account> retrieved = accountsManager.get("+14152222222"); Optional<Account> retrieved = accountsManager.get("+14152222222");
assertTrue(retrieved.isPresent()); assertTrue(retrieved.isPresent());
assertSame(retrieved.get(), account); assertSame(retrieved.get(), account);
@ -270,25 +225,20 @@ class AccountsManagerTest {
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString()); verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
verifyNoMoreInteractions(commands); verifyNoMoreInteractions(commands);
verify(accounts, times(1)).get(eq("+14152222222")); verify(accountsDynamoDb, times(1)).get(eq("+14152222222"));
verifyNoMoreInteractions(accounts);
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq("+14152222222"));
verifyNoMoreInteractions(accountsDynamoDb); verifyNoMoreInteractions(accountsDynamoDb);
} }
@ParameterizedTest @Test
@ValueSource(booleans = {true, false}) void testGetAccountByUuidBrokenCache() {
void testGetAccountByUuidBrokenCache(boolean dynamoEnabled) { final boolean dynamoEnabled = true;
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
enableDynamo(dynamoEnabled);
when(commands.get(eq("Account3::" + uuid))).thenThrow(new RedisException("Connection lost!")); when(commands.get(eq("Account3::" + uuid))).thenThrow(new RedisException("Connection lost!"));
when(accounts.get(eq(uuid))).thenReturn(Optional.of(account)); when(accountsDynamoDb.get(eq(uuid))).thenReturn(Optional.of(account));
Optional<Account> retrieved = accountsManager.get(uuid); Optional<Account> retrieved = accountsManager.get(uuid);
assertTrue(retrieved.isPresent()); assertTrue(retrieved.isPresent());
assertSame(retrieved.get(), account); assertSame(retrieved.get(), account);
@ -298,26 +248,25 @@ class AccountsManagerTest {
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString()); verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
verifyNoMoreInteractions(commands); verifyNoMoreInteractions(commands);
verify(accounts, times(1)).get(eq(uuid)); verify(accountsDynamoDb, times(1)).get(eq(uuid));
verifyNoMoreInteractions(accounts);
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq(uuid));
verifyNoMoreInteractions(accountsDynamoDb); verifyNoMoreInteractions(accountsDynamoDb);
} }
@ParameterizedTest // TODO delete
@ValueSource(booleans = {true, false}) @Disabled("migration specific")
void testUpdate_dynamoDbMigration(boolean dynamoEnabled) throws IOException { @Test
void testUpdate_dynamoDbMigration() throws IOException {
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
enableDynamo(dynamoEnabled);
when(commands.get(eq("Account3::" + uuid))).thenReturn(null); when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
// database fetches should always return new instances // database fetches should always return new instances
when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); when(accountsDynamoDb.get(uuid)).thenReturn(
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accounts).update(any(Account.class)); when(accountsDynamoDb.get(uuid)).thenReturn(
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accountsDynamoDb).update(any(Account.class));
Account updatedAccount = accountsManager.update(account, a -> a.setProfileName("name")); Account updatedAccount = accountsManager.update(account, a -> a.setProfileName("name"));
@ -325,17 +274,13 @@ class AccountsManagerTest {
assertNotSame(updatedAccount, account); assertNotSame(updatedAccount, account);
verify(accounts, times(1)).update(account); verify(accountsDynamoDb, times(1)).update(account);
verifyNoMoreInteractions(accounts); verifyNoMoreInteractions(accountsDynamoDb);
if (dynamoEnabled) { ArgumentCaptor<Account> argumentCaptor = ArgumentCaptor.forClass(Account.class);
ArgumentCaptor<Account> argumentCaptor = ArgumentCaptor.forClass(Account.class); verify(accountsDynamoDb, times(1)).update(argumentCaptor.capture());
verify(accountsDynamoDb, times(1)).update(argumentCaptor.capture()); assertEquals(uuid, argumentCaptor.getValue().getUuid());
assertEquals(uuid, argumentCaptor.getValue().getUuid()); verify(accountsDynamoDb, times(1)).get(uuid);
} else {
verify(accountsDynamoDb, never()).update(any());
}
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(uuid);
verifyNoMoreInteractions(accountsDynamoDb); verifyNoMoreInteractions(accountsDynamoDb);
ArgumentCaptor<String> redisSetArgumentCapture = ArgumentCaptor.forClass(String.class); ArgumentCaptor<String> redisSetArgumentCapture = ArgumentCaptor.forClass(String.class);
@ -347,25 +292,26 @@ class AccountsManagerTest {
// uuid is @JsonIgnore, so we need to set it for compareAccounts to work // uuid is @JsonIgnore, so we need to set it for compareAccounts to work
accountCached.setUuid(uuid); accountCached.setUuid(uuid);
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(updatedAccount), Optional.of(accountCached))); assertEquals(Optional.empty(),
accountsManager.compareAccounts(Optional.of(updatedAccount), Optional.of(accountCached)));
} }
// TODO delete
@Disabled("migration specific")
@Test @Test
void testUpdate_dynamoMissing() { void testUpdate_dynamoMissing() {
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
enableDynamo(true);
when(commands.get(eq("Account3::" + uuid))).thenReturn(null); when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty()); when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty());
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accounts).update(any());
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accountsDynamoDb).update(any()); doAnswer(ACCOUNT_UPDATE_ANSWER).when(accountsDynamoDb).update(any());
Account updatedAccount = accountsManager.update(account, a -> {}); Account updatedAccount = accountsManager.update(account, a -> {
});
verify(accounts, times(1)).update(account); verify(accountsDynamoDb, times(1)).update(account);
verifyNoMoreInteractions(accounts); verifyNoMoreInteractions(accountsDynamoDb);
verify(accountsDynamoDb, never()).update(account); verify(accountsDynamoDb, never()).update(account);
verify(accountsDynamoDb, times(1)).get(uuid); verify(accountsDynamoDb, times(1)).get(uuid);
@ -376,19 +322,19 @@ class AccountsManagerTest {
@Test @Test
void testUpdate_optimisticLockingFailure() { void testUpdate_optimisticLockingFailure() {
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
enableDynamo(true);
when(commands.get(eq("Account3::" + uuid))).thenReturn(null); when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); when(accountsDynamoDb.get(uuid)).thenReturn(
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
doThrow(ContestedOptimisticLockException.class) doThrow(ContestedOptimisticLockException.class)
.doAnswer(ACCOUNT_UPDATE_ANSWER) .doAnswer(ACCOUNT_UPDATE_ANSWER)
.when(accounts).update(any()); .when(accountsDynamoDb).update(any());
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); when(accountsDynamoDb.get(uuid)).thenReturn(
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
doThrow(ContestedOptimisticLockException.class) doThrow(ContestedOptimisticLockException.class)
.doAnswer(ACCOUNT_UPDATE_ANSWER) .doAnswer(ACCOUNT_UPDATE_ANSWER)
.when(accountsDynamoDb).update(any()); .when(accountsDynamoDb).update(any());
@ -398,12 +344,7 @@ class AccountsManagerTest {
assertEquals(1, account.getVersion()); assertEquals(1, account.getVersion());
assertEquals("name", account.getProfileName()); assertEquals("name", account.getProfileName());
verify(accounts, times(1)).get(uuid); verify(accountsDynamoDb, times(1)).get(uuid);
verify(accounts, times(2)).update(any());
verifyNoMoreInteractions(accounts);
// dynamo has an extra get() because the account is fetched before every update
verify(accountsDynamoDb, times(2)).get(uuid);
verify(accountsDynamoDb, times(2)).update(any()); verify(accountsDynamoDb, times(2)).update(any());
verifyNoMoreInteractions(accountsDynamoDb); verifyNoMoreInteractions(accountsDynamoDb);
} }
@ -413,8 +354,6 @@ class AccountsManagerTest {
UUID uuid = UUID.randomUUID(); UUID uuid = UUID.randomUUID();
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
enableDynamo(true);
when(commands.get(eq("Account3::" + uuid))).thenReturn(null); when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty()) when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty())
.thenReturn(Optional.of(account)); .thenReturn(Optional.of(account));
@ -422,10 +361,7 @@ class AccountsManagerTest {
accountsManager.update(account, a -> {}); accountsManager.update(account, a -> {});
verify(accounts, times(1)).update(account); verify(accountsDynamoDb, times(1)).update(account);
verifyNoMoreInteractions(accounts);
verify(accountsDynamoDb, times(1)).get(uuid);
verifyNoMoreInteractions(accountsDynamoDb); verifyNoMoreInteractions(accountsDynamoDb);
} }
@ -436,7 +372,8 @@ class AccountsManagerTest {
final UUID uuid = UUID.randomUUID(); final UUID uuid = UUID.randomUUID();
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]); Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16]))); when(accountsDynamoDb.get(uuid)).thenReturn(
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
assertTrue(account.getDevices().isEmpty()); assertTrue(account.getDevices().isEmpty());
@ -463,6 +400,8 @@ class AccountsManagerTest {
verify(unknownDeviceUpdater, never()).accept(any(Device.class)); verify(unknownDeviceUpdater, never()).accept(any(Device.class));
} }
// TODO delete
@Disabled("migration specific")
@Test @Test
void testCompareAccounts() throws Exception { void testCompareAccounts() throws Exception {
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.empty(), Optional.empty())); assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.empty(), Optional.empty()));
@ -538,13 +477,13 @@ class AccountsManagerTest {
@Test @Test
void testCreateFreshAccount() throws InterruptedException { void testCreateFreshAccount() throws InterruptedException {
when(accounts.create(any())).thenReturn(true); when(accountsDynamoDb.create(any())).thenReturn(true);
final String e164 = "+18005550123"; final String e164 = "+18005550123";
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null); final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
accountsManager.create(e164, "password", null, attributes); accountsManager.create(e164, "password", null, attributes);
verify(accounts).create(argThat(account -> e164.equals(account.getNumber()))); verify(accountsDynamoDb).create(argThat(account -> e164.equals(account.getNumber())));
verifyNoInteractions(keys); verifyNoInteractions(keys);
verifyNoInteractions(messagesManager); verifyNoInteractions(messagesManager);
verifyNoInteractions(profilesManager); verifyNoInteractions(profilesManager);
@ -554,7 +493,7 @@ class AccountsManagerTest {
void testReregisterAccount() throws InterruptedException { void testReregisterAccount() throws InterruptedException {
final UUID existingUuid = UUID.randomUUID(); final UUID existingUuid = UUID.randomUUID();
when(accounts.create(any())).thenAnswer(invocation -> { when(accountsDynamoDb.create(any())).thenAnswer(invocation -> {
invocation.getArgument(0, Account.class).setUuid(existingUuid); invocation.getArgument(0, Account.class).setUuid(existingUuid);
return false; return false;
}); });
@ -563,7 +502,8 @@ class AccountsManagerTest {
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null); final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
accountsManager.create(e164, "password", null, attributes); accountsManager.create(e164, "password", null, attributes);
verify(accounts).create(argThat(account -> e164.equals(account.getNumber()) && existingUuid.equals(account.getUuid()))); verify(accountsDynamoDb).create(
argThat(account -> e164.equals(account.getNumber()) && existingUuid.equals(account.getUuid())));
verify(keys).delete(existingUuid); verify(keys).delete(existingUuid);
verify(messagesManager).clear(existingUuid); verify(messagesManager).clear(existingUuid);
verify(profilesManager).deleteAll(existingUuid); verify(profilesManager).deleteAll(existingUuid);
@ -579,13 +519,14 @@ class AccountsManagerTest {
return null; return null;
}).when(deletedAccountsManager).lockAndTake(anyString(), any()); }).when(deletedAccountsManager).lockAndTake(anyString(), any());
when(accounts.create(any())).thenReturn(true); when(accountsDynamoDb.create(any())).thenReturn(true);
final String e164 = "+18005550123"; final String e164 = "+18005550123";
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null); final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
accountsManager.create(e164, "password", null, attributes); accountsManager.create(e164, "password", null, attributes);
verify(accounts).create(argThat(account -> e164.equals(account.getNumber()) && recentlyDeletedUuid.equals(account.getUuid()))); verify(accountsDynamoDb).create(
argThat(account -> e164.equals(account.getNumber()) && recentlyDeletedUuid.equals(account.getUuid())));
verifyNoInteractions(keys); verifyNoInteractions(keys);
verifyNoInteractions(messagesManager); verifyNoInteractions(messagesManager);
verifyNoInteractions(profilesManager); verifyNoInteractions(profilesManager);
@ -634,6 +575,7 @@ class AccountsManagerTest {
verify(directoryQueue, times(expectRefresh ? 1 : 0)).refreshAccount(updatedAccount); verify(directoryQueue, times(expectRefresh ? 1 : 0)).refreshAccount(updatedAccount);
} }
@SuppressWarnings("unused")
private static Stream<Arguments> testUpdateDirectoryQueue() { private static Stream<Arguments> testUpdateDirectoryQueue() {
return Stream.of( return Stream.of(
Arguments.of(false, false, false), Arguments.of(false, false, false),
@ -654,7 +596,7 @@ class AccountsManagerTest {
accountsManager.updateDeviceLastSeen(account, device, updatedLastSeen); accountsManager.updateDeviceLastSeen(account, device, updatedLastSeen);
assertEquals(expectUpdate ? updatedLastSeen : initialLastSeen, device.getLastSeen()); assertEquals(expectUpdate ? updatedLastSeen : initialLastSeen, device.getLastSeen());
verify(accounts, expectUpdate ? times(1) : never()).update(account); verify(accountsDynamoDb, expectUpdate ? times(1) : never()).update(account);
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")

View File

@ -1,386 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.tests.storage;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import com.fasterxml.uuid.UUIDComparator;
import com.opentable.db.postgres.embedded.LiquibasePreparer;
import com.opentable.db.postgres.junit.EmbeddedPostgresRules;
import com.opentable.db.postgres.junit.PreparedDbRule;
import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import org.jdbi.v3.core.HandleConsumer;
import org.jdbi.v3.core.Jdbi;
import org.jdbi.v3.core.transaction.TransactionException;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper;
public class AccountsTest {
@Rule
public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml"));
private Accounts accounts;
@Before
public void setupAccountsDao() {
FaultTolerantDatabase faultTolerantDatabase = new FaultTolerantDatabase("accountsTest",
Jdbi.create(db.getTestDatabase()),
new CircuitBreakerConfiguration());
this.accounts = new Accounts(faultTolerantDatabase);
}
@Test
public void testStore() throws SQLException, IOException {
Device device = generateDevice (1 );
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
boolean freshUser = accounts.create(account);
assertThat(freshUser).isTrue();
PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?");
verifyStoredState(statement, "+14151112222", account.getUuid(), account);
freshUser = accounts.create(account);
assertThat(freshUser).isTrue();
statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?");
verifyStoredState(statement, "+14151112222", account.getUuid(), account);
}
@Test
public void testStoreMulti() throws SQLException, IOException {
Set<Device> devices = new HashSet<>();
devices.add(generateDevice(1));
devices.add(generateDevice(2));
Account account = generateAccount("+14151112222", UUID.randomUUID(), devices);
accounts.create(account);
PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?");
verifyStoredState(statement, "+14151112222", account.getUuid(), account);
}
@Test
public void testRetrieve() {
Set<Device> devicesFirst = new HashSet<>();
devicesFirst.add(generateDevice(1));
devicesFirst.add(generateDevice(2));
UUID uuidFirst = UUID.randomUUID();
Account accountFirst = generateAccount("+14151112222", uuidFirst, devicesFirst);
Set<Device> devicesSecond = new HashSet<>();
devicesSecond.add(generateDevice(1));
devicesSecond.add(generateDevice(2));
UUID uuidSecond = UUID.randomUUID();
Account accountSecond = generateAccount("+14152221111", uuidSecond, devicesSecond);
accounts.create(accountFirst);
accounts.create(accountSecond);
Optional<Account> retrievedFirst = accounts.get("+14151112222");
Optional<Account> retrievedSecond = accounts.get("+14152221111");
assertThat(retrievedFirst.isPresent()).isTrue();
assertThat(retrievedSecond.isPresent()).isTrue();
verifyStoredState("+14151112222", uuidFirst, retrievedFirst.get(), accountFirst);
verifyStoredState("+14152221111", uuidSecond, retrievedSecond.get(), accountSecond);
retrievedFirst = accounts.get(uuidFirst);
retrievedSecond = accounts.get(uuidSecond);
assertThat(retrievedFirst.isPresent()).isTrue();
assertThat(retrievedSecond.isPresent()).isTrue();
verifyStoredState("+14151112222", uuidFirst, retrievedFirst.get(), accountFirst);
verifyStoredState("+14152221111", uuidSecond, retrievedSecond.get(), accountSecond);
}
@Test
public void testOverwrite() throws Exception {
Device device = generateDevice (1 );
UUID firstUuid = UUID.randomUUID();
Account account = generateAccount("+14151112222", firstUuid, Collections.singleton(device));
accounts.create(account);
PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?");
verifyStoredState(statement, "+14151112222", account.getUuid(), account);
UUID secondUuid = UUID.randomUUID();
device = generateDevice(1);
account = generateAccount("+14151112222", secondUuid, Collections.singleton(device));
final boolean freshUser = accounts.create(account);
assertThat(freshUser).isFalse();
verifyStoredState(statement, "+14151112222", firstUuid, account);
device = generateDevice(1);
Account invalidAccount = generateAccount("+14151113333", firstUuid, Collections.singleton(device));
assertThatThrownBy(() -> accounts.create(invalidAccount));
}
@Test
public void testUpdate() {
Device device = generateDevice (1 );
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
accounts.create(account);
device.setName("foobar");
accounts.update(account);
account.setProfileName("profileName");
accounts.update(account);
assertThat(account.getVersion()).isEqualTo(2);
Optional<Account> retrieved = accounts.get("+14151112222");
assertThat(retrieved.isPresent()).isTrue();
verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account);
retrieved = accounts.get(account.getUuid());
assertThat(retrieved.isPresent()).isTrue();
verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account);
}
@Test
public void testRetrieveFrom() {
List<Account> users = new ArrayList<>();
for (int i=1;i<=100;i++) {
Account account = generateAccount("+1" + String.format("%03d", i), UUID.randomUUID());
users.add(account);
accounts.create(account);
}
users.sort((account, t1) -> UUIDComparator.staticCompare(account.getUuid(), t1.getUuid()));
AccountCrawlChunk retrieved = accounts.getAllFrom(10);
assertThat(retrieved.getAccounts().size()).isEqualTo(10);
for (int i=0;i<retrieved.getAccounts().size();i++) {
verifyStoredState(users.get(i).getNumber(), users.get(i).getUuid(), retrieved.getAccounts().get(i), users.get(i));
}
for (int j=0;j<9;j++) {
retrieved = accounts.getAllFrom(retrieved.getLastUuid().orElseThrow(), 10);
assertThat(retrieved.getAccounts().size()).isEqualTo(10);
for (int i=0;i<retrieved.getAccounts().size();i++) {
verifyStoredState(users.get(10 + (j * 10) + i).getNumber(), users.get(10 + (j * 10) + i).getUuid(), retrieved.getAccounts().get(i), users.get(10 + (j * 10) + i));
}
}
}
@Test
public void testDelete() {
final Device deletedDevice = generateDevice (1);
final Account deletedAccount = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(deletedDevice));
final Device retainedDevice = generateDevice (1);
final Account retainedAccount = generateAccount("+14151112345", UUID.randomUUID(), Collections.singleton(retainedDevice));
accounts.create(deletedAccount);
accounts.create(retainedAccount);
assertThat(accounts.get(deletedAccount.getUuid())).isPresent();
assertThat(accounts.get(retainedAccount.getUuid())).isPresent();
accounts.delete(deletedAccount.getUuid());
assertThat(accounts.get(deletedAccount.getUuid())).isNotPresent();
verifyStoredState(retainedAccount.getNumber(), retainedAccount.getUuid(), accounts.get(retainedAccount.getUuid()).get(), retainedAccount);
{
final Account recreatedAccount = generateAccount(deletedAccount.getNumber(), UUID.randomUUID(),
Collections.singleton(generateDevice(1)));
final boolean freshUser = accounts.create(recreatedAccount);
assertThat(freshUser).isTrue();
assertThat(accounts.get(recreatedAccount.getUuid())).isPresent();
verifyStoredState(recreatedAccount.getNumber(), recreatedAccount.getUuid(),
accounts.get(recreatedAccount.getUuid()).get(), recreatedAccount);
}
}
@Test
public void testVacuum() {
Device device = generateDevice (1 );
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
accounts.create(account);
accounts.vacuum();
Optional<Account> retrieved = accounts.get("+14151112222");
assertThat(retrieved.isPresent()).isTrue();
verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account);
}
@Test
public void testMissing() {
Device device = generateDevice (1 );
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
accounts.create(account);
Optional<Account> retrieved = accounts.get("+11111111");
assertThat(retrieved.isPresent()).isFalse();
retrieved = accounts.get(UUID.randomUUID());
assertThat(retrieved.isPresent()).isFalse();
}
@Test
public void testBreaker() throws InterruptedException {
Jdbi jdbi = mock(Jdbi.class);
doThrow(new TransactionException("Database error!")).when(jdbi).useHandle(any(HandleConsumer.class));
CircuitBreakerConfiguration configuration = new CircuitBreakerConfiguration();
configuration.setWaitDurationInOpenStateInSeconds(1);
configuration.setRingBufferSizeInHalfOpenState(1);
configuration.setRingBufferSizeInClosedState(2);
configuration.setFailureRateThreshold(50);
Accounts accounts = new Accounts(new FaultTolerantDatabase("testAccountBreaker", jdbi, configuration));
Account account = generateAccount("+14151112222", UUID.randomUUID());
try {
accounts.update(account);
throw new AssertionError();
} catch (TransactionException e) {
// good
}
try {
accounts.update(account);
throw new AssertionError();
} catch (TransactionException e) {
// good
}
try {
accounts.update(account);
throw new AssertionError();
} catch (CallNotPermittedException e) {
// good
}
Thread.sleep(1100);
try {
accounts.update(account);
throw new AssertionError();
} catch (TransactionException e) {
// good
}
}
private Device generateDevice(long id) {
Random random = new Random(System.currentTimeMillis());
SignedPreKey signedPreKey = new SignedPreKey(random.nextInt(), "testPublicKey-" + random.nextInt(), "testSignature-" + random.nextInt());
return new Device(id, "testName-" + random.nextInt(), "testAuthToken-" + random.nextInt(), "testSalt-" + random.nextInt(),
"testGcmId-" + random.nextInt(), "testApnId-" + random.nextInt(), "testVoipApnId-" + random.nextInt(), random.nextBoolean(), random.nextInt(), signedPreKey, random.nextInt(), random.nextInt(), "testUserAgent-" + random.nextInt() , 0, new Device.DeviceCapabilities(random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
random.nextBoolean(), random.nextBoolean(), random.nextBoolean()));
}
private Account generateAccount(String number, UUID uuid) {
Device device = generateDevice(1);
return generateAccount(number, uuid, Collections.singleton(device));
}
private Account generateAccount(String number, UUID uuid, Set<Device> devices) {
byte[] unidentifiedAccessKey = new byte[16];
Random random = new Random(System.currentTimeMillis());
Arrays.fill(unidentifiedAccessKey, (byte)random.nextInt(255));
return new Account(number, uuid, devices, unidentifiedAccessKey);
}
private void verifyStoredState(PreparedStatement statement, String number, UUID uuid, Account expecting)
throws SQLException, IOException
{
statement.setString(1, number);
ResultSet resultSet = statement.executeQuery();
if (resultSet.next()) {
String data = resultSet.getString("data");
assertThat(data).isNotEmpty();
Account result = new AccountRowMapper().map(resultSet, null);
verifyStoredState(number, uuid, result, expecting);
} else {
throw new AssertionError("No data");
}
assertThat(resultSet.next()).isFalse();
}
private void verifyStoredState(String number, UUID uuid, Account result, Account expecting) {
assertThat(result.getNumber()).isEqualTo(number);
assertThat(result.getLastSeen()).isEqualTo(expecting.getLastSeen());
assertThat(result.getUuid()).isEqualTo(uuid);
assertThat(result.getVersion()).isEqualTo(expecting.getVersion());
assertThat(Arrays.equals(result.getUnidentifiedAccessKey().get(), expecting.getUnidentifiedAccessKey().get())).isTrue();
for (Device expectingDevice : expecting.getDevices()) {
Device resultDevice = result.getDevice(expectingDevice.getId()).get();
assertThat(resultDevice.getApnId()).isEqualTo(expectingDevice.getApnId());
assertThat(resultDevice.getGcmId()).isEqualTo(expectingDevice.getGcmId());
assertThat(resultDevice.getLastSeen()).isEqualTo(expectingDevice.getLastSeen());
assertThat(resultDevice.getSignedPreKey().getPublicKey()).isEqualTo(expectingDevice.getSignedPreKey().getPublicKey());
assertThat(resultDevice.getSignedPreKey().getKeyId()).isEqualTo(expectingDevice.getSignedPreKey().getKeyId());
assertThat(resultDevice.getSignedPreKey().getSignature()).isEqualTo(expectingDevice.getSignedPreKey().getSignature());
assertThat(resultDevice.getFetchesMessages()).isEqualTo(expectingDevice.getFetchesMessages());
assertThat(resultDevice.getUserAgent()).isEqualTo(expectingDevice.getUserAgent());
assertThat(resultDevice.getName()).isEqualTo(expectingDevice.getName());
assertThat(resultDevice.getCreated()).isEqualTo(expectingDevice.getCreated());
}
}
}