Remove Accounts Postgres
This commit is contained in:
parent
8161f55a82
commit
2a67b2e610
|
@ -46,9 +46,7 @@ import java.util.Optional;
|
||||||
import java.util.concurrent.ArrayBlockingQueue;
|
import java.util.concurrent.ArrayBlockingQueue;
|
||||||
import java.util.concurrent.BlockingQueue;
|
import java.util.concurrent.BlockingQueue;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.LinkedBlockingDeque;
|
|
||||||
import java.util.concurrent.ScheduledExecutorService;
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
import java.util.concurrent.ThreadPoolExecutor;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import javax.servlet.DispatcherType;
|
import javax.servlet.DispatcherType;
|
||||||
import javax.servlet.FilterRegistration;
|
import javax.servlet.FilterRegistration;
|
||||||
|
@ -62,13 +60,13 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.whispersystems.dispatch.DispatchManager;
|
import org.whispersystems.dispatch.DispatchManager;
|
||||||
import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
|
import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
|
||||||
import org.whispersystems.textsecuregcm.auth.WebsocketRefreshApplicationEventListener;
|
|
||||||
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
|
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
|
||||||
import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
|
import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
|
||||||
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccountAuthenticator;
|
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccountAuthenticator;
|
||||||
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount;
|
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount;
|
||||||
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
|
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
|
||||||
import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
|
import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
|
||||||
|
import org.whispersystems.textsecuregcm.auth.WebsocketRefreshApplicationEventListener;
|
||||||
import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter;
|
import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter;
|
||||||
import org.whispersystems.textsecuregcm.badges.ProfileBadgeConverter;
|
import org.whispersystems.textsecuregcm.badges.ProfileBadgeConverter;
|
||||||
import org.whispersystems.textsecuregcm.configuration.DirectoryServerConfiguration;
|
import org.whispersystems.textsecuregcm.configuration.DirectoryServerConfiguration;
|
||||||
|
@ -156,9 +154,7 @@ import org.whispersystems.textsecuregcm.storage.AccountCleaner;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
|
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
|
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
|
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
|
||||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDbMigrator;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.ContactDiscoveryWriter;
|
import org.whispersystems.textsecuregcm.storage.ContactDiscoveryWriter;
|
||||||
import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
|
import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
|
||||||
|
@ -174,11 +170,6 @@ import org.whispersystems.textsecuregcm.storage.MessagePersister;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesCache;
|
import org.whispersystems.textsecuregcm.storage.MessagesCache;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccountsTableCrawler;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationRetryAccountsTableCrawler;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Profiles;
|
import org.whispersystems.textsecuregcm.storage.Profiles;
|
||||||
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.PubSubManager;
|
import org.whispersystems.textsecuregcm.storage.PubSubManager;
|
||||||
|
@ -211,14 +202,12 @@ import org.whispersystems.textsecuregcm.workers.ServerVersionCommand;
|
||||||
import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask;
|
import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask;
|
||||||
import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask;
|
import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask;
|
||||||
import org.whispersystems.textsecuregcm.workers.SetUserDiscoverabilityCommand;
|
import org.whispersystems.textsecuregcm.workers.SetUserDiscoverabilityCommand;
|
||||||
import org.whispersystems.textsecuregcm.workers.VacuumCommand;
|
|
||||||
import org.whispersystems.textsecuregcm.workers.ZkParamsCommand;
|
import org.whispersystems.textsecuregcm.workers.ZkParamsCommand;
|
||||||
import org.whispersystems.websocket.WebSocketResourceProviderFactory;
|
import org.whispersystems.websocket.WebSocketResourceProviderFactory;
|
||||||
import org.whispersystems.websocket.setup.WebSocketEnvironment;
|
import org.whispersystems.websocket.setup.WebSocketEnvironment;
|
||||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||||
import software.amazon.awssdk.regions.Region;
|
import software.amazon.awssdk.regions.Region;
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||||
import software.amazon.awssdk.services.s3.S3Client;
|
import software.amazon.awssdk.services.s3.S3Client;
|
||||||
|
|
||||||
|
@ -228,7 +217,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void initialize(Bootstrap<WhisperServerConfiguration> bootstrap) {
|
public void initialize(Bootstrap<WhisperServerConfiguration> bootstrap) {
|
||||||
bootstrap.addCommand(new VacuumCommand());
|
|
||||||
bootstrap.addCommand(new DeleteUserCommand());
|
bootstrap.addCommand(new DeleteUserCommand());
|
||||||
bootstrap.addCommand(new CertificateCommand());
|
bootstrap.addCommand(new CertificateCommand());
|
||||||
bootstrap.addCommand(new ZkParamsCommand());
|
bootstrap.addCommand(new ZkParamsCommand());
|
||||||
|
@ -243,7 +231,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("abusedb", "abusedb.xml") {
|
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("abusedb", "abusedb.xml") {
|
||||||
@Override
|
@Override
|
||||||
public PooledDataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) {
|
public PooledDataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) {
|
||||||
|
@ -316,20 +303,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(config.getAccountsDynamoDbConfiguration(),
|
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(config.getAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
|
||||||
// The thread pool core & max sizes are set via dynamic configuration within AccountsDynamoDb
|
|
||||||
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
|
|
||||||
new LinkedBlockingDeque<>());
|
|
||||||
|
|
||||||
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(config.getAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
|
|
||||||
accountsDynamoDbMigrationThreadPool);
|
|
||||||
|
|
||||||
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(config.getDeletedAccountsDynamoDbConfiguration(),
|
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(config.getDeletedAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
|
||||||
DynamoDbClient recentlyDeletedAccountsDynamoDb = DynamoDbFromConfig.client(config.getMigrationDeletedAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
|
|
||||||
DynamoDbClient pushChallengeDynamoDbClient = DynamoDbFromConfig.client(
|
DynamoDbClient pushChallengeDynamoDbClient = DynamoDbFromConfig.client(
|
||||||
config.getPushChallengeDynamoDbConfiguration(),
|
config.getPushChallengeDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
@ -338,14 +314,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
config.getReportMessageDynamoDbConfiguration(),
|
config.getReportMessageDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
|
||||||
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(
|
|
||||||
config.getMigrationRetryAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
|
|
||||||
DynamoDbClient migrationMismatchedAccountsDynamoDb = DynamoDbFromConfig.client(
|
|
||||||
config.getMigrationMismatchedAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
|
|
||||||
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig.client(
|
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig.client(
|
||||||
config.getPendingAccountsDynamoDbConfiguration(),
|
config.getPendingAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
@ -366,19 +334,11 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
|
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
|
||||||
config.getDeletedAccountsDynamoDbConfiguration().getTableName(),
|
config.getDeletedAccountsDynamoDbConfiguration().getTableName(),
|
||||||
config.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
|
config.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
|
||||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(recentlyDeletedAccountsDynamoDb,
|
|
||||||
config.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
|
|
||||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb,
|
|
||||||
config.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
|
|
||||||
MigrationMismatchedAccounts mismatchedAccounts = new MigrationMismatchedAccounts(
|
|
||||||
migrationMismatchedAccountsDynamoDb,
|
|
||||||
config.getMigrationMismatchedAccountsDynamoDbConfiguration().getTableName());
|
|
||||||
|
|
||||||
Accounts accounts = new Accounts(accountDatabase);
|
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient,
|
||||||
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient,
|
config.getAccountsDynamoDbConfiguration().getTableName(),
|
||||||
accountsDynamoDbMigrationThreadPool, config.getAccountsDynamoDbConfiguration().getTableName(),
|
config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName()
|
||||||
config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts,
|
);
|
||||||
migrationRetryAccounts);
|
|
||||||
Usernames usernames = new Usernames(accountDatabase);
|
Usernames usernames = new Usernames(accountDatabase);
|
||||||
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
|
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
|
||||||
Profiles profiles = new Profiles(accountDatabase);
|
Profiles profiles = new Profiles(accountDatabase);
|
||||||
|
@ -431,8 +391,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
ExecutorService donationExecutor = environment.lifecycle().executorService(name(getClass(), "donation-%d")).maxThreads(1).minThreads(1).build();
|
ExecutorService donationExecutor = environment.lifecycle().executorService(name(getClass(), "donation-%d")).maxThreads(1).minThreads(1).build();
|
||||||
ExecutorService multiRecipientMessageExecutor = environment.lifecycle()
|
ExecutorService multiRecipientMessageExecutor = environment.lifecycle()
|
||||||
.executorService(name(getClass(), "multiRecipientMessage-%d")).minThreads(64).maxThreads(64).build();
|
.executorService(name(getClass(), "multiRecipientMessage-%d")).minThreads(64).maxThreads(64).build();
|
||||||
ExecutorService accountsCrawlerChunkPreReadExecutor = environment.lifecycle()
|
|
||||||
.executorService(name(getClass(), "accountsCrawler-%d")).maxThreads(2).minThreads(2).build();
|
|
||||||
|
|
||||||
ExternalServiceCredentialGenerator directoryCredentialsGenerator = new ExternalServiceCredentialGenerator(config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenSharedSecret(),
|
ExternalServiceCredentialGenerator directoryCredentialsGenerator = new ExternalServiceCredentialGenerator(config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenSharedSecret(),
|
||||||
config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenUserIdSecret(),
|
config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenUserIdSecret(),
|
||||||
|
@ -464,9 +422,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager, reportMessageManager);
|
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager, reportMessageManager);
|
||||||
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
|
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
|
||||||
deletedAccountsLockDynamoDbClient, config.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
|
deletedAccountsLockDynamoDbClient, config.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
|
||||||
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster,
|
AccountsManager accountsManager = new AccountsManager(accountsDynamoDb, cacheCluster,
|
||||||
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, mismatchedAccounts, usernamesManager,
|
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager,
|
||||||
profilesManager, pendingAccountsManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager,
|
profilesManager, pendingAccountsManager, secureStorageClient, secureBackupClient,
|
||||||
dynamicConfigurationManager);
|
dynamicConfigurationManager);
|
||||||
RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs);
|
RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs);
|
||||||
DeadLetterHandler deadLetterHandler = new DeadLetterHandler(accountsManager, messagesManager);
|
DeadLetterHandler deadLetterHandler = new DeadLetterHandler(accountsManager, messagesManager);
|
||||||
|
@ -539,27 +497,10 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
AccountDatabaseCrawler accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager,
|
AccountDatabaseCrawler accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager,
|
||||||
accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
|
accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
|
||||||
config.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
|
config.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
|
||||||
config.getAccountDatabaseCrawlerConfiguration().getChunkIntervalMs(),
|
config.getAccountDatabaseCrawlerConfiguration().getChunkIntervalMs()
|
||||||
accountsCrawlerChunkPreReadExecutor,
|
);
|
||||||
dynamicConfigurationManager);
|
|
||||||
|
|
||||||
AccountDatabaseCrawlerCache dynamoDbMigrationCrawlerCache = new AccountDatabaseCrawlerCache(cacheCluster);
|
|
||||||
dynamoDbMigrationCrawlerCache.setPrefix("DynamoMigration");
|
|
||||||
AccountDatabaseCrawler accountDynamoDbMigrationCrawler = new AccountDatabaseCrawler(accountsManager,
|
|
||||||
dynamoDbMigrationCrawlerCache,
|
|
||||||
List.of(new AccountsDynamoDbMigrator(accountsDynamoDb, dynamicConfigurationManager)),
|
|
||||||
config.getDynamoDbMigrationCrawlerConfiguration().getChunkSize(),
|
|
||||||
config.getDynamoDbMigrationCrawlerConfiguration().getChunkIntervalMs(),
|
|
||||||
accountsCrawlerChunkPreReadExecutor,
|
|
||||||
dynamicConfigurationManager);
|
|
||||||
accountDynamoDbMigrationCrawler.setDedicatedDynamoMigrationCrawler(true);
|
|
||||||
|
|
||||||
DeletedAccountsTableCrawler deletedAccountsTableCrawler = new DeletedAccountsTableCrawler(deletedAccountsManager, deletedAccountsDirectoryReconcilers, cacheCluster, recurringJobExecutor);
|
DeletedAccountsTableCrawler deletedAccountsTableCrawler = new DeletedAccountsTableCrawler(deletedAccountsManager, deletedAccountsDirectoryReconcilers, cacheCluster, recurringJobExecutor);
|
||||||
MigrationRetryAccountsTableCrawler migrationRetryAccountsTableCrawler = new MigrationRetryAccountsTableCrawler(
|
|
||||||
migrationRetryAccounts, accountsManager, accountsDynamoDb, cacheCluster, recurringJobExecutor);
|
|
||||||
MigrationMismatchedAccountsTableCrawler migrationMismatchedAccountsTableCrawler = new MigrationMismatchedAccountsTableCrawler(
|
|
||||||
mismatchedAccounts, accountsManager, accounts, accountsDynamoDb, dynamicConfigurationManager, cacheCluster,
|
|
||||||
recurringJobExecutor);
|
|
||||||
|
|
||||||
apnSender.setApnFallbackManager(apnFallbackManager);
|
apnSender.setApnFallbackManager(apnFallbackManager);
|
||||||
environment.lifecycle().manage(new ApplicationShutdownMonitor());
|
environment.lifecycle().manage(new ApplicationShutdownMonitor());
|
||||||
|
@ -567,10 +508,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
||||||
environment.lifecycle().manage(pubSubManager);
|
environment.lifecycle().manage(pubSubManager);
|
||||||
environment.lifecycle().manage(messageSender);
|
environment.lifecycle().manage(messageSender);
|
||||||
environment.lifecycle().manage(accountDatabaseCrawler);
|
environment.lifecycle().manage(accountDatabaseCrawler);
|
||||||
environment.lifecycle().manage(accountDynamoDbMigrationCrawler);
|
|
||||||
environment.lifecycle().manage(deletedAccountsTableCrawler);
|
environment.lifecycle().manage(deletedAccountsTableCrawler);
|
||||||
environment.lifecycle().manage(migrationRetryAccountsTableCrawler);
|
|
||||||
environment.lifecycle().manage(migrationMismatchedAccountsTableCrawler);
|
|
||||||
environment.lifecycle().manage(remoteConfigsManager);
|
environment.lifecycle().manage(remoteConfigsManager);
|
||||||
environment.lifecycle().manage(messagesCache);
|
environment.lifecycle().manage(messagesCache);
|
||||||
environment.lifecycle().manage(messagePersister);
|
environment.lifecycle().manage(messagePersister);
|
||||||
|
|
|
@ -1,112 +1,15 @@
|
||||||
package org.whispersystems.textsecuregcm.configuration.dynamic;
|
package org.whispersystems.textsecuregcm.configuration.dynamic;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
|
||||||
|
|
||||||
public class DynamicAccountsDynamoDbMigrationConfiguration {
|
public class DynamicAccountsDynamoDbMigrationConfiguration {
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean dynamoPrimary;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean backgroundMigrationEnabled;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
int backgroundMigrationExecutorThreads = 1;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean deleteEnabled;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean writeEnabled;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean readEnabled;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean postCheckMismatches;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean logMismatches;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean crawlerPreReadNextChunkEnabled;
|
|
||||||
|
|
||||||
@JsonProperty
|
|
||||||
boolean dynamoCrawlerEnabled;
|
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
int dynamoCrawlerScanPageSize = 10;
|
int dynamoCrawlerScanPageSize = 10;
|
||||||
|
|
||||||
public boolean isDynamoPrimary() {
|
// TODO move out of "migration" configuration
|
||||||
return dynamoPrimary;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isBackgroundMigrationEnabled() {
|
|
||||||
return backgroundMigrationEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getBackgroundMigrationExecutorThreads() {
|
|
||||||
return backgroundMigrationExecutorThreads;
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
public void setBackgroundMigrationEnabled(boolean backgroundMigrationEnabled) {
|
|
||||||
this.backgroundMigrationEnabled = backgroundMigrationEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDeleteEnabled(boolean deleteEnabled) {
|
|
||||||
this.deleteEnabled = deleteEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isDeleteEnabled() {
|
|
||||||
return deleteEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setWriteEnabled(boolean writeEnabled) {
|
|
||||||
this.writeEnabled = writeEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isWriteEnabled() {
|
|
||||||
return writeEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
public void setReadEnabled(boolean readEnabled) {
|
|
||||||
this.readEnabled = readEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isReadEnabled() {
|
|
||||||
return readEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isPostCheckMismatches() {
|
|
||||||
return postCheckMismatches;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isLogMismatches() {
|
|
||||||
return logMismatches;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isCrawlerPreReadNextChunkEnabled() {
|
|
||||||
return crawlerPreReadNextChunkEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isDynamoCrawlerEnabled() {
|
|
||||||
return dynamoCrawlerEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getDynamoCrawlerScanPageSize() {
|
public int getDynamoCrawlerScanPageSize() {
|
||||||
return dynamoCrawlerScanPageSize;
|
return dynamoCrawlerScanPageSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
public void setLogMismatches(boolean logMismatches) {
|
|
||||||
this.logMismatches = logMismatches;
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
public void setBackgroundMigrationExecutorThreads(int threads) {
|
|
||||||
this.backgroundMigrationExecutorThreads = threads;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,7 +14,6 @@ import io.dropwizard.lifecycle.Managed;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -41,32 +40,22 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
|
||||||
private final String workerId;
|
private final String workerId;
|
||||||
private final AccountDatabaseCrawlerCache cache;
|
private final AccountDatabaseCrawlerCache cache;
|
||||||
private final List<AccountDatabaseCrawlerListener> listeners;
|
private final List<AccountDatabaseCrawlerListener> listeners;
|
||||||
private final ExecutorService chunkPreReadExecutorService;
|
|
||||||
|
|
||||||
private final DynamicConfigurationManager dynamicConfigurationManager;
|
|
||||||
|
|
||||||
private AtomicBoolean running = new AtomicBoolean(false);
|
private AtomicBoolean running = new AtomicBoolean(false);
|
||||||
private boolean finished;
|
private boolean finished;
|
||||||
|
|
||||||
// temporary to control behavior during the Postgres → Dynamo transition
|
|
||||||
private boolean dedicatedDynamoMigrationCrawler;
|
|
||||||
|
|
||||||
public AccountDatabaseCrawler(AccountsManager accounts,
|
public AccountDatabaseCrawler(AccountsManager accounts,
|
||||||
AccountDatabaseCrawlerCache cache,
|
AccountDatabaseCrawlerCache cache,
|
||||||
List<AccountDatabaseCrawlerListener> listeners,
|
List<AccountDatabaseCrawlerListener> listeners,
|
||||||
int chunkSize,
|
int chunkSize,
|
||||||
long chunkIntervalMs,
|
long chunkIntervalMs) {
|
||||||
ExecutorService chunkPreReadExecutorService,
|
|
||||||
DynamicConfigurationManager dynamicConfigurationManager) {
|
|
||||||
this.accounts = accounts;
|
this.accounts = accounts;
|
||||||
this.chunkSize = chunkSize;
|
this.chunkSize = chunkSize;
|
||||||
this.chunkIntervalMs = chunkIntervalMs;
|
this.chunkIntervalMs = chunkIntervalMs;
|
||||||
this.workerId = UUID.randomUUID().toString();
|
this.workerId = UUID.randomUUID().toString();
|
||||||
this.cache = cache;
|
this.cache = cache;
|
||||||
this.listeners = listeners;
|
this.listeners = listeners;
|
||||||
this.chunkPreReadExecutorService = chunkPreReadExecutorService;
|
|
||||||
|
|
||||||
this.dynamicConfigurationManager = dynamicConfigurationManager;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -131,25 +120,19 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
|
||||||
|
|
||||||
try (Timer.Context timer = processChunkTimer.time()) {
|
try (Timer.Context timer = processChunkTimer.time()) {
|
||||||
|
|
||||||
final boolean useDynamo = !dedicatedDynamoMigrationCrawler && dynamicConfigurationManager.getConfiguration()
|
final Optional<UUID> fromUuid = getLastUuid();
|
||||||
.getAccountsDynamoDbMigrationConfiguration()
|
|
||||||
.isDynamoCrawlerEnabled();
|
|
||||||
|
|
||||||
final Optional<UUID> fromUuid = getLastUuid(useDynamo);
|
|
||||||
|
|
||||||
if (fromUuid.isEmpty()) {
|
if (fromUuid.isEmpty()) {
|
||||||
logger.info("Started crawl");
|
logger.info("Started crawl");
|
||||||
listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart);
|
listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart);
|
||||||
}
|
}
|
||||||
|
|
||||||
final AccountCrawlChunk chunkAccounts = readChunk(fromUuid, chunkSize, useDynamo);
|
final AccountCrawlChunk chunkAccounts = readChunk(fromUuid, chunkSize);
|
||||||
|
|
||||||
primeDatabaseForNextChunkAsync(chunkAccounts.getLastUuid(), chunkSize, useDynamo);
|
|
||||||
|
|
||||||
if (chunkAccounts.getAccounts().isEmpty()) {
|
if (chunkAccounts.getAccounts().isEmpty()) {
|
||||||
logger.info("Finished crawl");
|
logger.info("Finished crawl");
|
||||||
listeners.forEach(listener -> listener.onCrawlEnd(fromUuid));
|
listeners.forEach(listener -> listener.onCrawlEnd(fromUuid));
|
||||||
cacheLastUuid(Optional.empty(), useDynamo);
|
cacheLastUuid(Optional.empty());
|
||||||
cache.setAccelerated(false);
|
cache.setAccelerated(false);
|
||||||
} else {
|
} else {
|
||||||
logger.info("Processing chunk");
|
logger.info("Processing chunk");
|
||||||
|
@ -157,70 +140,42 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
|
||||||
for (AccountDatabaseCrawlerListener listener : listeners) {
|
for (AccountDatabaseCrawlerListener listener : listeners) {
|
||||||
listener.timeAndProcessCrawlChunk(fromUuid, chunkAccounts.getAccounts());
|
listener.timeAndProcessCrawlChunk(fromUuid, chunkAccounts.getAccounts());
|
||||||
}
|
}
|
||||||
cacheLastUuid(chunkAccounts.getLastUuid(), useDynamo);
|
cacheLastUuid(chunkAccounts.getLastUuid());
|
||||||
} catch (AccountDatabaseCrawlerRestartException e) {
|
} catch (AccountDatabaseCrawlerRestartException e) {
|
||||||
cacheLastUuid(Optional.empty(), useDynamo);
|
cacheLastUuid(Optional.empty());
|
||||||
cache.setAccelerated(false);
|
cache.setAccelerated(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private AccountCrawlChunk readChunk(Optional<UUID> fromUuid, int chunkSize) {
|
||||||
* This is an optimization based on the observation that cold reads of chunks are slow, but subsequent reads of the
|
return readChunk(fromUuid, chunkSize, readChunkTimer);
|
||||||
* same chunk (within a few minutes) are fast. We can’t easily store the actual result data, since the next chunk
|
|
||||||
* might be processed elsewhere, but the time savings are still substantial.
|
|
||||||
*/
|
|
||||||
private void primeDatabaseForNextChunkAsync(Optional<UUID> fromUuid, int chunkSize, boolean useDynamo) {
|
|
||||||
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
|
|
||||||
.isCrawlerPreReadNextChunkEnabled()) {
|
|
||||||
if (!useDynamo && fromUuid.isPresent()) {
|
|
||||||
chunkPreReadExecutorService.submit(() -> readChunk(fromUuid, chunkSize, false, preReadChunkTimer));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private AccountCrawlChunk readChunk(Optional<UUID> fromUuid, int chunkSize, boolean useDynamo) {
|
private AccountCrawlChunk readChunk(Optional<UUID> fromUuid, int chunkSize, Timer readTimer) {
|
||||||
return readChunk(fromUuid, chunkSize, useDynamo, readChunkTimer);
|
|
||||||
}
|
|
||||||
|
|
||||||
private AccountCrawlChunk readChunk(Optional<UUID> fromUuid, int chunkSize, boolean useDynamo, Timer readTimer) {
|
|
||||||
try (Timer.Context timer = readTimer.time()) {
|
try (Timer.Context timer = readTimer.time()) {
|
||||||
|
|
||||||
if (fromUuid.isPresent()) {
|
if (fromUuid.isPresent()) {
|
||||||
return useDynamo
|
return accounts.getAllFromDynamo(fromUuid.get(), chunkSize);
|
||||||
? accounts.getAllFromDynamo(fromUuid.get(), chunkSize)
|
|
||||||
: accounts.getAllFrom(fromUuid.get(), chunkSize);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return useDynamo
|
return accounts.getAllFromDynamo(chunkSize);
|
||||||
? accounts.getAllFromDynamo(chunkSize)
|
|
||||||
: accounts.getAllFrom(chunkSize);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Optional<UUID> getLastUuid(final boolean useDynamo) {
|
private Optional<UUID> getLastUuid() {
|
||||||
if (useDynamo) {
|
return cache.getLastUuidDynamo();
|
||||||
return cache.getLastUuidDynamo();
|
|
||||||
} else {
|
|
||||||
return cache.getLastUuid();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void cacheLastUuid(final Optional<UUID> lastUuid, final boolean useDynamo) {
|
private void cacheLastUuid(final Optional<UUID> lastUuid) {
|
||||||
if (useDynamo) {
|
cache.setLastUuidDynamo(lastUuid);
|
||||||
cache.setLastUuidDynamo(lastUuid);
|
|
||||||
} else {
|
|
||||||
cache.setLastUuid(lastUuid);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDedicatedDynamoMigrationCrawler(final boolean dedicatedDynamoMigrationCrawler) {
|
|
||||||
this.dedicatedDynamoMigrationCrawler = dedicatedDynamoMigrationCrawler;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private synchronized void sleepWhileRunning(long delayMs) {
|
private synchronized void sleepWhileRunning(long delayMs) {
|
||||||
if (running.get()) Util.wait(this, delayMs);
|
if (running.get()) {
|
||||||
|
Util.wait(this, delayMs);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,8 +27,6 @@ public class AccountDatabaseCrawlerCache {
|
||||||
private final FaultTolerantRedisCluster cacheCluster;
|
private final FaultTolerantRedisCluster cacheCluster;
|
||||||
private final ClusterLuaScript unlockClusterScript;
|
private final ClusterLuaScript unlockClusterScript;
|
||||||
|
|
||||||
private String prefix = "";
|
|
||||||
|
|
||||||
public AccountDatabaseCrawlerCache(FaultTolerantRedisCluster cacheCluster) throws IOException {
|
public AccountDatabaseCrawlerCache(FaultTolerantRedisCluster cacheCluster) throws IOException {
|
||||||
this.cacheCluster = cacheCluster;
|
this.cacheCluster = cacheCluster;
|
||||||
this.unlockClusterScript = ClusterLuaScript.fromResource(cacheCluster, "lua/account_database_crawler/unlock.lua",
|
this.unlockClusterScript = ClusterLuaScript.fromResource(cacheCluster, "lua/account_database_crawler/unlock.lua",
|
||||||
|
@ -37,9 +35,9 @@ public class AccountDatabaseCrawlerCache {
|
||||||
|
|
||||||
public void setAccelerated(final boolean accelerated) {
|
public void setAccelerated(final boolean accelerated) {
|
||||||
if (accelerated) {
|
if (accelerated) {
|
||||||
cacheCluster.useCluster(connection -> connection.sync().set(getPrefixedKey(ACCELERATE_KEY), "1"));
|
cacheCluster.useCluster(connection -> connection.sync().set(ACCELERATE_KEY, "1"));
|
||||||
} else {
|
} else {
|
||||||
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(ACCELERATE_KEY)));
|
cacheCluster.useCluster(connection -> connection.sync().del(ACCELERATE_KEY));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -49,16 +47,16 @@ public class AccountDatabaseCrawlerCache {
|
||||||
|
|
||||||
public boolean claimActiveWork(String workerId, long ttlMs) {
|
public boolean claimActiveWork(String workerId, long ttlMs) {
|
||||||
return "OK".equals(cacheCluster.withCluster(connection -> connection.sync()
|
return "OK".equals(cacheCluster.withCluster(connection -> connection.sync()
|
||||||
.set(getPrefixedKey(ACTIVE_WORKER_KEY), workerId, SetArgs.Builder.nx().px(ttlMs))));
|
.set(ACTIVE_WORKER_KEY, workerId, SetArgs.Builder.nx().px(ttlMs))));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void releaseActiveWork(String workerId) {
|
public void releaseActiveWork(String workerId) {
|
||||||
unlockClusterScript.execute(List.of(getPrefixedKey(ACTIVE_WORKER_KEY)), List.of(workerId));
|
unlockClusterScript.execute(List.of(ACTIVE_WORKER_KEY), List.of(workerId));
|
||||||
}
|
}
|
||||||
|
|
||||||
public Optional<UUID> getLastUuid() {
|
public Optional<UUID> getLastUuid() {
|
||||||
final String lastUuidString = cacheCluster.withCluster(
|
final String lastUuidString = cacheCluster.withCluster(
|
||||||
connection -> connection.sync().get(getPrefixedKey(LAST_UUID_KEY)));
|
connection -> connection.sync().get(LAST_UUID_KEY));
|
||||||
|
|
||||||
if (lastUuidString == null) {
|
if (lastUuidString == null) {
|
||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
|
@ -70,15 +68,15 @@ public class AccountDatabaseCrawlerCache {
|
||||||
public void setLastUuid(Optional<UUID> lastUuid) {
|
public void setLastUuid(Optional<UUID> lastUuid) {
|
||||||
if (lastUuid.isPresent()) {
|
if (lastUuid.isPresent()) {
|
||||||
cacheCluster.useCluster(connection -> connection.sync()
|
cacheCluster.useCluster(connection -> connection.sync()
|
||||||
.psetex(getPrefixedKey(LAST_UUID_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
|
.psetex(LAST_UUID_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
|
||||||
} else {
|
} else {
|
||||||
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_KEY)));
|
cacheCluster.useCluster(connection -> connection.sync().del(LAST_UUID_KEY));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public Optional<UUID> getLastUuidDynamo() {
|
public Optional<UUID> getLastUuidDynamo() {
|
||||||
final String lastUuidString = cacheCluster.withCluster(
|
final String lastUuidString = cacheCluster.withCluster(
|
||||||
connection -> connection.sync().get(getPrefixedKey(LAST_UUID_DYNAMO_KEY)));
|
connection -> connection.sync().get(LAST_UUID_DYNAMO_KEY));
|
||||||
|
|
||||||
if (lastUuidString == null) {
|
if (lastUuidString == null) {
|
||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
|
@ -91,21 +89,9 @@ public class AccountDatabaseCrawlerCache {
|
||||||
if (lastUuid.isPresent()) {
|
if (lastUuid.isPresent()) {
|
||||||
cacheCluster.useCluster(
|
cacheCluster.useCluster(
|
||||||
connection -> connection.sync()
|
connection -> connection.sync()
|
||||||
.psetex(getPrefixedKey(LAST_UUID_DYNAMO_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
|
.psetex(LAST_UUID_DYNAMO_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
|
||||||
} else {
|
} else {
|
||||||
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_DYNAMO_KEY)));
|
cacheCluster.useCluster(connection -> connection.sync().del(LAST_UUID_DYNAMO_KEY));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getPrefixedKey(final String key) {
|
|
||||||
return prefix + key;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Set a cache key prefix, allowing for uses beyond the canonical crawler
|
|
||||||
*/
|
|
||||||
public void setPrefix(final String prefix) {
|
|
||||||
this.prefix = prefix + "::";
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,172 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2013-2020 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import static com.codahale.metrics.MetricRegistry.name;
|
|
||||||
|
|
||||||
import com.codahale.metrics.MetricRegistry;
|
|
||||||
import com.codahale.metrics.SharedMetricRegistries;
|
|
||||||
import com.codahale.metrics.Timer;
|
|
||||||
import com.codahale.metrics.Timer.Context;
|
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.UUID;
|
|
||||||
import org.jdbi.v3.core.transaction.TransactionIsolationLevel;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper;
|
|
||||||
import org.whispersystems.textsecuregcm.util.Constants;
|
|
||||||
import org.whispersystems.textsecuregcm.util.SystemMapper;
|
|
||||||
|
|
||||||
public class Accounts implements AccountStore {
|
|
||||||
|
|
||||||
public static final String ID = "id";
|
|
||||||
public static final String UID = "uuid";
|
|
||||||
public static final String NUMBER = "number";
|
|
||||||
public static final String DATA = "data";
|
|
||||||
public static final String VERSION = "version";
|
|
||||||
|
|
||||||
private static final ObjectMapper mapper = SystemMapper.getMapper();
|
|
||||||
|
|
||||||
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
|
|
||||||
private final Timer createTimer = metricRegistry.timer(name(Accounts.class, "create" ));
|
|
||||||
private final Timer updateTimer = metricRegistry.timer(name(Accounts.class, "update" ));
|
|
||||||
private final Timer getByNumberTimer = metricRegistry.timer(name(Accounts.class, "getByNumber" ));
|
|
||||||
private final Timer getByUuidTimer = metricRegistry.timer(name(Accounts.class, "getByUuid" ));
|
|
||||||
private final Timer getAllFromTimer = metricRegistry.timer(name(Accounts.class, "getAllFrom" ));
|
|
||||||
private final Timer getAllFromOffsetTimer = metricRegistry.timer(name(Accounts.class, "getAllFromOffset"));
|
|
||||||
private final Timer deleteTimer = metricRegistry.timer(name(Accounts.class, "delete" ));
|
|
||||||
private final Timer vacuumTimer = metricRegistry.timer(name(Accounts.class, "vacuum" ));
|
|
||||||
|
|
||||||
private final FaultTolerantDatabase database;
|
|
||||||
|
|
||||||
public Accounts(FaultTolerantDatabase database) {
|
|
||||||
this.database = database;
|
|
||||||
this.database.getDatabase().registerRowMapper(new AccountRowMapper());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean create(Account account) {
|
|
||||||
return database.with(jdbi -> jdbi.inTransaction(TransactionIsolationLevel.SERIALIZABLE, handle -> {
|
|
||||||
try (Timer.Context ignored = createTimer.time()) {
|
|
||||||
final Map<String, Object> resultMap = handle.createQuery("INSERT INTO accounts (" + NUMBER + ", " + UID + ", " + DATA + ") VALUES (:number, :uuid, CAST(:data AS json)) ON CONFLICT(number) DO UPDATE SET " + DATA + " = EXCLUDED.data, " + VERSION + " = accounts.version + 1 RETURNING uuid, version")
|
|
||||||
.bind("number", account.getNumber())
|
|
||||||
.bind("uuid", account.getUuid())
|
|
||||||
.bind("data", mapper.writeValueAsString(account))
|
|
||||||
.mapToMap()
|
|
||||||
.findOnly();
|
|
||||||
|
|
||||||
final UUID uuid = (UUID) resultMap.get(UID);
|
|
||||||
final int version = (int) resultMap.get(VERSION);
|
|
||||||
|
|
||||||
boolean isNew = uuid.equals(account.getUuid());
|
|
||||||
account.setUuid(uuid);
|
|
||||||
account.setVersion(version);
|
|
||||||
return isNew;
|
|
||||||
} catch (JsonProcessingException e) {
|
|
||||||
throw new IllegalArgumentException(e);
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void update(Account account) throws ContestedOptimisticLockException {
|
|
||||||
database.use(jdbi -> jdbi.useHandle(handle -> {
|
|
||||||
try (Timer.Context ignored = updateTimer.time()) {
|
|
||||||
final int newVersion = account.getVersion() + 1;
|
|
||||||
int rowsModified = handle.createUpdate("UPDATE accounts SET " + DATA + " = CAST(:data AS json), " + VERSION + " = :newVersion WHERE " + UID + " = :uuid AND " + VERSION + " = :version")
|
|
||||||
.bind("uuid", account.getUuid())
|
|
||||||
.bind("data", mapper.writeValueAsString(account))
|
|
||||||
.bind("version", account.getVersion())
|
|
||||||
.bind("newVersion", newVersion)
|
|
||||||
.execute();
|
|
||||||
|
|
||||||
if (rowsModified == 0) {
|
|
||||||
throw new ContestedOptimisticLockException();
|
|
||||||
}
|
|
||||||
|
|
||||||
account.setVersion(newVersion);
|
|
||||||
|
|
||||||
} catch (JsonProcessingException e) {
|
|
||||||
throw new IllegalArgumentException(e);
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Optional<Account> get(String number) {
|
|
||||||
return database.with(jdbi -> jdbi.withHandle(handle -> {
|
|
||||||
try (Timer.Context ignored = getByNumberTimer.time()) {
|
|
||||||
return handle.createQuery("SELECT * FROM accounts WHERE " + NUMBER + " = :number")
|
|
||||||
.bind("number", number)
|
|
||||||
.mapTo(Account.class)
|
|
||||||
.findFirst();
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Optional<Account> get(UUID uuid) {
|
|
||||||
return database.with(jdbi -> jdbi.withHandle(handle -> {
|
|
||||||
try (Timer.Context ignored = getByUuidTimer.time()) {
|
|
||||||
return handle.createQuery("SELECT * FROM accounts WHERE " + UID + " = :uuid")
|
|
||||||
.bind("uuid", uuid)
|
|
||||||
.mapTo(Account.class)
|
|
||||||
.findFirst();
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
public AccountCrawlChunk getAllFrom(UUID from, int length) {
|
|
||||||
final List<Account> accounts = database.with(jdbi -> jdbi.withHandle(handle -> {
|
|
||||||
try (Context ignored = getAllFromOffsetTimer.time()) {
|
|
||||||
return handle.createQuery("SELECT * FROM accounts WHERE " + UID + " > :from ORDER BY " + UID + " LIMIT :limit")
|
|
||||||
.bind("from", from)
|
|
||||||
.bind("limit", length)
|
|
||||||
.mapTo(Account.class)
|
|
||||||
.list();
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
return buildChunkForAccounts(accounts);
|
|
||||||
}
|
|
||||||
|
|
||||||
public AccountCrawlChunk getAllFrom(int length) {
|
|
||||||
final List<Account> accounts = database.with(jdbi -> jdbi.withHandle(handle -> {
|
|
||||||
try (Timer.Context ignored = getAllFromTimer.time()) {
|
|
||||||
return handle.createQuery("SELECT * FROM accounts ORDER BY " + UID + " LIMIT :limit")
|
|
||||||
.bind("limit", length)
|
|
||||||
.mapTo(Account.class)
|
|
||||||
.list();
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
return buildChunkForAccounts(accounts);
|
|
||||||
}
|
|
||||||
|
|
||||||
private AccountCrawlChunk buildChunkForAccounts(final List<Account> accounts) {
|
|
||||||
return new AccountCrawlChunk(accounts, accounts.isEmpty() ? null : accounts.get(accounts.size() - 1).getUuid());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void delete(final UUID uuid) {
|
|
||||||
database.use(jdbi -> jdbi.useHandle(handle -> {
|
|
||||||
try (Timer.Context ignored = deleteTimer.time()) {
|
|
||||||
handle.createUpdate("DELETE FROM accounts WHERE " + UID + " = :uuid")
|
|
||||||
.bind("uuid", uuid)
|
|
||||||
.execute();
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void vacuum() {
|
|
||||||
database.use(jdbi -> jdbi.useHandle(handle -> {
|
|
||||||
try (Timer.Context ignored = vacuumTimer.time()) {
|
|
||||||
handle.execute("VACUUM accounts");
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -8,7 +8,6 @@ import static com.codahale.metrics.MetricRegistry.name;
|
||||||
|
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import io.micrometer.core.instrument.Counter;
|
|
||||||
import io.micrometer.core.instrument.Metrics;
|
import io.micrometer.core.instrument.Metrics;
|
||||||
import io.micrometer.core.instrument.Timer;
|
import io.micrometer.core.instrument.Timer;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -17,16 +16,10 @@ import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.CompletableFuture;
|
|
||||||
import java.util.concurrent.CompletionException;
|
|
||||||
import java.util.concurrent.ThreadPoolExecutor;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||||
import org.whispersystems.textsecuregcm.util.SystemMapper;
|
import org.whispersystems.textsecuregcm.util.SystemMapper;
|
||||||
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
import org.whispersystems.textsecuregcm.util.UUIDUtil;
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.CancellationReason;
|
import software.amazon.awssdk.services.dynamodb.model.CancellationReason;
|
||||||
|
@ -59,12 +52,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||||
static final String ATTR_CANONICALLY_DISCOVERABLE = "C";
|
static final String ATTR_CANONICALLY_DISCOVERABLE = "C";
|
||||||
|
|
||||||
private final DynamoDbClient client;
|
private final DynamoDbClient client;
|
||||||
private final DynamoDbAsyncClient asyncClient;
|
|
||||||
|
|
||||||
private final ThreadPoolExecutor migrationThreadPool;
|
|
||||||
|
|
||||||
private final MigrationDeletedAccounts migrationDeletedAccounts;
|
|
||||||
private final MigrationRetryAccounts migrationRetryAccounts;
|
|
||||||
|
|
||||||
private final String phoneNumbersTableName;
|
private final String phoneNumbersTableName;
|
||||||
private final String accountsTableName;
|
private final String accountsTableName;
|
||||||
|
@ -76,26 +63,15 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||||
private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFrom"));
|
private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFrom"));
|
||||||
private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFromOffset"));
|
private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "getAllFromOffset"));
|
||||||
private static final Timer DELETE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "delete"));
|
private static final Timer DELETE_TIMER = Metrics.timer(name(AccountsDynamoDb.class, "delete"));
|
||||||
private static final Timer DELETE_RECENTLY_DELETED_UUIDS_TIMER = Metrics.timer(
|
|
||||||
name(AccountsDynamoDb.class, "deleteRecentlyDeletedUuids"));
|
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(AccountsDynamoDb.class);
|
|
||||||
|
|
||||||
public AccountsDynamoDb(DynamoDbClient client, DynamoDbAsyncClient asyncClient,
|
public AccountsDynamoDb(DynamoDbClient client, String accountsTableName, String phoneNumbersTableName) {
|
||||||
ThreadPoolExecutor migrationThreadPool, String accountsTableName, String phoneNumbersTableName,
|
|
||||||
MigrationDeletedAccounts migrationDeletedAccounts,
|
|
||||||
MigrationRetryAccounts accountsMigrationErrors) {
|
|
||||||
|
|
||||||
super(client);
|
super(client);
|
||||||
|
|
||||||
this.client = client;
|
this.client = client;
|
||||||
this.asyncClient = asyncClient;
|
|
||||||
this.phoneNumbersTableName = phoneNumbersTableName;
|
this.phoneNumbersTableName = phoneNumbersTableName;
|
||||||
this.accountsTableName = accountsTableName;
|
this.accountsTableName = accountsTableName;
|
||||||
this.migrationThreadPool = migrationThreadPool;
|
|
||||||
|
|
||||||
this.migrationDeletedAccounts = migrationDeletedAccounts;
|
|
||||||
this.migrationRetryAccounts = accountsMigrationErrors;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -215,29 +191,19 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
try {
|
UpdateItemResponse response = client.updateItem(updateItemRequest);
|
||||||
UpdateItemResponse response = client.updateItem(updateItemRequest);
|
|
||||||
|
|
||||||
account.setVersion(AttributeValues.getInt(response.attributes(), "V", account.getVersion() + 1));
|
account.setVersion(AttributeValues.getInt(response.attributes(), "V", account.getVersion() + 1));
|
||||||
} catch (final TransactionConflictException e) {
|
} catch (final TransactionConflictException e) {
|
||||||
|
|
||||||
throw new ContestedOptimisticLockException();
|
throw new ContestedOptimisticLockException();
|
||||||
|
|
||||||
} catch (final ConditionalCheckFailedException e) {
|
} catch (final ConditionalCheckFailedException e) {
|
||||||
|
|
||||||
// the exception doesn’t give details about which condition failed,
|
// the exception doesn’t give details about which condition failed,
|
||||||
// but we can infer it was an optimistic locking failure if the UUID is known
|
// but we can infer it was an optimistic locking failure if the UUID is known
|
||||||
throw get(account.getUuid()).isPresent() ? new ContestedOptimisticLockException() : e;
|
throw get(account.getUuid()).isPresent() ? new ContestedOptimisticLockException() : e;
|
||||||
}
|
|
||||||
} catch (final Exception e) {
|
|
||||||
if (!(e instanceof ContestedOptimisticLockException)) {
|
|
||||||
// the Dynamo account now lags the Postgres account version. Put it in the migration retry table so that it will
|
|
||||||
// get updated faster—otherwise it will be stale until the accounts crawler runs again
|
|
||||||
migrationRetryAccounts.put(account.getUuid());
|
|
||||||
}
|
|
||||||
throw e;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -279,10 +245,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||||
DELETE_TIMER.record(() -> delete(uuid, true));
|
DELETE_TIMER.record(() -> delete(uuid, true));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteInvalidMigration(UUID uuid) {
|
|
||||||
DELETE_TIMER.record(() -> delete(uuid, false));
|
|
||||||
}
|
|
||||||
|
|
||||||
public AccountCrawlChunk getAllFrom(final UUID from, final int maxCount, final int pageSize) {
|
public AccountCrawlChunk getAllFrom(final UUID from, final int maxCount, final int pageSize) {
|
||||||
final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder()
|
final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder()
|
||||||
.limit(pageSize)
|
.limit(pageSize)
|
||||||
|
@ -312,10 +274,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||||
|
|
||||||
private void delete(UUID uuid, boolean saveInDeletedAccountsTable) {
|
private void delete(UUID uuid, boolean saveInDeletedAccountsTable) {
|
||||||
|
|
||||||
if (saveInDeletedAccountsTable) {
|
|
||||||
migrationDeletedAccounts.put(uuid);
|
|
||||||
}
|
|
||||||
|
|
||||||
Optional<Account> maybeAccount = get(uuid);
|
Optional<Account> maybeAccount = get(uuid);
|
||||||
|
|
||||||
maybeAccount.ifPresent(account -> {
|
maybeAccount.ifPresent(account -> {
|
||||||
|
@ -341,105 +299,6 @@ public class AccountsDynamoDb extends AbstractDynamoDbStore implements AccountSt
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final Counter MIGRATED_COUNTER = Metrics.counter(name(AccountsDynamoDb.class, "migration", "count"));
|
|
||||||
private static final Counter ERROR_COUNTER = Metrics.counter(name(AccountsDynamoDb.class, "migration", "error"));
|
|
||||||
|
|
||||||
public CompletableFuture<Void> migrate(List<Account> accounts, int threads) {
|
|
||||||
|
|
||||||
if (threads > migrationThreadPool.getMaximumPoolSize()) {
|
|
||||||
migrationThreadPool.setMaximumPoolSize(threads);
|
|
||||||
migrationThreadPool.setCorePoolSize(threads);
|
|
||||||
} else {
|
|
||||||
migrationThreadPool.setCorePoolSize(threads);
|
|
||||||
migrationThreadPool.setMaximumPoolSize(threads);
|
|
||||||
}
|
|
||||||
|
|
||||||
final List<CompletableFuture<?>> futures = accounts.stream()
|
|
||||||
.map(this::migrate)
|
|
||||||
.map(f -> f.whenCompleteAsync((migrated, e) -> {
|
|
||||||
if (e == null) {
|
|
||||||
MIGRATED_COUNTER.increment(migrated ? 1 : 0);
|
|
||||||
} else {
|
|
||||||
ERROR_COUNTER.increment();
|
|
||||||
}
|
|
||||||
}, migrationThreadPool))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
|
|
||||||
CompletableFuture<Void> migrationBatch = CompletableFuture.allOf(futures.toArray(new CompletableFuture[]{}));
|
|
||||||
|
|
||||||
return migrationBatch.whenCompleteAsync((result, exception) -> {
|
|
||||||
if (exception != null) {
|
|
||||||
logger.warn("Exception migrating batch", exception);
|
|
||||||
}
|
|
||||||
deleteRecentlyDeletedUuids();
|
|
||||||
}, migrationThreadPool);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void deleteRecentlyDeletedUuids() {
|
|
||||||
|
|
||||||
DELETE_RECENTLY_DELETED_UUIDS_TIMER.record(() -> {
|
|
||||||
|
|
||||||
final List<UUID> recentlyDeletedUuids = migrationDeletedAccounts.getRecentlyDeletedUuids();
|
|
||||||
|
|
||||||
for (UUID recentlyDeletedUuid : recentlyDeletedUuids) {
|
|
||||||
delete(recentlyDeletedUuid, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
migrationDeletedAccounts.delete(recentlyDeletedUuids);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public CompletableFuture<Boolean> migrate(Account account) {
|
|
||||||
try {
|
|
||||||
TransactWriteItem phoneNumberConstraintPut = buildPutWriteItemForPhoneNumberConstraint(account, account.getUuid());
|
|
||||||
|
|
||||||
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid(), Put.builder()
|
|
||||||
.conditionExpression("attribute_not_exists(#uuid) OR (attribute_exists(#uuid) AND #version < :version)")
|
|
||||||
.expressionAttributeNames(Map.of(
|
|
||||||
"#uuid", KEY_ACCOUNT_UUID,
|
|
||||||
"#version", ATTR_VERSION))
|
|
||||||
.expressionAttributeValues(Map.of(
|
|
||||||
":version", AttributeValues.fromInt(account.getVersion()))));
|
|
||||||
|
|
||||||
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
|
|
||||||
.transactItems(phoneNumberConstraintPut, accountPut).build();
|
|
||||||
|
|
||||||
final CompletableFuture<Boolean> resultFuture = new CompletableFuture<>();
|
|
||||||
asyncClient.transactWriteItems(request).whenCompleteAsync((result, exception) -> {
|
|
||||||
if (result != null) {
|
|
||||||
resultFuture.complete(true);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (exception instanceof CompletionException) {
|
|
||||||
// whenCompleteAsync can wrap exceptions in a CompletionException; unwrap it to get to the root cause.
|
|
||||||
exception = exception.getCause();
|
|
||||||
}
|
|
||||||
if (exception instanceof TransactionCanceledException) {
|
|
||||||
// account is already migrated
|
|
||||||
resultFuture.complete(false);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
migrationRetryAccounts.put(account.getUuid());
|
|
||||||
} catch (final Exception e) {
|
|
||||||
logger.error("Could not store account {}", account.getUuid());
|
|
||||||
}
|
|
||||||
resultFuture.completeExceptionally(exception);
|
|
||||||
}, migrationThreadPool);
|
|
||||||
return resultFuture;
|
|
||||||
} catch (Exception e) {
|
|
||||||
return CompletableFuture.failedFuture(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void putUuidForMigrationRetry(final UUID uuid) {
|
|
||||||
try {
|
|
||||||
migrationRetryAccounts.put(uuid);
|
|
||||||
} catch (final Exception e) {
|
|
||||||
logger.error("Failed to store for retry: {}", uuid, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String extractCancellationReasonCodes(final TransactionCanceledException exception) {
|
private static String extractCancellationReasonCodes(final TransactionCanceledException exception) {
|
||||||
return exception.cancellationReasons().stream()
|
return exception.cancellationReasons().stream()
|
||||||
.map(CancellationReason::code)
|
.map(CancellationReason::code)
|
||||||
|
|
|
@ -1,41 +0,0 @@
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.concurrent.CompletableFuture;
|
|
||||||
|
|
||||||
public class AccountsDynamoDbMigrator extends AccountDatabaseCrawlerListener {
|
|
||||||
|
|
||||||
private final AccountsDynamoDb accountsDynamoDb;
|
|
||||||
private final DynamicConfigurationManager dynamicConfigurationManager;
|
|
||||||
|
|
||||||
public AccountsDynamoDbMigrator(final AccountsDynamoDb accountsDynamoDb, final DynamicConfigurationManager dynamicConfigurationManager) {
|
|
||||||
this.accountsDynamoDb = accountsDynamoDb;
|
|
||||||
this.dynamicConfigurationManager = dynamicConfigurationManager;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onCrawlStart() {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void onCrawlEnd(Optional<UUID> fromUuid) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void onCrawlChunk(Optional<UUID> fromUuid, List<Account> chunkAccounts) {
|
|
||||||
|
|
||||||
if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
|
|
||||||
.isBackgroundMigrationEnabled()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
final CompletableFuture<Void> migrationBatch = accountsDynamoDb.migrate(chunkAccounts,
|
|
||||||
dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().getBackgroundMigrationExecutorThreads());
|
|
||||||
|
|
||||||
migrationBatch.join();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -11,22 +11,17 @@ import com.codahale.metrics.Meter;
|
||||||
import com.codahale.metrics.MetricRegistry;
|
import com.codahale.metrics.MetricRegistry;
|
||||||
import com.codahale.metrics.SharedMetricRegistries;
|
import com.codahale.metrics.SharedMetricRegistries;
|
||||||
import com.codahale.metrics.Timer;
|
import com.codahale.metrics.Timer;
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import io.lettuce.core.RedisException;
|
import io.lettuce.core.RedisException;
|
||||||
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
|
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
|
||||||
import io.micrometer.core.instrument.Counter;
|
|
||||||
import io.micrometer.core.instrument.Metrics;
|
import io.micrometer.core.instrument.Metrics;
|
||||||
import io.micrometer.core.instrument.Tags;
|
import io.micrometer.core.instrument.Tags;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.Callable;
|
|
||||||
import java.util.concurrent.CompletableFuture;
|
import java.util.concurrent.CompletableFuture;
|
||||||
import java.util.function.BiFunction;
|
|
||||||
import java.util.function.Consumer;
|
import java.util.function.Consumer;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.function.Supplier;
|
import java.util.function.Supplier;
|
||||||
|
@ -37,7 +32,6 @@ import org.slf4j.LoggerFactory;
|
||||||
import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
|
import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
|
||||||
import org.whispersystems.textsecuregcm.controllers.AccountController;
|
import org.whispersystems.textsecuregcm.controllers.AccountController;
|
||||||
import org.whispersystems.textsecuregcm.entities.AccountAttributes;
|
import org.whispersystems.textsecuregcm.entities.AccountAttributes;
|
||||||
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
|
|
||||||
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
||||||
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
||||||
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
||||||
|
@ -69,20 +63,14 @@ public class AccountsManager {
|
||||||
private static final String COUNTRY_CODE_TAG_NAME = "country";
|
private static final String COUNTRY_CODE_TAG_NAME = "country";
|
||||||
private static final String DELETION_REASON_TAG_NAME = "reason";
|
private static final String DELETION_REASON_TAG_NAME = "reason";
|
||||||
|
|
||||||
private static final String DYNAMO_MIGRATION_ERROR_COUNTER_NAME = name(AccountsManager.class, "migration", "error");
|
|
||||||
private static final Counter DYNAMO_MIGRATION_COMPARISON_COUNTER = Metrics.counter(name(AccountsManager.class, "migration", "comparisons"));
|
|
||||||
private static final String DYNAMO_MIGRATION_MISMATCH_COUNTER_NAME = name(AccountsManager.class, "migration", "mismatches");
|
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(AccountsManager.class);
|
private final Logger logger = LoggerFactory.getLogger(AccountsManager.class);
|
||||||
|
|
||||||
private final Accounts accounts;
|
|
||||||
private final AccountsDynamoDb accountsDynamoDb;
|
private final AccountsDynamoDb accountsDynamoDb;
|
||||||
private final FaultTolerantRedisCluster cacheCluster;
|
private final FaultTolerantRedisCluster cacheCluster;
|
||||||
private final DeletedAccountsManager deletedAccountsManager;
|
private final DeletedAccountsManager deletedAccountsManager;
|
||||||
private final DirectoryQueue directoryQueue;
|
private final DirectoryQueue directoryQueue;
|
||||||
private final KeysDynamoDb keysDynamoDb;
|
private final KeysDynamoDb keysDynamoDb;
|
||||||
private final MessagesManager messagesManager;
|
private final MessagesManager messagesManager;
|
||||||
private final MigrationMismatchedAccounts mismatchedAccounts;
|
|
||||||
private final UsernamesManager usernamesManager;
|
private final UsernamesManager usernamesManager;
|
||||||
private final ProfilesManager profilesManager;
|
private final ProfilesManager profilesManager;
|
||||||
private final StoredVerificationCodeManager pendingAccounts;
|
private final StoredVerificationCodeManager pendingAccounts;
|
||||||
|
@ -90,10 +78,7 @@ public class AccountsManager {
|
||||||
private final SecureBackupClient secureBackupClient;
|
private final SecureBackupClient secureBackupClient;
|
||||||
private final ObjectMapper mapper;
|
private final ObjectMapper mapper;
|
||||||
|
|
||||||
private final ObjectMapper migrationComparisonMapper;
|
|
||||||
|
|
||||||
private final DynamicConfigurationManager dynamicConfigurationManager;
|
private final DynamicConfigurationManager dynamicConfigurationManager;
|
||||||
private final ExperimentEnrollmentManager experimentEnrollmentManager;
|
|
||||||
|
|
||||||
public enum DeletionReason {
|
public enum DeletionReason {
|
||||||
ADMIN_DELETED("admin"),
|
ADMIN_DELETED("admin"),
|
||||||
|
@ -107,25 +92,22 @@ public class AccountsManager {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public AccountsManager(Accounts accounts, AccountsDynamoDb accountsDynamoDb, FaultTolerantRedisCluster cacheCluster,
|
public AccountsManager(AccountsDynamoDb accountsDynamoDb, FaultTolerantRedisCluster cacheCluster,
|
||||||
final DeletedAccountsManager deletedAccountsManager,
|
final DeletedAccountsManager deletedAccountsManager,
|
||||||
final DirectoryQueue directoryQueue,
|
final DirectoryQueue directoryQueue,
|
||||||
final KeysDynamoDb keysDynamoDb, final MessagesManager messagesManager,
|
final KeysDynamoDb keysDynamoDb, final MessagesManager messagesManager,
|
||||||
final MigrationMismatchedAccounts mismatchedAccounts, final UsernamesManager usernamesManager,
|
final UsernamesManager usernamesManager,
|
||||||
final ProfilesManager profilesManager,
|
final ProfilesManager profilesManager,
|
||||||
final StoredVerificationCodeManager pendingAccounts,
|
final StoredVerificationCodeManager pendingAccounts,
|
||||||
final SecureStorageClient secureStorageClient,
|
final SecureStorageClient secureStorageClient,
|
||||||
final SecureBackupClient secureBackupClient,
|
final SecureBackupClient secureBackupClient,
|
||||||
final ExperimentEnrollmentManager experimentEnrollmentManager,
|
|
||||||
final DynamicConfigurationManager dynamicConfigurationManager) {
|
final DynamicConfigurationManager dynamicConfigurationManager) {
|
||||||
this.accounts = accounts;
|
this.accountsDynamoDb = accountsDynamoDb;
|
||||||
this.accountsDynamoDb = accountsDynamoDb;
|
|
||||||
this.cacheCluster = cacheCluster;
|
this.cacheCluster = cacheCluster;
|
||||||
this.deletedAccountsManager = deletedAccountsManager;
|
this.deletedAccountsManager = deletedAccountsManager;
|
||||||
this.directoryQueue = directoryQueue;
|
this.directoryQueue = directoryQueue;
|
||||||
this.keysDynamoDb = keysDynamoDb;
|
this.keysDynamoDb = keysDynamoDb;
|
||||||
this.messagesManager = messagesManager;
|
this.messagesManager = messagesManager;
|
||||||
this.mismatchedAccounts = mismatchedAccounts;
|
|
||||||
this.usernamesManager = usernamesManager;
|
this.usernamesManager = usernamesManager;
|
||||||
this.profilesManager = profilesManager;
|
this.profilesManager = profilesManager;
|
||||||
this.pendingAccounts = pendingAccounts;
|
this.pendingAccounts = pendingAccounts;
|
||||||
|
@ -133,11 +115,7 @@ public class AccountsManager {
|
||||||
this.secureBackupClient = secureBackupClient;
|
this.secureBackupClient = secureBackupClient;
|
||||||
this.mapper = SystemMapper.getMapper();
|
this.mapper = SystemMapper.getMapper();
|
||||||
|
|
||||||
this.migrationComparisonMapper = mapper.copy();
|
|
||||||
migrationComparisonMapper.addMixIn(Device.class, DeviceComparisonMixin.class);
|
|
||||||
|
|
||||||
this.dynamicConfigurationManager = dynamicConfigurationManager;
|
this.dynamicConfigurationManager = dynamicConfigurationManager;
|
||||||
this.experimentEnrollmentManager = experimentEnrollmentManager;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Account create(final String number,
|
public Account create(final String number,
|
||||||
|
@ -170,36 +148,12 @@ public class AccountsManager {
|
||||||
|
|
||||||
final UUID originalUuid = account.getUuid();
|
final UUID originalUuid = account.getUuid();
|
||||||
|
|
||||||
boolean freshUser = primaryCreate(account);
|
boolean freshUser = dynamoCreate(account);
|
||||||
|
|
||||||
// create() sometimes updates the UUID, if there was a number conflict.
|
// create() sometimes updates the UUID, if there was a number conflict.
|
||||||
// for metrics, we want secondary to run with the same original UUID
|
// for metrics, we want secondary to run with the same original UUID
|
||||||
final UUID actualUuid = account.getUuid();
|
final UUID actualUuid = account.getUuid();
|
||||||
|
|
||||||
try {
|
|
||||||
if (secondaryWriteEnabled()) {
|
|
||||||
|
|
||||||
account.setUuid(originalUuid);
|
|
||||||
|
|
||||||
runSafelyAndRecordMetrics(() -> secondaryCreate(account), Optional.of(account.getUuid()), freshUser,
|
|
||||||
(primaryResult, secondaryResult) -> {
|
|
||||||
|
|
||||||
if (primaryResult.equals(secondaryResult)) {
|
|
||||||
return Optional.empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (secondaryResult) {
|
|
||||||
return Optional.of("secondaryFreshUser");
|
|
||||||
}
|
|
||||||
|
|
||||||
return Optional.of("primaryFreshUser");
|
|
||||||
},
|
|
||||||
"create");
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
account.setUuid(actualUuid);
|
|
||||||
}
|
|
||||||
|
|
||||||
redisSet(account);
|
redisSet(account);
|
||||||
|
|
||||||
pendingAccounts.remove(number);
|
pendingAccounts.remove(number);
|
||||||
|
@ -293,26 +247,7 @@ public class AccountsManager {
|
||||||
|
|
||||||
final UUID uuid = account.getUuid();
|
final UUID uuid = account.getUuid();
|
||||||
|
|
||||||
updatedAccount = updateWithRetries(account, updater, this::primaryUpdate, () -> primaryGet(uuid).get());
|
updatedAccount = updateWithRetries(account, updater, this::dynamoUpdate, () -> dynamoGet(uuid).get());
|
||||||
|
|
||||||
if (secondaryWriteEnabled()) {
|
|
||||||
runSafelyAndRecordMetrics(() -> secondaryGet(uuid).map(secondaryAccount -> {
|
|
||||||
try {
|
|
||||||
return updateWithRetries(secondaryAccount, updater, this::secondaryUpdate, () -> secondaryGet(uuid).get());
|
|
||||||
} catch (final OptimisticLockRetryLimitExceededException e) {
|
|
||||||
if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
|
|
||||||
.isDynamoPrimary()) {
|
|
||||||
accountsDynamoDb.putUuidForMigrationRetry(uuid);
|
|
||||||
}
|
|
||||||
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
Optional.of(uuid),
|
|
||||||
Optional.of(updatedAccount),
|
|
||||||
this::compareAccounts,
|
|
||||||
"update");
|
|
||||||
}
|
|
||||||
|
|
||||||
redisSet(updatedAccount);
|
redisSet(updatedAccount);
|
||||||
}
|
}
|
||||||
|
@ -378,14 +313,9 @@ public class AccountsManager {
|
||||||
try (Timer.Context ignored = getByNumberTimer.time()) {
|
try (Timer.Context ignored = getByNumberTimer.time()) {
|
||||||
Optional<Account> account = redisGet(number);
|
Optional<Account> account = redisGet(number);
|
||||||
|
|
||||||
if (!account.isPresent()) {
|
if (account.isEmpty()) {
|
||||||
account = primaryGet(number);
|
account = dynamoGet(number);
|
||||||
account.ifPresent(value -> redisSet(value));
|
account.ifPresent(this::redisSet);
|
||||||
|
|
||||||
if (secondaryReadEnabled()) {
|
|
||||||
runSafelyAndRecordMetrics(() -> secondaryGet(number), Optional.empty(), account, this::compareAccounts,
|
|
||||||
"getByNumber");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return account;
|
return account;
|
||||||
|
@ -396,29 +326,15 @@ public class AccountsManager {
|
||||||
try (Timer.Context ignored = getByUuidTimer.time()) {
|
try (Timer.Context ignored = getByUuidTimer.time()) {
|
||||||
Optional<Account> account = redisGet(uuid);
|
Optional<Account> account = redisGet(uuid);
|
||||||
|
|
||||||
if (!account.isPresent()) {
|
if (account.isEmpty()) {
|
||||||
account = primaryGet(uuid);
|
account = dynamoGet(uuid);
|
||||||
account.ifPresent(value -> redisSet(value));
|
account.ifPresent(this::redisSet);
|
||||||
|
|
||||||
if (secondaryReadEnabled()) {
|
|
||||||
runSafelyAndRecordMetrics(() -> secondaryGet(uuid), Optional.of(uuid), account, this::compareAccounts,
|
|
||||||
"getByUuid");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return account;
|
return account;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public AccountCrawlChunk getAllFrom(int length) {
|
|
||||||
return accounts.getAllFrom(length);
|
|
||||||
}
|
|
||||||
|
|
||||||
public AccountCrawlChunk getAllFrom(UUID uuid, int length) {
|
|
||||||
return accounts.getAllFrom(uuid, length);
|
|
||||||
}
|
|
||||||
|
|
||||||
public AccountCrawlChunk getAllFromDynamo(int length) {
|
public AccountCrawlChunk getAllFromDynamo(int length) {
|
||||||
final int maxPageSize = dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
|
final int maxPageSize = dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
|
||||||
.getDynamoCrawlerScanPageSize();
|
.getDynamoCrawlerScanPageSize();
|
||||||
|
@ -447,16 +363,7 @@ public class AccountsManager {
|
||||||
deleteBackupServiceDataFuture.join();
|
deleteBackupServiceDataFuture.join();
|
||||||
|
|
||||||
redisDelete(account);
|
redisDelete(account);
|
||||||
primaryDelete(account);
|
dynamoDelete(account);
|
||||||
|
|
||||||
if (secondaryDeleteEnabled()) {
|
|
||||||
try {
|
|
||||||
secondaryDelete(account);
|
|
||||||
} catch (final Exception e) {
|
|
||||||
logger.error("Could not delete account {} from secondary", account.getUuid().toString());
|
|
||||||
Metrics.counter(DYNAMO_MIGRATION_ERROR_COUNTER_NAME, "action", "delete").increment();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return account.getUuid();
|
return account.getUuid();
|
||||||
});
|
});
|
||||||
|
@ -537,100 +444,6 @@ public class AccountsManager {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Optional<Account> primaryGet(String number) {
|
|
||||||
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
|
|
||||||
?
|
|
||||||
dynamoGet(number) :
|
|
||||||
databaseGet(number);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Optional<Account> secondaryGet(String number) {
|
|
||||||
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
|
|
||||||
?
|
|
||||||
databaseGet(number) :
|
|
||||||
dynamoGet(number);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Optional<Account> primaryGet(UUID uuid) {
|
|
||||||
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
|
|
||||||
?
|
|
||||||
dynamoGet(uuid) :
|
|
||||||
databaseGet(uuid);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Optional<Account> secondaryGet(UUID uuid) {
|
|
||||||
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
|
|
||||||
?
|
|
||||||
databaseGet(uuid) :
|
|
||||||
dynamoGet(uuid);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean primaryCreate(Account account) {
|
|
||||||
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
|
|
||||||
?
|
|
||||||
dynamoCreate(account) :
|
|
||||||
databaseCreate(account);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean secondaryCreate(Account account) {
|
|
||||||
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()
|
|
||||||
?
|
|
||||||
databaseCreate(account) :
|
|
||||||
dynamoCreate(account);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void primaryUpdate(Account account) {
|
|
||||||
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) {
|
|
||||||
dynamoUpdate(account);
|
|
||||||
} else {
|
|
||||||
databaseUpdate(account);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void secondaryUpdate(Account account) {
|
|
||||||
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) {
|
|
||||||
databaseUpdate(account);
|
|
||||||
} else {
|
|
||||||
dynamoUpdate(account);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void primaryDelete(Account account) {
|
|
||||||
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) {
|
|
||||||
dynamoDelete(account);
|
|
||||||
} else {
|
|
||||||
databaseDelete(account);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void secondaryDelete(Account account) {
|
|
||||||
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDynamoPrimary()) {
|
|
||||||
databaseDelete(account);
|
|
||||||
} else {
|
|
||||||
dynamoDelete(account);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Optional<Account> databaseGet(String number) {
|
|
||||||
return accounts.get(number);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Optional<Account> databaseGet(UUID uuid) {
|
|
||||||
return accounts.get(uuid);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean databaseCreate(Account account) {
|
|
||||||
return accounts.create(account);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void databaseUpdate(Account account) {
|
|
||||||
accounts.update(account);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void databaseDelete(final Account account) {
|
|
||||||
accounts.delete(account.getUuid());
|
|
||||||
}
|
|
||||||
|
|
||||||
private Optional<Account> dynamoGet(String number) {
|
private Optional<Account> dynamoGet(String number) {
|
||||||
return accountsDynamoDb.get(number);
|
return accountsDynamoDb.get(number);
|
||||||
}
|
}
|
||||||
|
@ -651,175 +464,14 @@ public class AccountsManager {
|
||||||
accountsDynamoDb.delete(account.getUuid());
|
accountsDynamoDb.delete(account.getUuid());
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean secondaryDeleteEnabled() {
|
// TODO delete
|
||||||
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isDeleteEnabled();
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean secondaryReadEnabled() {
|
|
||||||
return dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isReadEnabled();
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean secondaryWriteEnabled() {
|
|
||||||
return secondaryDeleteEnabled()
|
|
||||||
&& dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration().isWriteEnabled();
|
|
||||||
}
|
|
||||||
|
|
||||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
||||||
|
@Deprecated
|
||||||
public Optional<String> compareAccounts(final Optional<Account> maybePrimaryAccount,
|
public Optional<String> compareAccounts(final Optional<Account> maybePrimaryAccount,
|
||||||
final Optional<Account> maybeSecondaryAccount) {
|
final Optional<Account> maybeSecondaryAccount) {
|
||||||
|
|
||||||
if (maybePrimaryAccount.isEmpty() && maybeSecondaryAccount.isEmpty()) {
|
|
||||||
return Optional.empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (maybePrimaryAccount.isEmpty()) {
|
|
||||||
return Optional.of("primaryMissing");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (maybeSecondaryAccount.isEmpty()) {
|
|
||||||
return Optional.of("secondaryMissing");
|
|
||||||
}
|
|
||||||
|
|
||||||
final Account primaryAccount = maybePrimaryAccount.get();
|
|
||||||
final Account secondaryAccount = maybeSecondaryAccount.get();
|
|
||||||
|
|
||||||
final int uuidCompare = primaryAccount.getUuid().compareTo(secondaryAccount.getUuid());
|
|
||||||
|
|
||||||
if (uuidCompare != 0) {
|
|
||||||
return Optional.of("uuid");
|
|
||||||
}
|
|
||||||
|
|
||||||
final int numberCompare = primaryAccount.getNumber().compareTo(secondaryAccount.getNumber());
|
|
||||||
|
|
||||||
if (numberCompare != 0) {
|
|
||||||
return Optional.of("number");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!Objects.equals(primaryAccount.getIdentityKey(), secondaryAccount.getIdentityKey())) {
|
|
||||||
return Optional.of("identityKey");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!Objects.equals(primaryAccount.getCurrentProfileVersion(), secondaryAccount.getCurrentProfileVersion())) {
|
|
||||||
return Optional.of("currentProfileVersion");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!Objects.equals(primaryAccount.getProfileName(), secondaryAccount.getProfileName())) {
|
|
||||||
return Optional.of("profileName");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!Objects.equals(primaryAccount.getAvatar(), secondaryAccount.getAvatar())) {
|
|
||||||
return Optional.of("avatar");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!Objects.equals(primaryAccount.getUnidentifiedAccessKey(), secondaryAccount.getUnidentifiedAccessKey())) {
|
|
||||||
if (primaryAccount.getUnidentifiedAccessKey().isPresent() && secondaryAccount.getUnidentifiedAccessKey()
|
|
||||||
.isPresent()) {
|
|
||||||
|
|
||||||
if (Arrays.compare(primaryAccount.getUnidentifiedAccessKey().get(),
|
|
||||||
secondaryAccount.getUnidentifiedAccessKey().get()) != 0) {
|
|
||||||
return Optional.of("unidentifiedAccessKey");
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
return Optional.of("unidentifiedAccessKey");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!Objects.equals(primaryAccount.isUnrestrictedUnidentifiedAccess(),
|
|
||||||
secondaryAccount.isUnrestrictedUnidentifiedAccess())) {
|
|
||||||
return Optional.of("unrestrictedUnidentifiedAccess");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!Objects.equals(primaryAccount.isDiscoverableByPhoneNumber(), secondaryAccount.isDiscoverableByPhoneNumber())) {
|
|
||||||
return Optional.of("discoverableByPhoneNumber");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (primaryAccount.getMasterDevice().isPresent() && secondaryAccount.getMasterDevice().isPresent()) {
|
|
||||||
if (!Objects.equals(primaryAccount.getMasterDevice().get().getSignedPreKey(),
|
|
||||||
secondaryAccount.getMasterDevice().get().getSignedPreKey())) {
|
|
||||||
return Optional.of("masterDeviceSignedPreKey");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (!serializedEquals(primaryAccount.getDevices(), secondaryAccount.getDevices())) {
|
|
||||||
return Optional.of("devices");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (primaryAccount.getVersion() != secondaryAccount.getVersion()) {
|
|
||||||
return Optional.of("version");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (primaryAccount.getMasterDevice().isPresent() && secondaryAccount.getMasterDevice().isPresent()) {
|
|
||||||
if (Math.abs(primaryAccount.getMasterDevice().get().getPushTimestamp() -
|
|
||||||
secondaryAccount.getMasterDevice().get().getPushTimestamp()) > 60 * 1_000L) {
|
|
||||||
// These are generally few milliseconds off, because the setter uses System.currentTimeMillis() internally,
|
|
||||||
// but we can be more relaxed
|
|
||||||
return Optional.of("masterDevicePushTimestamp");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!serializedEquals(primaryAccount, secondaryAccount)) {
|
|
||||||
return Optional.of("serialization");
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (JsonProcessingException e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
|
|
||||||
return Optional.empty();
|
return Optional.empty();
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
|
||||||
private <T> void runSafelyAndRecordMetrics(Callable<T> callable, Optional<UUID> maybeUuid, final T primaryResult,
|
|
||||||
final BiFunction<T, T, Optional<String>> mismatchClassifier, final String action) {
|
|
||||||
|
|
||||||
if (maybeUuid.isPresent()) {
|
|
||||||
// the only time we don’t have a UUID is in getByNumber, which is sufficiently low volume to not be a concern, and
|
|
||||||
// it will also be gated by the global readEnabled configuration
|
|
||||||
final boolean enrolled = experimentEnrollmentManager.isEnrolled(maybeUuid.get(), "accountsDynamoDbMigration");
|
|
||||||
|
|
||||||
if (!enrolled) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
|
|
||||||
final T secondaryResult = callable.call();
|
|
||||||
compare(primaryResult, secondaryResult, mismatchClassifier, action, maybeUuid);
|
|
||||||
|
|
||||||
} catch (final Exception e) {
|
|
||||||
logger.error("Error running " + action + " in Dynamo", e);
|
|
||||||
|
|
||||||
Metrics.counter(DYNAMO_MIGRATION_ERROR_COUNTER_NAME, "action", action).increment();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
|
||||||
private <T> void compare(final T primaryResult, final T secondaryResult,
|
|
||||||
final BiFunction<T, T, Optional<String>> mismatchClassifier, final String action,
|
|
||||||
final Optional<UUID> maybeUUid) {
|
|
||||||
|
|
||||||
DYNAMO_MIGRATION_COMPARISON_COUNTER.increment();
|
|
||||||
|
|
||||||
mismatchClassifier.apply(primaryResult, secondaryResult)
|
|
||||||
.ifPresent(mismatchType -> {
|
|
||||||
final String mismatchDescription = action + ":" + mismatchType;
|
|
||||||
Metrics.counter(DYNAMO_MIGRATION_MISMATCH_COUNTER_NAME,
|
|
||||||
"mismatchType", mismatchDescription)
|
|
||||||
.increment();
|
|
||||||
|
|
||||||
maybeUUid.ifPresent(uuid -> {
|
|
||||||
|
|
||||||
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
|
|
||||||
.isPostCheckMismatches()) {
|
|
||||||
mismatchedAccounts.put(uuid);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getAbbreviatedCallChain(final StackTraceElement[] stackTrace) {
|
private String getAbbreviatedCallChain(final StackTraceElement[] stackTrace) {
|
||||||
return Arrays.stream(stackTrace)
|
return Arrays.stream(stackTrace)
|
||||||
.filter(stackTraceElement -> stackTraceElement.getClassName().contains("org.whispersystems"))
|
.filter(stackTraceElement -> stackTraceElement.getClassName().contains("org.whispersystems"))
|
||||||
|
@ -827,22 +479,4 @@ public class AccountsManager {
|
||||||
.map(stackTraceElement -> StringUtils.substringAfterLast(stackTraceElement.getClassName(), ".") + ":" + stackTraceElement.getMethodName())
|
.map(stackTraceElement -> StringUtils.substringAfterLast(stackTraceElement.getClassName(), ".") + ":" + stackTraceElement.getMethodName())
|
||||||
.collect(Collectors.joining(" -> "));
|
.collect(Collectors.joining(" -> "));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static abstract class DeviceComparisonMixin extends Device {
|
|
||||||
|
|
||||||
@JsonIgnore
|
|
||||||
private long lastSeen;
|
|
||||||
|
|
||||||
@JsonIgnore
|
|
||||||
private long pushTimestamp;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean serializedEquals(final Object primary, final Object secondary) throws JsonProcessingException {
|
|
||||||
final byte[] primarySerialized = migrationComparisonMapper.writeValueAsBytes(primary);
|
|
||||||
final byte[] secondarySerialized = migrationComparisonMapper.writeValueAsBytes(secondary);
|
|
||||||
final int serializeCompare = Arrays.compare(primarySerialized, secondarySerialized);
|
|
||||||
|
|
||||||
return serializeCompare == 0;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,71 +0,0 @@
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
|
|
||||||
|
|
||||||
public class MigrationDeletedAccounts extends AbstractDynamoDbStore {
|
|
||||||
|
|
||||||
private final String tableName;
|
|
||||||
|
|
||||||
static final String KEY_UUID = "U";
|
|
||||||
|
|
||||||
public MigrationDeletedAccounts(DynamoDbClient dynamoDb, String tableName) {
|
|
||||||
super(dynamoDb);
|
|
||||||
this.tableName = tableName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void put(UUID uuid) {
|
|
||||||
db().putItem(PutItemRequest.builder()
|
|
||||||
.tableName(tableName)
|
|
||||||
.item(primaryKey(uuid))
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<UUID> getRecentlyDeletedUuids() {
|
|
||||||
|
|
||||||
final List<UUID> uuids = new ArrayList<>();
|
|
||||||
Optional<ScanResponse> firstPage = db().scanPaginator(ScanRequest.builder()
|
|
||||||
.tableName(tableName)
|
|
||||||
.build()).stream().findAny(); // get the first available response
|
|
||||||
|
|
||||||
if (firstPage.isPresent()) {
|
|
||||||
for (Map<String, AttributeValue> item : firstPage.get().items()) {
|
|
||||||
// only process one page each time. If we have a significant backlog at the end of the migration
|
|
||||||
// we can handle it separately
|
|
||||||
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return uuids;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void delete(List<UUID> uuids) {
|
|
||||||
|
|
||||||
writeInBatches(uuids, (batch) -> {
|
|
||||||
List<WriteRequest> deletes = batch.stream().map((uuid) -> WriteRequest.builder().deleteRequest(DeleteRequest.builder()
|
|
||||||
.key(primaryKey(uuid))
|
|
||||||
.build()).build()).collect(Collectors.toList());
|
|
||||||
|
|
||||||
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
|
|
||||||
return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,111 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2021 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
|
||||||
import java.time.Clock;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.paginators.ScanIterable;
|
|
||||||
|
|
||||||
public class MigrationMismatchedAccounts extends AbstractDynamoDbStore {
|
|
||||||
|
|
||||||
static final String KEY_UUID = "U";
|
|
||||||
static final String ATTR_TIMESTAMP = "T";
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
static final long MISMATCH_CHECK_DELAY_MILLIS = Duration.ofMinutes(1).toMillis();
|
|
||||||
|
|
||||||
private final String tableName;
|
|
||||||
private final Clock clock;
|
|
||||||
|
|
||||||
public void put(UUID uuid) {
|
|
||||||
final Map<String, AttributeValue> item = primaryKey(uuid);
|
|
||||||
item.put("T", AttributeValues.fromLong(clock.millis()));
|
|
||||||
db().putItem(PutItemRequest.builder()
|
|
||||||
.tableName(tableName)
|
|
||||||
.item(item)
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
public MigrationMismatchedAccounts(DynamoDbClient dynamoDb, String tableName) {
|
|
||||||
this(dynamoDb, tableName, Clock.systemUTC());
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
MigrationMismatchedAccounts(DynamoDbClient dynamoDb, String tableName, final Clock clock) {
|
|
||||||
super(dynamoDb);
|
|
||||||
|
|
||||||
this.tableName = tableName;
|
|
||||||
this.clock = clock;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* returns a list of UUIDs stored in the table that have passed {@link #MISMATCH_CHECK_DELAY_MILLIS}
|
|
||||||
*/
|
|
||||||
public List<UUID> getUuids(int max) {
|
|
||||||
|
|
||||||
final List<UUID> uuids = new ArrayList<>();
|
|
||||||
|
|
||||||
final ScanIterable scanPaginator = db().scanPaginator(ScanRequest.builder()
|
|
||||||
.tableName(tableName)
|
|
||||||
.filterExpression("#timestamp <= :timestamp")
|
|
||||||
.expressionAttributeNames(Map.of("#timestamp", ATTR_TIMESTAMP))
|
|
||||||
.expressionAttributeValues(Map.of(":timestamp",
|
|
||||||
AttributeValues.fromLong(clock.millis() - MISMATCH_CHECK_DELAY_MILLIS)))
|
|
||||||
.build());
|
|
||||||
|
|
||||||
for (ScanResponse response : scanPaginator) {
|
|
||||||
|
|
||||||
for (Map<String, AttributeValue> item : response.items()) {
|
|
||||||
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
|
|
||||||
|
|
||||||
if (uuids.size() >= max) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (uuids.size() >= max) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return uuids;
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
|
|
||||||
final HashMap<String, AttributeValue> item = new HashMap<>();
|
|
||||||
item.put(KEY_UUID, AttributeValues.fromUUID(uuid));
|
|
||||||
return item;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void delete(final List<UUID> uuidsToDelete) {
|
|
||||||
|
|
||||||
writeInBatches(uuidsToDelete, (uuids -> {
|
|
||||||
|
|
||||||
final List<WriteRequest> deletes = uuids.stream()
|
|
||||||
.map(uuid -> WriteRequest.builder().deleteRequest(
|
|
||||||
DeleteRequest.builder().key(Map.of(KEY_UUID, AttributeValues.fromUUID(uuid))).build()).build())
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
|
|
||||||
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,112 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2021 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
|
|
||||||
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
|
|
||||||
|
|
||||||
import io.micrometer.core.instrument.Counter;
|
|
||||||
import io.micrometer.core.instrument.Metrics;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.concurrent.ScheduledExecutorService;
|
|
||||||
import net.logstash.logback.argument.StructuredArguments;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
|
||||||
|
|
||||||
public class MigrationMismatchedAccountsTableCrawler extends ManagedPeriodicWork {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(MigrationMismatchedAccountsTableCrawler.class);
|
|
||||||
|
|
||||||
private static final Duration WORKER_TTL = Duration.ofMinutes(2);
|
|
||||||
private static final Duration RUN_INTERVAL = Duration.ofMinutes(1);
|
|
||||||
private static final String ACTIVE_WORKER_KEY = "migration_mismatched_accounts_crawler_cache_active_worker";
|
|
||||||
|
|
||||||
private static final int MAX_BATCH_SIZE = 5_000;
|
|
||||||
|
|
||||||
private static final Counter COMPARISONS_COUNTER = Metrics.counter(
|
|
||||||
name(MigrationMismatchedAccountsTableCrawler.class, "comparisons"));
|
|
||||||
private static final String MISMATCH_COUNTER_NAME = name(MigrationMismatchedAccountsTableCrawler.class, "mismatches");
|
|
||||||
private static final Counter ERRORS_COUNTER = Metrics.counter(
|
|
||||||
name(MigrationMismatchedAccountsTableCrawler.class, "errors"));
|
|
||||||
|
|
||||||
private final MigrationMismatchedAccounts mismatchedAccounts;
|
|
||||||
private final AccountsManager accountsManager;
|
|
||||||
private final Accounts accountsDb;
|
|
||||||
private final AccountsDynamoDb accountsDynamoDb;
|
|
||||||
|
|
||||||
private final DynamicConfigurationManager dynamicConfigurationManager;
|
|
||||||
|
|
||||||
public MigrationMismatchedAccountsTableCrawler(
|
|
||||||
final MigrationMismatchedAccounts mismatchedAccounts,
|
|
||||||
final AccountsManager accountsManager,
|
|
||||||
final Accounts accountsDb,
|
|
||||||
final AccountsDynamoDb accountsDynamoDb,
|
|
||||||
final DynamicConfigurationManager dynamicConfigurationManager,
|
|
||||||
final FaultTolerantRedisCluster cluster,
|
|
||||||
final ScheduledExecutorService executorService) throws IOException {
|
|
||||||
|
|
||||||
super(new ManagedPeriodicWorkLock(ACTIVE_WORKER_KEY, cluster), WORKER_TTL, RUN_INTERVAL, executorService);
|
|
||||||
|
|
||||||
this.mismatchedAccounts = mismatchedAccounts;
|
|
||||||
this.accountsManager = accountsManager;
|
|
||||||
this.accountsDb = accountsDb;
|
|
||||||
this.accountsDynamoDb = accountsDynamoDb;
|
|
||||||
this.dynamicConfigurationManager = dynamicConfigurationManager;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doPeriodicWork() {
|
|
||||||
|
|
||||||
if (!dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
|
|
||||||
.isPostCheckMismatches()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
final List<UUID> uuids = this.mismatchedAccounts.getUuids(MAX_BATCH_SIZE);
|
|
||||||
|
|
||||||
final List<UUID> processedUuids = new ArrayList<>(uuids.size());
|
|
||||||
|
|
||||||
try {
|
|
||||||
for (UUID uuid : uuids) {
|
|
||||||
|
|
||||||
try {
|
|
||||||
|
|
||||||
final Optional<String> result = accountsManager.compareAccounts(accountsDb.get(uuid),
|
|
||||||
accountsDynamoDb.get(uuid));
|
|
||||||
|
|
||||||
COMPARISONS_COUNTER.increment();
|
|
||||||
|
|
||||||
result.ifPresent(mismatchType -> {
|
|
||||||
Metrics.counter(MISMATCH_COUNTER_NAME, "type", mismatchType)
|
|
||||||
.increment();
|
|
||||||
|
|
||||||
if (dynamicConfigurationManager.getConfiguration().getAccountsDynamoDbMigrationConfiguration()
|
|
||||||
.isLogMismatches()) {
|
|
||||||
logger.info("Mismatch: {}", StructuredArguments.entries(Map.of(
|
|
||||||
"type", mismatchType,
|
|
||||||
"uuid", uuid)));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
processedUuids.add(uuid);
|
|
||||||
|
|
||||||
} catch (final Exception e) {
|
|
||||||
ERRORS_COUNTER.increment();
|
|
||||||
logger.warn("Failed to check account mismatch", e);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
this.mismatchedAccounts.delete(processedUuids);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,76 +0,0 @@
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.DeleteRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
|
|
||||||
|
|
||||||
public class MigrationRetryAccounts extends AbstractDynamoDbStore {
|
|
||||||
|
|
||||||
private final String tableName;
|
|
||||||
|
|
||||||
static final String KEY_UUID = "U";
|
|
||||||
|
|
||||||
public MigrationRetryAccounts(DynamoDbClient dynamoDb, String tableName) {
|
|
||||||
super(dynamoDb);
|
|
||||||
|
|
||||||
this.tableName = tableName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void put(UUID uuid) {
|
|
||||||
db().putItem(PutItemRequest.builder()
|
|
||||||
.tableName(tableName)
|
|
||||||
.item(primaryKey(uuid))
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<UUID> getUuids(int max) {
|
|
||||||
|
|
||||||
final List<UUID> uuids = new ArrayList<>();
|
|
||||||
|
|
||||||
for (ScanResponse response : db().scanPaginator(ScanRequest.builder().tableName(tableName).build())) {
|
|
||||||
|
|
||||||
for (Map<String, AttributeValue> item : response.items()) {
|
|
||||||
uuids.add(AttributeValues.getUUID(item, KEY_UUID, null));
|
|
||||||
|
|
||||||
if (uuids.size() >= max) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (uuids.size() >= max) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return uuids;
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
public static Map<String, AttributeValue> primaryKey(UUID uuid) {
|
|
||||||
return Map.of(KEY_UUID, AttributeValues.fromUUID(uuid));
|
|
||||||
}
|
|
||||||
|
|
||||||
public void delete(final List<UUID> uuidsToDelete) {
|
|
||||||
|
|
||||||
writeInBatches(uuidsToDelete, (uuids -> {
|
|
||||||
|
|
||||||
final List<WriteRequest> deletes = uuids.stream()
|
|
||||||
.map(uuid -> WriteRequest.builder().deleteRequest(
|
|
||||||
DeleteRequest.builder().key(Map.of(KEY_UUID, AttributeValues.fromUUID(uuid))).build()).build())
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
|
|
||||||
executeTableWriteItemsUntilComplete(Map.of(tableName, deletes));
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,88 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2013-2021 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import static com.codahale.metrics.MetricRegistry.name;
|
|
||||||
|
|
||||||
import io.micrometer.core.instrument.Counter;
|
|
||||||
import io.micrometer.core.instrument.Metrics;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.time.Duration;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.concurrent.ScheduledExecutorService;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
|
||||||
|
|
||||||
public class MigrationRetryAccountsTableCrawler extends ManagedPeriodicWork {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(MigrationRetryAccountsTableCrawler.class);
|
|
||||||
|
|
||||||
private static final Duration WORKER_TTL = Duration.ofMinutes(2);
|
|
||||||
private static final Duration RUN_INTERVAL = Duration.ofMinutes(15);
|
|
||||||
private static final String ACTIVE_WORKER_KEY = "migration_retry_accounts_crawler_cache_active_worker";
|
|
||||||
|
|
||||||
private static final int MAX_BATCH_SIZE = 5_000;
|
|
||||||
|
|
||||||
private static final Counter MIGRATED_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "migrated"));
|
|
||||||
private static final Counter ERROR_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "error"));
|
|
||||||
private static final Counter TOTAL_COUNTER = Metrics.counter(name(MigrationRetryAccountsTableCrawler.class, "total"));
|
|
||||||
|
|
||||||
private final MigrationRetryAccounts retryAccounts;
|
|
||||||
private final AccountsManager accountsManager;
|
|
||||||
private final AccountsDynamoDb accountsDynamoDb;
|
|
||||||
|
|
||||||
public MigrationRetryAccountsTableCrawler(
|
|
||||||
final MigrationRetryAccounts retryAccounts,
|
|
||||||
final AccountsManager accountsManager,
|
|
||||||
final AccountsDynamoDb accountsDynamoDb,
|
|
||||||
final FaultTolerantRedisCluster cluster,
|
|
||||||
final ScheduledExecutorService executorService) throws IOException {
|
|
||||||
|
|
||||||
super(new ManagedPeriodicWorkLock(ACTIVE_WORKER_KEY, cluster), WORKER_TTL, RUN_INTERVAL, executorService);
|
|
||||||
|
|
||||||
this.retryAccounts = retryAccounts;
|
|
||||||
this.accountsManager = accountsManager;
|
|
||||||
this.accountsDynamoDb = accountsDynamoDb;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doPeriodicWork() {
|
|
||||||
|
|
||||||
final List<UUID> uuids = this.retryAccounts.getUuids(MAX_BATCH_SIZE);
|
|
||||||
|
|
||||||
final List<UUID> processedUuids = new ArrayList<>(uuids.size());
|
|
||||||
|
|
||||||
try {
|
|
||||||
for (UUID uuid : uuids) {
|
|
||||||
|
|
||||||
try {
|
|
||||||
final Optional<Account> maybeDynamoAccount = accountsDynamoDb.get(uuid);
|
|
||||||
|
|
||||||
if (maybeDynamoAccount.isEmpty()) {
|
|
||||||
accountsManager.get(uuid).ifPresent(account -> {
|
|
||||||
accountsDynamoDb.migrate(account);
|
|
||||||
MIGRATED_COUNTER.increment();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
processedUuids.add(uuid);
|
|
||||||
|
|
||||||
TOTAL_COUNTER.increment();
|
|
||||||
|
|
||||||
} catch (final Exception e) {
|
|
||||||
ERROR_COUNTER.increment();
|
|
||||||
logger.warn("Failed to migrate account");
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
this.retryAccounts.delete(processedUuids);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -5,18 +5,16 @@
|
||||||
|
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
package org.whispersystems.textsecuregcm.storage;
|
||||||
|
|
||||||
|
import static com.codahale.metrics.MetricRegistry.name;
|
||||||
|
|
||||||
import com.codahale.metrics.MetricRegistry;
|
import com.codahale.metrics.MetricRegistry;
|
||||||
import com.codahale.metrics.SharedMetricRegistries;
|
import com.codahale.metrics.SharedMetricRegistries;
|
||||||
import com.codahale.metrics.Timer;
|
import com.codahale.metrics.Timer;
|
||||||
import org.jdbi.v3.core.JdbiException;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper;
|
|
||||||
import org.whispersystems.textsecuregcm.util.Constants;
|
|
||||||
|
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
import org.jdbi.v3.core.JdbiException;
|
||||||
import static com.codahale.metrics.MetricRegistry.name;
|
import org.whispersystems.textsecuregcm.util.Constants;
|
||||||
|
|
||||||
public class Usernames {
|
public class Usernames {
|
||||||
|
|
||||||
|
@ -34,7 +32,6 @@ public class Usernames {
|
||||||
|
|
||||||
public Usernames(FaultTolerantDatabase database) {
|
public Usernames(FaultTolerantDatabase database) {
|
||||||
this.database = database;
|
this.database = database;
|
||||||
this.database.getDatabase().registerRowMapper(new AccountRowMapper());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean put(UUID uuid, String username) {
|
public boolean put(UUID uuid, String username) {
|
||||||
|
|
|
@ -1,36 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2013-2020 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.whispersystems.textsecuregcm.storage.mappers;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import org.jdbi.v3.core.mapper.RowMapper;
|
|
||||||
import org.jdbi.v3.core.statement.StatementContext;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Account;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
|
||||||
import org.whispersystems.textsecuregcm.util.SystemMapper;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.sql.ResultSet;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.UUID;
|
|
||||||
|
|
||||||
public class AccountRowMapper implements RowMapper<Account> {
|
|
||||||
|
|
||||||
private static ObjectMapper mapper = SystemMapper.getMapper();
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Account map(ResultSet resultSet, StatementContext ctx) throws SQLException {
|
|
||||||
try {
|
|
||||||
Account account = mapper.readValue(resultSet.getString(Accounts.DATA), Account.class);
|
|
||||||
account.setNumber(resultSet.getString(Accounts.NUMBER));
|
|
||||||
account.setUuid(UUID.fromString(resultSet.getString(Accounts.UID)));
|
|
||||||
account.setVersion(resultSet.getInt(Accounts.VERSION));
|
|
||||||
return account;
|
|
||||||
} catch (IOException e) {
|
|
||||||
throw new SQLException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -20,9 +20,6 @@ import io.lettuce.core.resource.ClientResources;
|
||||||
import io.micrometer.core.instrument.Metrics;
|
import io.micrometer.core.instrument.Metrics;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.LinkedBlockingDeque;
|
|
||||||
import java.util.concurrent.ThreadPoolExecutor;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
import net.sourceforge.argparse4j.inf.Namespace;
|
import net.sourceforge.argparse4j.inf.Namespace;
|
||||||
import net.sourceforge.argparse4j.inf.Subparser;
|
import net.sourceforge.argparse4j.inf.Subparser;
|
||||||
import org.jdbi.v3.core.Jdbi;
|
import org.jdbi.v3.core.Jdbi;
|
||||||
|
@ -30,14 +27,12 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
|
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
|
||||||
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
|
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
|
||||||
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
|
|
||||||
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
|
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
|
||||||
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
||||||
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
||||||
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
||||||
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
||||||
import org.whispersystems.textsecuregcm.storage.Account;
|
import org.whispersystems.textsecuregcm.storage.Account;
|
||||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsManager.DeletionReason;
|
import org.whispersystems.textsecuregcm.storage.AccountsManager.DeletionReason;
|
||||||
|
@ -49,9 +44,6 @@ import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesCache;
|
import org.whispersystems.textsecuregcm.storage.MessagesCache;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Profiles;
|
import org.whispersystems.textsecuregcm.storage.Profiles;
|
||||||
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb;
|
||||||
|
@ -62,7 +54,6 @@ import org.whispersystems.textsecuregcm.storage.Usernames;
|
||||||
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
|
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
|
||||||
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
|
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||||
|
|
||||||
public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfiguration> {
|
public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfiguration> {
|
||||||
|
@ -70,11 +61,9 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
|
||||||
private final Logger logger = LoggerFactory.getLogger(DeleteUserCommand.class);
|
private final Logger logger = LoggerFactory.getLogger(DeleteUserCommand.class);
|
||||||
|
|
||||||
public DeleteUserCommand() {
|
public DeleteUserCommand() {
|
||||||
super(new Application<WhisperServerConfiguration>() {
|
super(new Application<>() {
|
||||||
@Override
|
@Override
|
||||||
public void run(WhisperServerConfiguration configuration, Environment environment)
|
public void run(WhisperServerConfiguration configuration, Environment environment) {
|
||||||
throws Exception
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}, "rmuser", "remove user");
|
}, "rmuser", "remove user");
|
||||||
|
@ -105,9 +94,6 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
|
||||||
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_delete_user", accountJdbi, configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
|
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_delete_user", accountJdbi, configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
|
||||||
ClientResources redisClusterClientResources = ClientResources.builder().build();
|
ClientResources redisClusterClientResources = ClientResources.builder().build();
|
||||||
|
|
||||||
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
|
|
||||||
new LinkedBlockingDeque<>());
|
|
||||||
|
|
||||||
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig.client(
|
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig.client(
|
||||||
configuration.getReportMessageDynamoDbConfiguration(),
|
configuration.getReportMessageDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
@ -118,16 +104,9 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
|
||||||
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(
|
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(
|
||||||
configuration.getAccountsDynamoDbConfiguration(),
|
configuration.getAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(
|
|
||||||
configuration.getAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
|
|
||||||
accountsDynamoDbMigrationThreadPool);
|
|
||||||
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(
|
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(
|
||||||
configuration.getDeletedAccountsDynamoDbConfiguration(),
|
configuration.getDeletedAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
DynamoDbClient migrationMismatchedAccountsDynamoDb = DynamoDbFromConfig.client(
|
|
||||||
configuration.getMigrationMismatchedAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
|
|
||||||
FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster",
|
FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster",
|
||||||
configuration.getCacheClusterConfiguration(), redisClusterClientResources);
|
configuration.getCacheClusterConfiguration(), redisClusterClientResources);
|
||||||
|
@ -151,39 +130,41 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
|
||||||
configuration.getAppConfig().getConfigurationName());
|
configuration.getAppConfig().getConfigurationName());
|
||||||
dynamicConfigurationManager.start();
|
dynamicConfigurationManager.start();
|
||||||
|
|
||||||
ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(
|
|
||||||
dynamicConfigurationManager);
|
|
||||||
|
|
||||||
DynamoDbClient migrationDeletedAccountsDynamoDb = DynamoDbFromConfig.client(
|
|
||||||
configuration.getMigrationDeletedAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig.client(
|
|
||||||
configuration.getMigrationRetryAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig.client(configuration.getPendingAccountsDynamoDbConfiguration(),
|
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig.client(configuration.getPendingAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
|
||||||
AmazonDynamoDB deletedAccountsLockDynamoDbClient = AmazonDynamoDBClientBuilder.standard()
|
AmazonDynamoDB deletedAccountsLockDynamoDbClient = AmazonDynamoDBClientBuilder.standard()
|
||||||
.withRegion(configuration.getDeletedAccountsLockDynamoDbConfiguration().getRegion())
|
.withRegion(configuration.getDeletedAccountsLockDynamoDbConfiguration().getRegion())
|
||||||
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(((int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientExecutionTimeout().toMillis()))
|
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(
|
||||||
.withRequestTimeout((int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientRequestTimeout().toMillis()))
|
((int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientExecutionTimeout()
|
||||||
|
.toMillis()))
|
||||||
|
.withRequestTimeout(
|
||||||
|
(int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientRequestTimeout()
|
||||||
|
.toMillis()))
|
||||||
.withCredentials(InstanceProfileCredentialsProvider.getInstance())
|
.withCredentials(InstanceProfileCredentialsProvider.getInstance())
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(), configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
|
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
|
||||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(migrationDeletedAccountsDynamoDb, configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
|
configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(),
|
||||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb, configuration.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
|
configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
|
||||||
VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient, configuration.getPendingAccountsDynamoDbConfiguration().getTableName());
|
VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient,
|
||||||
|
configuration.getPendingAccountsDynamoDbConfiguration().getTableName());
|
||||||
|
|
||||||
Accounts accounts = new Accounts(accountDatabase);
|
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient,
|
||||||
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient, accountsDynamoDbMigrationThreadPool, configuration.getAccountsDynamoDbConfiguration().getTableName(), configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts, migrationRetryAccounts);
|
configuration.getAccountsDynamoDbConfiguration().getTableName(),
|
||||||
Usernames usernames = new Usernames(accountDatabase);
|
configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName());
|
||||||
Profiles profiles = new Profiles(accountDatabase);
|
Usernames usernames = new Usernames(accountDatabase);
|
||||||
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
|
Profiles profiles = new Profiles(accountDatabase);
|
||||||
KeysDynamoDb keysDynamoDb = new KeysDynamoDb(preKeysDynamoDb, configuration.getKeysDynamoDbConfiguration().getTableName());
|
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
|
||||||
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb, configuration.getMessageDynamoDbConfiguration().getTableName(), configuration.getMessageDynamoDbConfiguration().getTimeToLive());
|
KeysDynamoDb keysDynamoDb = new KeysDynamoDb(preKeysDynamoDb,
|
||||||
FaultTolerantRedisCluster messageInsertCacheCluster = new FaultTolerantRedisCluster("message_insert_cluster", configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
|
configuration.getKeysDynamoDbConfiguration().getTableName());
|
||||||
FaultTolerantRedisCluster messageReadDeleteCluster = new FaultTolerantRedisCluster("message_read_delete_cluster", configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
|
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb,
|
||||||
|
configuration.getMessageDynamoDbConfiguration().getTableName(),
|
||||||
|
configuration.getMessageDynamoDbConfiguration().getTimeToLive());
|
||||||
|
FaultTolerantRedisCluster messageInsertCacheCluster = new FaultTolerantRedisCluster("message_insert_cluster",
|
||||||
|
configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
|
||||||
|
FaultTolerantRedisCluster messageReadDeleteCluster = new FaultTolerantRedisCluster("message_read_delete_cluster",
|
||||||
|
configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
|
||||||
FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster",
|
FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster",
|
||||||
configuration.getMetricsClusterConfiguration(), redisClusterClientResources);
|
configuration.getMetricsClusterConfiguration(), redisClusterClientResources);
|
||||||
SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor,
|
SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor,
|
||||||
|
@ -203,16 +184,13 @@ public class DeleteUserCommand extends EnvironmentCommand<WhisperServerConfigura
|
||||||
Metrics.globalRegistry);
|
Metrics.globalRegistry);
|
||||||
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager,
|
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager,
|
||||||
reportMessageManager);
|
reportMessageManager);
|
||||||
MigrationMismatchedAccounts mismatchedAccounts = new MigrationMismatchedAccounts(
|
|
||||||
migrationMismatchedAccountsDynamoDb,
|
|
||||||
configuration.getMigrationMismatchedAccountsDynamoDbConfiguration().getTableName());
|
|
||||||
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
|
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
|
||||||
deletedAccountsLockDynamoDbClient,
|
deletedAccountsLockDynamoDbClient,
|
||||||
configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
|
configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
|
||||||
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
|
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
|
||||||
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster,
|
AccountsManager accountsManager = new AccountsManager(accountsDynamoDb, cacheCluster,
|
||||||
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, mismatchedAccounts, usernamesManager,
|
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager,
|
||||||
profilesManager, pendingAccountsManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager,
|
profilesManager, pendingAccountsManager, secureStorageClient, secureBackupClient,
|
||||||
dynamicConfigurationManager);
|
dynamicConfigurationManager);
|
||||||
|
|
||||||
for (String user : users) {
|
for (String user : users) {
|
||||||
|
|
|
@ -21,22 +21,17 @@ import io.micrometer.core.instrument.Metrics;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.LinkedBlockingDeque;
|
|
||||||
import java.util.concurrent.ThreadPoolExecutor;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
import net.sourceforge.argparse4j.inf.Namespace;
|
import net.sourceforge.argparse4j.inf.Namespace;
|
||||||
import net.sourceforge.argparse4j.inf.Subparser;
|
import net.sourceforge.argparse4j.inf.Subparser;
|
||||||
import org.jdbi.v3.core.Jdbi;
|
import org.jdbi.v3.core.Jdbi;
|
||||||
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
|
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
|
||||||
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
|
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
|
||||||
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
|
|
||||||
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
|
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
|
||||||
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
|
||||||
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
||||||
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
||||||
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
||||||
import org.whispersystems.textsecuregcm.storage.Account;
|
import org.whispersystems.textsecuregcm.storage.Account;
|
||||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
|
import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
|
||||||
|
@ -47,9 +42,6 @@ import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesCache;
|
import org.whispersystems.textsecuregcm.storage.MessagesCache;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Profiles;
|
import org.whispersystems.textsecuregcm.storage.Profiles;
|
||||||
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb;
|
||||||
|
@ -60,7 +52,6 @@ import org.whispersystems.textsecuregcm.storage.Usernames;
|
||||||
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
|
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
|
||||||
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
|
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||||
|
|
||||||
public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperServerConfiguration> {
|
public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperServerConfiguration> {
|
||||||
|
@ -105,9 +96,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
|
||||||
configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
|
configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
|
||||||
ClientResources redisClusterClientResources = ClientResources.builder().build();
|
ClientResources redisClusterClientResources = ClientResources.builder().build();
|
||||||
|
|
||||||
ThreadPoolExecutor accountsDynamoDbMigrationThreadPool = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS,
|
|
||||||
new LinkedBlockingDeque<>());
|
|
||||||
|
|
||||||
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig
|
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig
|
||||||
.client(configuration.getReportMessageDynamoDbConfiguration(),
|
.client(configuration.getReportMessageDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
@ -118,10 +106,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
|
||||||
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig
|
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig
|
||||||
.client(configuration.getAccountsDynamoDbConfiguration(),
|
.client(configuration.getAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
DynamoDbAsyncClient accountsDynamoDbAsyncClient = DynamoDbFromConfig
|
|
||||||
.asyncClient(configuration.getAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create(),
|
|
||||||
accountsDynamoDbMigrationThreadPool);
|
|
||||||
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig
|
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig
|
||||||
.client(configuration.getDeletedAccountsDynamoDbConfiguration(),
|
.client(configuration.getDeletedAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
@ -148,18 +132,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
|
||||||
configuration.getAppConfig().getConfigurationName());
|
configuration.getAppConfig().getConfigurationName());
|
||||||
dynamicConfigurationManager.start();
|
dynamicConfigurationManager.start();
|
||||||
|
|
||||||
ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(
|
|
||||||
dynamicConfigurationManager);
|
|
||||||
|
|
||||||
DynamoDbClient migrationDeletedAccountsDynamoDb = DynamoDbFromConfig
|
|
||||||
.client(configuration.getMigrationDeletedAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
DynamoDbClient migrationMismatchedAccountsDynamoDb = DynamoDbFromConfig
|
|
||||||
.client(configuration.getMigrationMismatchedAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
DynamoDbClient migrationRetryAccountsDynamoDb = DynamoDbFromConfig
|
|
||||||
.client(configuration.getMigrationRetryAccountsDynamoDbConfiguration(),
|
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
|
||||||
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig
|
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig
|
||||||
.client(configuration.getPendingAccountsDynamoDbConfiguration(),
|
.client(configuration.getPendingAccountsDynamoDbConfiguration(),
|
||||||
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
|
||||||
|
@ -178,18 +150,13 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
|
||||||
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
|
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
|
||||||
configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(),
|
configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(),
|
||||||
configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
|
configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
|
||||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(migrationDeletedAccountsDynamoDb,
|
|
||||||
configuration.getMigrationDeletedAccountsDynamoDbConfiguration().getTableName());
|
|
||||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(migrationRetryAccountsDynamoDb,
|
|
||||||
configuration.getMigrationRetryAccountsDynamoDbConfiguration().getTableName());
|
|
||||||
VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient,
|
VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient,
|
||||||
configuration.getPendingAccountsDynamoDbConfiguration().getTableName());
|
configuration.getPendingAccountsDynamoDbConfiguration().getTableName());
|
||||||
|
|
||||||
Accounts accounts = new Accounts(accountDatabase);
|
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient,
|
||||||
AccountsDynamoDb accountsDynamoDb = new AccountsDynamoDb(accountsDynamoDbClient, accountsDynamoDbAsyncClient,
|
configuration.getAccountsDynamoDbConfiguration().getTableName(),
|
||||||
accountsDynamoDbMigrationThreadPool, configuration.getAccountsDynamoDbConfiguration().getTableName(),
|
configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName()
|
||||||
configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), migrationDeletedAccounts,
|
);
|
||||||
migrationRetryAccounts);
|
|
||||||
Usernames usernames = new Usernames(accountDatabase);
|
Usernames usernames = new Usernames(accountDatabase);
|
||||||
Profiles profiles = new Profiles(accountDatabase);
|
Profiles profiles = new Profiles(accountDatabase);
|
||||||
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
|
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
|
||||||
|
@ -221,17 +188,14 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand<WhisperSer
|
||||||
Metrics.globalRegistry);
|
Metrics.globalRegistry);
|
||||||
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager,
|
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager,
|
||||||
reportMessageManager);
|
reportMessageManager);
|
||||||
MigrationMismatchedAccounts mismatchedAccounts = new MigrationMismatchedAccounts(
|
|
||||||
migrationMismatchedAccountsDynamoDb,
|
|
||||||
configuration.getMigrationMismatchedAccountsDynamoDbConfiguration().getTableName());
|
|
||||||
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
|
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
|
||||||
deletedAccountsLockDynamoDbClient,
|
deletedAccountsLockDynamoDbClient,
|
||||||
configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
|
configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
|
||||||
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
|
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
|
||||||
AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster,
|
AccountsManager accountsManager = new AccountsManager(accountsDynamoDb, cacheCluster,
|
||||||
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, mismatchedAccounts, usernamesManager,
|
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager,
|
||||||
profilesManager,
|
profilesManager,
|
||||||
pendingAccountsManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager,
|
pendingAccountsManager, secureStorageClient, secureBackupClient,
|
||||||
dynamicConfigurationManager);
|
dynamicConfigurationManager);
|
||||||
|
|
||||||
Optional<Account> maybeAccount;
|
Optional<Account> maybeAccount;
|
||||||
|
|
|
@ -1,47 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2013-2020 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.whispersystems.textsecuregcm.workers;
|
|
||||||
|
|
||||||
import net.sourceforge.argparse4j.inf.Namespace;
|
|
||||||
import org.jdbi.v3.core.Jdbi;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
|
|
||||||
|
|
||||||
import io.dropwizard.cli.ConfiguredCommand;
|
|
||||||
import io.dropwizard.setup.Bootstrap;
|
|
||||||
|
|
||||||
|
|
||||||
public class VacuumCommand extends ConfiguredCommand<WhisperServerConfiguration> {
|
|
||||||
|
|
||||||
private final Logger logger = LoggerFactory.getLogger(VacuumCommand.class);
|
|
||||||
|
|
||||||
public VacuumCommand() {
|
|
||||||
super("vacuum", "Vacuum Postgres Tables");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void run(Bootstrap<WhisperServerConfiguration> bootstrap,
|
|
||||||
Namespace namespace,
|
|
||||||
WhisperServerConfiguration config)
|
|
||||||
throws Exception
|
|
||||||
{
|
|
||||||
DatabaseConfiguration accountDbConfig = config.getAbuseDatabaseConfiguration();
|
|
||||||
Jdbi accountJdbi = Jdbi.create(accountDbConfig.getUrl(), accountDbConfig.getUser(), accountDbConfig.getPassword());
|
|
||||||
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_vacuum", accountJdbi, accountDbConfig.getCircuitBreakerConfiguration());
|
|
||||||
|
|
||||||
Accounts accounts = new Accounts(accountDatabase);
|
|
||||||
|
|
||||||
logger.info("Vacuuming accounts...");
|
|
||||||
accounts.vacuum();
|
|
||||||
|
|
||||||
Thread.sleep(3000);
|
|
||||||
System.exit(0);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -320,30 +320,19 @@ class DynamicConfigurationTest {
|
||||||
final DynamicConfiguration emptyConfig =
|
final DynamicConfiguration emptyConfig =
|
||||||
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
DynamicConfigurationManager.parseConfiguration(emptyConfigYaml).orElseThrow();
|
||||||
|
|
||||||
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isBackgroundMigrationEnabled());
|
assertEquals(10, emptyConfig.getAccountsDynamoDbMigrationConfiguration().getDynamoCrawlerScanPageSize());
|
||||||
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isDeleteEnabled());
|
|
||||||
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isWriteEnabled());
|
|
||||||
assertFalse(emptyConfig.getAccountsDynamoDbMigrationConfiguration().isReadEnabled());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
final String accountsDynamoDbMigrationConfig =
|
final String accountsDynamoDbMigrationConfig =
|
||||||
"accountsDynamoDbMigration:\n"
|
"accountsDynamoDbMigration:\n"
|
||||||
+ " backgroundMigrationEnabled: true\n"
|
+ " dynamoCrawlerScanPageSize: 5000";
|
||||||
+ " backgroundMigrationExecutorThreads: 100\n"
|
|
||||||
+ " deleteEnabled: true\n"
|
|
||||||
+ " readEnabled: true\n"
|
|
||||||
+ " writeEnabled: true";
|
|
||||||
|
|
||||||
final DynamicAccountsDynamoDbMigrationConfiguration config =
|
final DynamicAccountsDynamoDbMigrationConfiguration config =
|
||||||
DynamicConfigurationManager.parseConfiguration(accountsDynamoDbMigrationConfig).orElseThrow()
|
DynamicConfigurationManager.parseConfiguration(accountsDynamoDbMigrationConfig).orElseThrow()
|
||||||
.getAccountsDynamoDbMigrationConfiguration();
|
.getAccountsDynamoDbMigrationConfiguration();
|
||||||
|
|
||||||
assertTrue(config.isBackgroundMigrationEnabled());
|
assertEquals(5000, config.getDynamoCrawlerScanPageSize());
|
||||||
assertEquals(100, config.getBackgroundMigrationExecutorThreads());
|
|
||||||
assertTrue(config.isDeleteEnabled());
|
|
||||||
assertTrue(config.isWriteEnabled());
|
|
||||||
assertTrue(config.isReadEnabled());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -18,11 +18,8 @@ import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
|
import org.whispersystems.textsecuregcm.redis.AbstractRedisClusterTest;
|
||||||
|
|
||||||
public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterTest {
|
public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterTest {
|
||||||
|
@ -58,18 +55,15 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT
|
||||||
when(firstAccount.getUuid()).thenReturn(FIRST_UUID);
|
when(firstAccount.getUuid()).thenReturn(FIRST_UUID);
|
||||||
when(secondAccount.getUuid()).thenReturn(SECOND_UUID);
|
when(secondAccount.getUuid()).thenReturn(SECOND_UUID);
|
||||||
|
|
||||||
when(accountsManager.getAllFrom(CHUNK_SIZE)).thenReturn(new AccountCrawlChunk(List.of(firstAccount), FIRST_UUID));
|
when(accountsManager.getAllFromDynamo(CHUNK_SIZE)).thenReturn(
|
||||||
when(accountsManager.getAllFrom(any(UUID.class), eq(CHUNK_SIZE)))
|
new AccountCrawlChunk(List.of(firstAccount), FIRST_UUID));
|
||||||
|
when(accountsManager.getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE)))
|
||||||
.thenReturn(new AccountCrawlChunk(List.of(secondAccount), SECOND_UUID))
|
.thenReturn(new AccountCrawlChunk(List.of(secondAccount), SECOND_UUID))
|
||||||
.thenReturn(new AccountCrawlChunk(Collections.emptyList(), null));
|
.thenReturn(new AccountCrawlChunk(Collections.emptyList(), null));
|
||||||
|
|
||||||
final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
|
|
||||||
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
|
|
||||||
when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn(mock(DynamicAccountsDynamoDbMigrationConfiguration.class));
|
|
||||||
|
|
||||||
final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache(getRedisCluster());
|
final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache(getRedisCluster());
|
||||||
accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, crawlerCache, List.of(listener), CHUNK_SIZE,
|
accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, crawlerCache, List.of(listener), CHUNK_SIZE,
|
||||||
CHUNK_INTERVAL_MS, mock(ExecutorService.class), dynamicConfigurationManager);
|
CHUNK_INTERVAL_MS);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -78,9 +72,9 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT
|
||||||
assertFalse(accountDatabaseCrawler.doPeriodicWork());
|
assertFalse(accountDatabaseCrawler.doPeriodicWork());
|
||||||
assertFalse(accountDatabaseCrawler.doPeriodicWork());
|
assertFalse(accountDatabaseCrawler.doPeriodicWork());
|
||||||
|
|
||||||
verify(accountsManager).getAllFrom(CHUNK_SIZE);
|
verify(accountsManager).getAllFromDynamo(CHUNK_SIZE);
|
||||||
verify(accountsManager).getAllFrom(FIRST_UUID, CHUNK_SIZE);
|
verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE);
|
||||||
verify(accountsManager).getAllFrom(SECOND_UUID, CHUNK_SIZE);
|
verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE);
|
||||||
|
|
||||||
verify(listener).onCrawlStart();
|
verify(listener).onCrawlStart();
|
||||||
verify(listener).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));
|
verify(listener).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));
|
||||||
|
@ -98,9 +92,9 @@ public class AccountDatabaseCrawlerIntegrationTest extends AbstractRedisClusterT
|
||||||
assertFalse(accountDatabaseCrawler.doPeriodicWork());
|
assertFalse(accountDatabaseCrawler.doPeriodicWork());
|
||||||
assertFalse(accountDatabaseCrawler.doPeriodicWork());
|
assertFalse(accountDatabaseCrawler.doPeriodicWork());
|
||||||
|
|
||||||
verify(accountsManager, times(2)).getAllFrom(CHUNK_SIZE);
|
verify(accountsManager, times(2)).getAllFromDynamo(CHUNK_SIZE);
|
||||||
verify(accountsManager).getAllFrom(FIRST_UUID, CHUNK_SIZE);
|
verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE);
|
||||||
verify(accountsManager).getAllFrom(SECOND_UUID, CHUNK_SIZE);
|
verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE);
|
||||||
|
|
||||||
verify(listener, times(2)).onCrawlStart();
|
verify(listener, times(2)).onCrawlStart();
|
||||||
verify(listener, times(2)).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));
|
verify(listener, times(2)).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount));
|
||||||
|
|
|
@ -1,271 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2021 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import com.opentable.db.postgres.embedded.LiquibasePreparer;
|
|
||||||
import com.opentable.db.postgres.junit5.EmbeddedPostgresExtension;
|
|
||||||
import com.opentable.db.postgres.junit5.PreparedDbExtension;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
import java.util.concurrent.LinkedBlockingDeque;
|
|
||||||
import java.util.concurrent.ThreadPoolExecutor;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
import org.jdbi.v3.core.Jdbi;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
|
|
||||||
import org.whispersystems.textsecuregcm.limits.RateLimiter;
|
|
||||||
import org.whispersystems.textsecuregcm.limits.RateLimiters;
|
|
||||||
import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
|
|
||||||
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
|
||||||
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
|
||||||
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
|
||||||
import org.whispersystems.textsecuregcm.tests.util.SynchronousExecutorService;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.KeyType;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.Projection;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ProjectionType;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
|
||||||
|
|
||||||
class AccountsDynamoDbMigrationCrawlerIntegrationTest {
|
|
||||||
|
|
||||||
private static final int CHUNK_SIZE = 20;
|
|
||||||
private static final long CHUNK_INTERVAL_MS = 0;
|
|
||||||
|
|
||||||
private static final String ACCOUNTS_TABLE_NAME = "accounts_test";
|
|
||||||
private static final String KEYS_TABLE_NAME = "keys_test";
|
|
||||||
private static final String MIGRATION_DELETED_ACCOUNTS_TABLE_NAME = "migration_deleted_accounts_test";
|
|
||||||
private static final String MIGRATION_RETRY_ACCOUNTS_TABLE_NAME = "migration_retry_accounts_test";
|
|
||||||
private static final String NUMBERS_TABLE_NAME = "numbers_test";
|
|
||||||
private static final String VERIFICATION_CODE_TABLE_NAME = "verification_code_test";
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static final RedisClusterExtension REDIS_CLUSTER_EXTENSION = RedisClusterExtension.builder().build();
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static final DynamoDbExtension KEYS_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
|
|
||||||
.tableName(KEYS_TABLE_NAME)
|
|
||||||
.hashKey("U")
|
|
||||||
.rangeKey("DK")
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName("U")
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName("DK")
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static final DynamoDbExtension VERIFICATION_CODE_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
|
|
||||||
.tableName(VERIFICATION_CODE_TABLE_NAME)
|
|
||||||
.hashKey("P")
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName("P")
|
|
||||||
.attributeType(ScalarAttributeType.S)
|
|
||||||
.build())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static PreparedDbExtension db = EmbeddedPostgresExtension
|
|
||||||
.preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml"));
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static DynamoDbExtension ACCOUNTS_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
|
|
||||||
.tableName(ACCOUNTS_TABLE_NAME)
|
|
||||||
.hashKey("U")
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName("U")
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
private static final String NEEDS_RECONCILIATION_INDEX_NAME = "needs_reconciliation_test";
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static final DynamoDbExtension DELETED_ACCOUNTS_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
|
|
||||||
.tableName("deleted_accounts_test")
|
|
||||||
.hashKey(DeletedAccounts.KEY_ACCOUNT_E164)
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName(DeletedAccounts.KEY_ACCOUNT_E164)
|
|
||||||
.attributeType(ScalarAttributeType.S).build())
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName(DeletedAccounts.ATTR_NEEDS_CDS_RECONCILIATION)
|
|
||||||
.attributeType(ScalarAttributeType.N)
|
|
||||||
.build())
|
|
||||||
.globalSecondaryIndex(GlobalSecondaryIndex.builder()
|
|
||||||
.indexName(NEEDS_RECONCILIATION_INDEX_NAME)
|
|
||||||
.keySchema(
|
|
||||||
KeySchemaElement.builder().attributeName(DeletedAccounts.KEY_ACCOUNT_E164).keyType(KeyType.HASH).build(),
|
|
||||||
KeySchemaElement.builder().attributeName(DeletedAccounts.ATTR_NEEDS_CDS_RECONCILIATION)
|
|
||||||
.keyType(KeyType.RANGE).build())
|
|
||||||
.projection(Projection.builder().projectionType(ProjectionType.INCLUDE)
|
|
||||||
.nonKeyAttributes(DeletedAccounts.ATTR_ACCOUNT_UUID).build())
|
|
||||||
.provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(10L).writeCapacityUnits(10L).build())
|
|
||||||
.build())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static DynamoDbExtension DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION = DynamoDbExtension.builder()
|
|
||||||
.tableName("deleted_accounts_lock_test")
|
|
||||||
.hashKey(DeletedAccounts.KEY_ACCOUNT_E164)
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName(DeletedAccounts.KEY_ACCOUNT_E164)
|
|
||||||
.attributeType(ScalarAttributeType.S).build())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
private DynamicAccountsDynamoDbMigrationConfiguration accountMigrationConfiguration;
|
|
||||||
|
|
||||||
private AccountsManager accountsManager;
|
|
||||||
private AccountDatabaseCrawler accountDatabaseCrawler;
|
|
||||||
private Accounts accounts;
|
|
||||||
private AccountsDynamoDb accountsDynamoDb;
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() throws Exception {
|
|
||||||
|
|
||||||
createAdditionalDynamoDbTables();
|
|
||||||
|
|
||||||
final DeletedAccounts deletedAccounts = new DeletedAccounts(DELETED_ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(),
|
|
||||||
DELETED_ACCOUNTS_DYNAMODB_EXTENSION.getTableName(),
|
|
||||||
NEEDS_RECONCILIATION_INDEX_NAME);
|
|
||||||
|
|
||||||
final DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
|
|
||||||
DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION.getLegacyDynamoClient(),
|
|
||||||
DELETED_ACCOUNTS_LOCK_DYNAMODB_EXTENSION.getTableName());
|
|
||||||
|
|
||||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(
|
|
||||||
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(), MIGRATION_DELETED_ACCOUNTS_TABLE_NAME);
|
|
||||||
|
|
||||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(
|
|
||||||
(ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient()),
|
|
||||||
MIGRATION_RETRY_ACCOUNTS_TABLE_NAME);
|
|
||||||
|
|
||||||
accountsDynamoDb = new AccountsDynamoDb(
|
|
||||||
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient(),
|
|
||||||
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbAsyncClient(),
|
|
||||||
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
|
|
||||||
ACCOUNTS_DYNAMODB_EXTENSION.getTableName(),
|
|
||||||
NUMBERS_TABLE_NAME,
|
|
||||||
migrationDeletedAccounts,
|
|
||||||
migrationRetryAccounts);
|
|
||||||
|
|
||||||
final KeysDynamoDb keysDynamoDb = new KeysDynamoDb(KEYS_DYNAMODB_EXTENSION.getDynamoDbClient(), KEYS_TABLE_NAME);
|
|
||||||
|
|
||||||
accounts = new Accounts(new FaultTolerantDatabase("accountsTest",
|
|
||||||
Jdbi.create(db.getTestDatabase()),
|
|
||||||
new CircuitBreakerConfiguration()));
|
|
||||||
|
|
||||||
final DirectoryQueue directoryQueue = mock(DirectoryQueue.class);
|
|
||||||
|
|
||||||
final RateLimiters rateLimiters = mock(RateLimiters.class);
|
|
||||||
when(rateLimiters.getVerifyLimiter()).thenReturn(mock(RateLimiter.class));
|
|
||||||
|
|
||||||
final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
|
|
||||||
final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
|
|
||||||
accountMigrationConfiguration = new DynamicAccountsDynamoDbMigrationConfiguration();
|
|
||||||
accountMigrationConfiguration.setBackgroundMigrationEnabled(true);
|
|
||||||
accountMigrationConfiguration.setLogMismatches(true);
|
|
||||||
|
|
||||||
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
|
|
||||||
when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn(accountMigrationConfiguration);
|
|
||||||
|
|
||||||
final ExperimentEnrollmentManager experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class);
|
|
||||||
when(experimentEnrollmentManager.isEnrolled(any(UUID.class), eq("accountsDynamoDbMigration"))).thenReturn(true);
|
|
||||||
|
|
||||||
accountsManager = new AccountsManager(
|
|
||||||
accounts,
|
|
||||||
accountsDynamoDb,
|
|
||||||
REDIS_CLUSTER_EXTENSION.getRedisCluster(),
|
|
||||||
deletedAccountsManager,
|
|
||||||
directoryQueue,
|
|
||||||
keysDynamoDb,
|
|
||||||
mock(MessagesManager.class),
|
|
||||||
mock(MigrationMismatchedAccounts.class),
|
|
||||||
mock(UsernamesManager.class),
|
|
||||||
mock(ProfilesManager.class),
|
|
||||||
mock(StoredVerificationCodeManager.class),
|
|
||||||
mock(SecureStorageClient.class),
|
|
||||||
mock(SecureBackupClient.class),
|
|
||||||
experimentEnrollmentManager,
|
|
||||||
dynamicConfigurationManager);
|
|
||||||
|
|
||||||
final AccountsDynamoDbMigrator dynamoDbMigrator = new AccountsDynamoDbMigrator(accountsDynamoDb,
|
|
||||||
dynamicConfigurationManager);
|
|
||||||
final PushFeedbackProcessor pushFeedbackProcessor = new PushFeedbackProcessor(accountsManager);
|
|
||||||
|
|
||||||
final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache(
|
|
||||||
REDIS_CLUSTER_EXTENSION.getRedisCluster());
|
|
||||||
|
|
||||||
// Using a synchronous service doesn’t meaningfully impact the test
|
|
||||||
final ExecutorService chunkPreReadExecutorService = new SynchronousExecutorService();
|
|
||||||
|
|
||||||
accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager, crawlerCache,
|
|
||||||
List.of(dynamoDbMigrator, pushFeedbackProcessor), CHUNK_SIZE,
|
|
||||||
CHUNK_INTERVAL_MS, chunkPreReadExecutorService, dynamicConfigurationManager);
|
|
||||||
}
|
|
||||||
|
|
||||||
void createAdditionalDynamoDbTables() {
|
|
||||||
CreateTableRequest createNumbersTableRequest = CreateTableRequest.builder()
|
|
||||||
.tableName(NUMBERS_TABLE_NAME)
|
|
||||||
.keySchema(KeySchemaElement.builder()
|
|
||||||
.attributeName("P")
|
|
||||||
.keyType(KeyType.HASH)
|
|
||||||
.build())
|
|
||||||
.attributeDefinitions(AttributeDefinition.builder()
|
|
||||||
.attributeName("P")
|
|
||||||
.attributeType(ScalarAttributeType.S)
|
|
||||||
.build())
|
|
||||||
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createNumbersTableRequest);
|
|
||||||
|
|
||||||
final CreateTableRequest createMigrationDeletedAccountsTableRequest = CreateTableRequest.builder()
|
|
||||||
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
|
|
||||||
.keySchema(KeySchemaElement.builder()
|
|
||||||
.attributeName("U")
|
|
||||||
.keyType(KeyType.HASH)
|
|
||||||
.build())
|
|
||||||
.attributeDefinitions(AttributeDefinition.builder()
|
|
||||||
.attributeName("U")
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createMigrationDeletedAccountsTableRequest);
|
|
||||||
|
|
||||||
final CreateTableRequest createMigrationRetryAccountsTableRequest = CreateTableRequest.builder()
|
|
||||||
.tableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME)
|
|
||||||
.keySchema(KeySchemaElement.builder()
|
|
||||||
.attributeName("U")
|
|
||||||
.keyType(KeyType.HASH)
|
|
||||||
.build())
|
|
||||||
.attributeDefinitions(AttributeDefinition.builder()
|
|
||||||
.attributeName("U")
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
ACCOUNTS_DYNAMODB_EXTENSION.getDynamoDbClient().createTable(createMigrationRetryAccountsTableRequest);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -24,10 +24,6 @@ import java.util.Optional;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.ExecutionException;
|
|
||||||
import java.util.concurrent.LinkedBlockingDeque;
|
|
||||||
import java.util.concurrent.ThreadPoolExecutor;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
import org.jdbi.v3.core.transaction.TransactionException;
|
import org.jdbi.v3.core.transaction.TransactionException;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Disabled;
|
import org.junit.jupiter.api.Disabled;
|
||||||
|
@ -36,21 +32,15 @@ import org.junit.jupiter.api.extension.RegisterExtension;
|
||||||
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
||||||
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
||||||
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
import org.whispersystems.textsecuregcm.util.AttributeValues;
|
||||||
import org.whispersystems.textsecuregcm.util.SystemMapper;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
|
import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
|
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
|
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
|
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
|
import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.KeyType;
|
import software.amazon.awssdk.services.dynamodb.model.KeyType;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest;
|
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.TransactionConflictException;
|
import software.amazon.awssdk.services.dynamodb.model.TransactionConflictException;
|
||||||
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
|
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
|
||||||
|
@ -59,8 +49,6 @@ class AccountsDynamoDbTest {
|
||||||
|
|
||||||
private static final String ACCOUNTS_TABLE_NAME = "accounts_test";
|
private static final String ACCOUNTS_TABLE_NAME = "accounts_test";
|
||||||
private static final String NUMBERS_TABLE_NAME = "numbers_test";
|
private static final String NUMBERS_TABLE_NAME = "numbers_test";
|
||||||
private static final String MIGRATION_DELETED_ACCOUNTS_TABLE_NAME = "migration_deleted_accounts_test";
|
|
||||||
private static final String MIGRATION_RETRY_ACCOUNTS_TABLE_NAME = "migration_retry_accounts_test";
|
|
||||||
|
|
||||||
@RegisterExtension
|
@RegisterExtension
|
||||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
||||||
|
@ -91,50 +79,11 @@ class AccountsDynamoDbTest {
|
||||||
|
|
||||||
dynamoDbExtension.getDynamoDbClient().createTable(createNumbersTableRequest);
|
dynamoDbExtension.getDynamoDbClient().createTable(createNumbersTableRequest);
|
||||||
|
|
||||||
final CreateTableRequest createMigrationDeletedAccountsTableRequest = CreateTableRequest.builder()
|
|
||||||
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
|
|
||||||
.keySchema(KeySchemaElement.builder()
|
|
||||||
.attributeName(MigrationDeletedAccounts.KEY_UUID)
|
|
||||||
.keyType(KeyType.HASH)
|
|
||||||
.build())
|
|
||||||
.attributeDefinitions(AttributeDefinition.builder()
|
|
||||||
.attributeName(MigrationDeletedAccounts.KEY_UUID)
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
dynamoDbExtension.getDynamoDbClient().createTable(createMigrationDeletedAccountsTableRequest);
|
|
||||||
|
|
||||||
MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(
|
|
||||||
dynamoDbExtension.getDynamoDbClient(), MIGRATION_DELETED_ACCOUNTS_TABLE_NAME);
|
|
||||||
|
|
||||||
final CreateTableRequest createMigrationRetryAccountsTableRequest = CreateTableRequest.builder()
|
|
||||||
.tableName(MIGRATION_RETRY_ACCOUNTS_TABLE_NAME)
|
|
||||||
.keySchema(KeySchemaElement.builder()
|
|
||||||
.attributeName(MigrationRetryAccounts.KEY_UUID)
|
|
||||||
.keyType(KeyType.HASH)
|
|
||||||
.build())
|
|
||||||
.attributeDefinitions(AttributeDefinition.builder()
|
|
||||||
.attributeName(MigrationRetryAccounts.KEY_UUID)
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.provisionedThroughput(DynamoDbExtension.DEFAULT_PROVISIONED_THROUGHPUT)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
dynamoDbExtension.getDynamoDbClient().createTable(createMigrationRetryAccountsTableRequest);
|
|
||||||
|
|
||||||
MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts((dynamoDbExtension.getDynamoDbClient()),
|
|
||||||
MIGRATION_RETRY_ACCOUNTS_TABLE_NAME);
|
|
||||||
|
|
||||||
this.accountsDynamoDb = new AccountsDynamoDb(
|
this.accountsDynamoDb = new AccountsDynamoDb(
|
||||||
dynamoDbExtension.getDynamoDbClient(),
|
dynamoDbExtension.getDynamoDbClient(),
|
||||||
dynamoDbExtension.getDynamoDbAsyncClient(),
|
|
||||||
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
|
|
||||||
dynamoDbExtension.getTableName(),
|
dynamoDbExtension.getTableName(),
|
||||||
NUMBERS_TABLE_NAME,
|
NUMBERS_TABLE_NAME
|
||||||
migrationDeletedAccounts,
|
);
|
||||||
migrationRetryAccounts);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -283,15 +232,13 @@ class AccountsDynamoDbTest {
|
||||||
void testUpdateWithMockTransactionConflictException() {
|
void testUpdateWithMockTransactionConflictException() {
|
||||||
|
|
||||||
final DynamoDbClient dynamoDbClient = mock(DynamoDbClient.class);
|
final DynamoDbClient dynamoDbClient = mock(DynamoDbClient.class);
|
||||||
accountsDynamoDb = new AccountsDynamoDb(dynamoDbClient, mock(DynamoDbAsyncClient.class),
|
accountsDynamoDb = new AccountsDynamoDb(dynamoDbClient,
|
||||||
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
|
dynamoDbExtension.getTableName(), NUMBERS_TABLE_NAME);
|
||||||
dynamoDbExtension.getTableName(), NUMBERS_TABLE_NAME, mock(MigrationDeletedAccounts.class),
|
|
||||||
mock(MigrationRetryAccounts.class));
|
|
||||||
|
|
||||||
when(dynamoDbClient.updateItem(any(UpdateItemRequest.class)))
|
when(dynamoDbClient.updateItem(any(UpdateItemRequest.class)))
|
||||||
.thenThrow(TransactionConflictException.class);
|
.thenThrow(TransactionConflictException.class);
|
||||||
|
|
||||||
Device device = generateDevice (1 );
|
Device device = generateDevice(1);
|
||||||
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
|
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
|
||||||
|
|
||||||
assertThatThrownBy(() -> accountsDynamoDb.update(account)).isInstanceOfAny(ContestedOptimisticLockException.class);
|
assertThatThrownBy(() -> accountsDynamoDb.update(account)).isInstanceOfAny(ContestedOptimisticLockException.class);
|
||||||
|
@ -376,33 +323,6 @@ class AccountsDynamoDbTest {
|
||||||
verifyStoredState(recreatedAccount.getNumber(), recreatedAccount.getUuid(),
|
verifyStoredState(recreatedAccount.getNumber(), recreatedAccount.getUuid(),
|
||||||
accountsDynamoDb.get(recreatedAccount.getUuid()).get(), recreatedAccount);
|
accountsDynamoDb.get(recreatedAccount.getUuid()).get(), recreatedAccount);
|
||||||
}
|
}
|
||||||
|
|
||||||
verifyRecentlyDeletedAccountsTableItemCount(1);
|
|
||||||
|
|
||||||
Map<String, AttributeValue> primaryKey = MigrationDeletedAccounts.primaryKey(deletedAccount.getUuid());
|
|
||||||
assertThat(dynamoDbExtension.getDynamoDbClient().getItem(GetItemRequest.builder()
|
|
||||||
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
|
|
||||||
.key(Map.of(MigrationDeletedAccounts.KEY_UUID, primaryKey.get(MigrationDeletedAccounts.KEY_UUID)))
|
|
||||||
.build()))
|
|
||||||
.isNotNull();
|
|
||||||
|
|
||||||
accountsDynamoDb.deleteRecentlyDeletedUuids();
|
|
||||||
|
|
||||||
verifyRecentlyDeletedAccountsTableItemCount(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void verifyRecentlyDeletedAccountsTableItemCount(int expectedItemCount) {
|
|
||||||
int totalItems = 0;
|
|
||||||
|
|
||||||
for (ScanResponse page : dynamoDbExtension.getDynamoDbClient().scanPaginator(ScanRequest.builder()
|
|
||||||
.tableName(MIGRATION_DELETED_ACCOUNTS_TABLE_NAME)
|
|
||||||
.build())) {
|
|
||||||
for (Map<String, AttributeValue> item : page.items()) {
|
|
||||||
totalItems++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
assertThat(totalItems).isEqualTo(expectedItemCount);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -437,9 +357,8 @@ class AccountsDynamoDbTest {
|
||||||
when(client.updateItem(any(UpdateItemRequest.class)))
|
when(client.updateItem(any(UpdateItemRequest.class)))
|
||||||
.thenThrow(RuntimeException.class);
|
.thenThrow(RuntimeException.class);
|
||||||
|
|
||||||
AccountsDynamoDb accounts = new AccountsDynamoDb(client, mock(DynamoDbAsyncClient.class), mock(ThreadPoolExecutor.class), ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME, mock(
|
AccountsDynamoDb accounts = new AccountsDynamoDb(client, ACCOUNTS_TABLE_NAME, NUMBERS_TABLE_NAME);
|
||||||
MigrationDeletedAccounts.class), mock(MigrationRetryAccounts.class));
|
Account account = generateAccount("+14151112222", UUID.randomUUID());
|
||||||
Account account = generateAccount("+14151112222", UUID.randomUUID());
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
accounts.update(account);
|
accounts.update(account);
|
||||||
|
@ -472,42 +391,6 @@ class AccountsDynamoDbTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
void testMigrate() throws ExecutionException, InterruptedException {
|
|
||||||
|
|
||||||
Device device = generateDevice (1 );
|
|
||||||
UUID firstUuid = UUID.randomUUID();
|
|
||||||
Account account = generateAccount("+14151112222", firstUuid, Collections.singleton(device));
|
|
||||||
|
|
||||||
boolean migrated = accountsDynamoDb.migrate(account).get();
|
|
||||||
|
|
||||||
assertThat(migrated).isTrue();
|
|
||||||
|
|
||||||
verifyStoredState("+14151112222", account.getUuid(), account, true);
|
|
||||||
|
|
||||||
migrated = accountsDynamoDb.migrate(account).get();
|
|
||||||
|
|
||||||
assertThat(migrated).isFalse();
|
|
||||||
|
|
||||||
verifyStoredState("+14151112222", account.getUuid(), account, true);
|
|
||||||
|
|
||||||
UUID secondUuid = UUID.randomUUID();
|
|
||||||
|
|
||||||
device = generateDevice(1);
|
|
||||||
Account accountRemigrationWithDifferentUuid = generateAccount("+14151112222", secondUuid, Collections.singleton(device));
|
|
||||||
|
|
||||||
migrated = accountsDynamoDb.migrate(accountRemigrationWithDifferentUuid).get();
|
|
||||||
|
|
||||||
assertThat(migrated).isFalse();
|
|
||||||
verifyStoredState("+14151112222", firstUuid, account, true);
|
|
||||||
|
|
||||||
account.setVersion(account.getVersion() + 1);
|
|
||||||
|
|
||||||
migrated = accountsDynamoDb.migrate(account).get();
|
|
||||||
|
|
||||||
assertThat(migrated).isTrue();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testCanonicallyDiscoverableSet() {
|
void testCanonicallyDiscoverableSet() {
|
||||||
Device device = generateDevice(1);
|
Device device = generateDevice(1);
|
||||||
|
|
|
@ -23,7 +23,6 @@ import com.opentable.db.postgres.junit5.PreparedDbExtension;
|
||||||
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
|
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
@ -34,18 +33,14 @@ import java.util.concurrent.ThreadPoolExecutor;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.function.Consumer;
|
import java.util.function.Consumer;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
import org.jdbi.v3.core.Jdbi;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||||
import org.mockito.ArgumentCaptor;
|
import org.mockito.ArgumentCaptor;
|
||||||
import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
|
import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
|
||||||
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
||||||
import org.whispersystems.textsecuregcm.entities.AccountAttributes;
|
import org.whispersystems.textsecuregcm.entities.AccountAttributes;
|
||||||
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
||||||
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
|
|
||||||
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
||||||
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
||||||
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
||||||
|
@ -76,8 +71,6 @@ class AccountsManagerConcurrentModificationIntegrationTest {
|
||||||
.build())
|
.build())
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
private Accounts accounts;
|
|
||||||
|
|
||||||
private AccountsDynamoDb accountsDynamoDb;
|
private AccountsDynamoDb accountsDynamoDb;
|
||||||
|
|
||||||
private AccountsManager accountsManager;
|
private AccountsManager accountsManager;
|
||||||
|
@ -108,22 +101,9 @@ class AccountsManagerConcurrentModificationIntegrationTest {
|
||||||
|
|
||||||
accountsDynamoDb = new AccountsDynamoDb(
|
accountsDynamoDb = new AccountsDynamoDb(
|
||||||
dynamoDbExtension.getDynamoDbClient(),
|
dynamoDbExtension.getDynamoDbClient(),
|
||||||
dynamoDbExtension.getDynamoDbAsyncClient(),
|
|
||||||
new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingDeque<>()),
|
|
||||||
dynamoDbExtension.getTableName(),
|
dynamoDbExtension.getTableName(),
|
||||||
NUMBERS_TABLE_NAME,
|
NUMBERS_TABLE_NAME
|
||||||
mock(MigrationDeletedAccounts.class),
|
);
|
||||||
mock(MigrationRetryAccounts.class));
|
|
||||||
|
|
||||||
{
|
|
||||||
final CircuitBreakerConfiguration circuitBreakerConfiguration = new CircuitBreakerConfiguration();
|
|
||||||
circuitBreakerConfiguration.setIgnoredExceptions(List.of("org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException"));
|
|
||||||
FaultTolerantDatabase faultTolerantDatabase = new FaultTolerantDatabase("accountsTest",
|
|
||||||
Jdbi.create(db.getTestDatabase()),
|
|
||||||
circuitBreakerConfiguration);
|
|
||||||
|
|
||||||
accounts = new Accounts(faultTolerantDatabase);
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
{
|
||||||
final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
|
final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
|
||||||
|
@ -131,17 +111,6 @@ class AccountsManagerConcurrentModificationIntegrationTest {
|
||||||
DynamicConfiguration dynamicConfiguration = new DynamicConfiguration();
|
DynamicConfiguration dynamicConfiguration = new DynamicConfiguration();
|
||||||
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
|
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
|
||||||
|
|
||||||
final ExperimentEnrollmentManager experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class);
|
|
||||||
|
|
||||||
final DynamicAccountsDynamoDbMigrationConfiguration config = dynamicConfiguration
|
|
||||||
.getAccountsDynamoDbMigrationConfiguration();
|
|
||||||
|
|
||||||
config.setDeleteEnabled(true);
|
|
||||||
config.setReadEnabled(true);
|
|
||||||
config.setWriteEnabled(true);
|
|
||||||
|
|
||||||
when(experimentEnrollmentManager.isEnrolled(any(UUID.class), anyString())).thenReturn(true);
|
|
||||||
|
|
||||||
commands = mock(RedisAdvancedClusterCommands.class);
|
commands = mock(RedisAdvancedClusterCommands.class);
|
||||||
|
|
||||||
final DeletedAccountsManager deletedAccountsManager = mock(DeletedAccountsManager.class);
|
final DeletedAccountsManager deletedAccountsManager = mock(DeletedAccountsManager.class);
|
||||||
|
@ -153,20 +122,17 @@ class AccountsManagerConcurrentModificationIntegrationTest {
|
||||||
}).when(deletedAccountsManager).lockAndTake(anyString(), any());
|
}).when(deletedAccountsManager).lockAndTake(anyString(), any());
|
||||||
|
|
||||||
accountsManager = new AccountsManager(
|
accountsManager = new AccountsManager(
|
||||||
accounts,
|
|
||||||
accountsDynamoDb,
|
accountsDynamoDb,
|
||||||
RedisClusterHelper.buildMockRedisCluster(commands),
|
RedisClusterHelper.buildMockRedisCluster(commands),
|
||||||
deletedAccountsManager,
|
deletedAccountsManager,
|
||||||
mock(DirectoryQueue.class),
|
mock(DirectoryQueue.class),
|
||||||
mock(KeysDynamoDb.class),
|
mock(KeysDynamoDb.class),
|
||||||
mock(MessagesManager.class),
|
mock(MessagesManager.class),
|
||||||
mock(MigrationMismatchedAccounts.class),
|
|
||||||
mock(UsernamesManager.class),
|
mock(UsernamesManager.class),
|
||||||
mock(ProfilesManager.class),
|
mock(ProfilesManager.class),
|
||||||
mock(StoredVerificationCodeManager.class),
|
mock(StoredVerificationCodeManager.class),
|
||||||
mock(SecureStorageClient.class),
|
mock(SecureStorageClient.class),
|
||||||
mock(SecureBackupClient.class),
|
mock(SecureBackupClient.class),
|
||||||
experimentEnrollmentManager,
|
|
||||||
dynamicConfigurationManager);
|
dynamicConfigurationManager);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -225,14 +191,12 @@ class AccountsManagerConcurrentModificationIntegrationTest {
|
||||||
).join();
|
).join();
|
||||||
|
|
||||||
final Account managerAccount = accountsManager.get(uuid).get();
|
final Account managerAccount = accountsManager.get(uuid).get();
|
||||||
final Account dbAccount = accounts.get(uuid).get();
|
|
||||||
final Account dynamoAccount = accountsDynamoDb.get(uuid).get();
|
final Account dynamoAccount = accountsDynamoDb.get(uuid).get();
|
||||||
|
|
||||||
final Account redisAccount = getLastAccountFromRedisMock(commands);
|
final Account redisAccount = getLastAccountFromRedisMock(commands);
|
||||||
|
|
||||||
Stream.of(
|
Stream.of(
|
||||||
new Pair<>("manager", managerAccount),
|
new Pair<>("manager", managerAccount),
|
||||||
new Pair<>("db", dbAccount),
|
|
||||||
new Pair<>("dynamo", dynamoAccount),
|
new Pair<>("dynamo", dynamoAccount),
|
||||||
new Pair<>("redis", redisAccount)
|
new Pair<>("redis", redisAccount)
|
||||||
).forEach(pair ->
|
).forEach(pair ->
|
||||||
|
|
|
@ -1,44 +0,0 @@
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.UUID;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
|
||||||
|
|
||||||
class MigrationDeletedAccountsTest {
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
|
||||||
.tableName("deleted_accounts_test")
|
|
||||||
.hashKey(MigrationDeletedAccounts.KEY_UUID)
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName(MigrationDeletedAccounts.KEY_UUID)
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test() {
|
|
||||||
|
|
||||||
final MigrationDeletedAccounts migrationDeletedAccounts = new MigrationDeletedAccounts(dynamoDbExtension.getDynamoDbClient(),
|
|
||||||
dynamoDbExtension.getTableName());
|
|
||||||
|
|
||||||
UUID firstUuid = UUID.randomUUID();
|
|
||||||
UUID secondUuid = UUID.randomUUID();
|
|
||||||
|
|
||||||
assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().isEmpty());
|
|
||||||
|
|
||||||
migrationDeletedAccounts.put(firstUuid);
|
|
||||||
migrationDeletedAccounts.put(secondUuid);
|
|
||||||
|
|
||||||
assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().containsAll(List.of(firstUuid, secondUuid)));
|
|
||||||
|
|
||||||
migrationDeletedAccounts.delete(List.of(firstUuid, secondUuid));
|
|
||||||
|
|
||||||
assertTrue(migrationDeletedAccounts.getRecentlyDeletedUuids().isEmpty());
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,56 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2021 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import java.time.Clock;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.UUID;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
|
||||||
|
|
||||||
class MigrationMismatchAccountsTest {
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
|
||||||
.tableName("account_migration_mismatches_test")
|
|
||||||
.hashKey(MigrationRetryAccounts.KEY_UUID)
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName(MigrationRetryAccounts.KEY_UUID)
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test() {
|
|
||||||
|
|
||||||
final Clock clock = mock(Clock.class);
|
|
||||||
when(clock.millis()).thenReturn(0L);
|
|
||||||
|
|
||||||
final MigrationMismatchedAccounts migrationMismatchedAccounts = new MigrationMismatchedAccounts(
|
|
||||||
dynamoDbExtension.getDynamoDbClient(),
|
|
||||||
dynamoDbExtension.getTableName(), clock);
|
|
||||||
|
|
||||||
UUID firstUuid = UUID.randomUUID();
|
|
||||||
UUID secondUuid = UUID.randomUUID();
|
|
||||||
|
|
||||||
assertTrue(migrationMismatchedAccounts.getUuids(10).isEmpty());
|
|
||||||
|
|
||||||
migrationMismatchedAccounts.put(firstUuid);
|
|
||||||
migrationMismatchedAccounts.put(secondUuid);
|
|
||||||
|
|
||||||
assertTrue(migrationMismatchedAccounts.getUuids(10).isEmpty());
|
|
||||||
|
|
||||||
when(clock.millis()).thenReturn(MigrationMismatchedAccounts.MISMATCH_CHECK_DELAY_MILLIS);
|
|
||||||
|
|
||||||
assertTrue(migrationMismatchedAccounts.getUuids(10).containsAll(List.of(firstUuid, secondUuid)));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,40 +0,0 @@
|
||||||
package org.whispersystems.textsecuregcm.storage;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.UUID;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
|
|
||||||
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
|
|
||||||
|
|
||||||
class MigrationRetryAccountsTest {
|
|
||||||
|
|
||||||
@RegisterExtension
|
|
||||||
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
|
|
||||||
.tableName("account_migration_errors_test")
|
|
||||||
.hashKey(MigrationRetryAccounts.KEY_UUID)
|
|
||||||
.attributeDefinition(AttributeDefinition.builder()
|
|
||||||
.attributeName(MigrationRetryAccounts.KEY_UUID)
|
|
||||||
.attributeType(ScalarAttributeType.B)
|
|
||||||
.build())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void test() {
|
|
||||||
|
|
||||||
final MigrationRetryAccounts migrationRetryAccounts = new MigrationRetryAccounts(dynamoDbExtension.getDynamoDbClient(),
|
|
||||||
dynamoDbExtension.getTableName());
|
|
||||||
|
|
||||||
UUID firstUuid = UUID.randomUUID();
|
|
||||||
UUID secondUuid = UUID.randomUUID();
|
|
||||||
|
|
||||||
assertTrue(migrationRetryAccounts.getUuids(10).isEmpty());
|
|
||||||
|
|
||||||
migrationRetryAccounts.put(firstUuid);
|
|
||||||
migrationRetryAccounts.put(secondUuid);
|
|
||||||
|
|
||||||
assertTrue(migrationRetryAccounts.getUuids(10).containsAll(List.of(firstUuid, secondUuid)));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -16,7 +16,6 @@ import static org.mockito.Mockito.times;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.verifyNoInteractions;
|
import static org.mockito.Mockito.verifyNoInteractions;
|
||||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||||
import static org.mockito.Mockito.verifyZeroInteractions;
|
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
@ -25,10 +24,7 @@ import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.params.ParameterizedTest;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.params.provider.ValueSource;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Account;
|
import org.whispersystems.textsecuregcm.storage.Account;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk;
|
import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
|
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
|
||||||
|
@ -36,7 +32,6 @@ import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
|
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerRestartException;
|
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerRestartException;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
|
|
||||||
|
|
||||||
class AccountDatabaseCrawlerTest {
|
class AccountDatabaseCrawlerTest {
|
||||||
|
|
||||||
|
@ -55,22 +50,14 @@ class AccountDatabaseCrawlerTest {
|
||||||
|
|
||||||
private final ExecutorService chunkPreReadExecutorService = mock(ExecutorService.class);
|
private final ExecutorService chunkPreReadExecutorService = mock(ExecutorService.class);
|
||||||
|
|
||||||
private final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
|
|
||||||
|
|
||||||
private final AccountDatabaseCrawler crawler = new AccountDatabaseCrawler(accounts, cache, Arrays.asList(listener),
|
private final AccountDatabaseCrawler crawler = new AccountDatabaseCrawler(accounts, cache, Arrays.asList(listener),
|
||||||
CHUNK_SIZE, CHUNK_INTERVAL_MS, chunkPreReadExecutorService, dynamicConfigurationManager);
|
CHUNK_SIZE, CHUNK_INTERVAL_MS);
|
||||||
private DynamicAccountsDynamoDbMigrationConfiguration dynamicAccountsDynamoDbMigrationConfiguration;
|
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
void setup() {
|
void setup() {
|
||||||
when(account1.getUuid()).thenReturn(ACCOUNT1);
|
when(account1.getUuid()).thenReturn(ACCOUNT1);
|
||||||
when(account2.getUuid()).thenReturn(ACCOUNT2);
|
when(account2.getUuid()).thenReturn(ACCOUNT2);
|
||||||
|
|
||||||
when(accounts.getAllFrom(anyInt())).thenReturn(new AccountCrawlChunk(Arrays.asList(account1, account2), ACCOUNT2));
|
|
||||||
when(accounts.getAllFrom(eq(ACCOUNT1), anyInt())).thenReturn(
|
|
||||||
new AccountCrawlChunk(Arrays.asList(account2), ACCOUNT2));
|
|
||||||
when(accounts.getAllFrom(eq(ACCOUNT2), anyInt())).thenReturn(new AccountCrawlChunk(Collections.emptyList(), null));
|
|
||||||
|
|
||||||
when(accounts.getAllFromDynamo(anyInt())).thenReturn(
|
when(accounts.getAllFromDynamo(anyInt())).thenReturn(
|
||||||
new AccountCrawlChunk(Arrays.asList(account1, account2), ACCOUNT2));
|
new AccountCrawlChunk(Arrays.asList(account1, account2), ACCOUNT2));
|
||||||
when(accounts.getAllFromDynamo(eq(ACCOUNT1), anyInt())).thenReturn(
|
when(accounts.getAllFromDynamo(eq(ACCOUNT1), anyInt())).thenReturn(
|
||||||
|
@ -81,17 +68,10 @@ class AccountDatabaseCrawlerTest {
|
||||||
when(cache.claimActiveWork(any(), anyLong())).thenReturn(true);
|
when(cache.claimActiveWork(any(), anyLong())).thenReturn(true);
|
||||||
when(cache.isAccelerated()).thenReturn(false);
|
when(cache.isAccelerated()).thenReturn(false);
|
||||||
|
|
||||||
final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class);
|
|
||||||
when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration);
|
|
||||||
dynamicAccountsDynamoDbMigrationConfiguration = mock(DynamicAccountsDynamoDbMigrationConfiguration.class);
|
|
||||||
when(dynamicConfiguration.getAccountsDynamoDbMigrationConfiguration()).thenReturn(
|
|
||||||
dynamicAccountsDynamoDbMigrationConfiguration);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testCrawlStart() throws AccountDatabaseCrawlerRestartException {
|
||||||
void testCrawlStart(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException {
|
|
||||||
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
|
|
||||||
when(cache.getLastUuid()).thenReturn(Optional.empty());
|
when(cache.getLastUuid()).thenReturn(Optional.empty());
|
||||||
when(cache.getLastUuidDynamo()).thenReturn(Optional.empty());
|
when(cache.getLastUuidDynamo()).thenReturn(Optional.empty());
|
||||||
|
|
||||||
|
@ -99,20 +79,15 @@ class AccountDatabaseCrawlerTest {
|
||||||
assertThat(accelerated).isFalse();
|
assertThat(accelerated).isFalse();
|
||||||
|
|
||||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).getLastUuid();
|
verify(cache, times(0)).getLastUuid();
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo();
|
verify(cache, times(1)).getLastUuidDynamo();
|
||||||
verify(listener, times(1)).onCrawlStart();
|
verify(listener, times(1)).onCrawlStart();
|
||||||
if (useDynamo) {
|
verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE));
|
verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE));
|
|
||||||
} else {
|
|
||||||
verify(accounts, times(1)).getAllFrom(eq(CHUNK_SIZE));
|
|
||||||
verify(accounts, times(0)).getAllFrom(any(UUID.class), eq(CHUNK_SIZE));
|
|
||||||
}
|
|
||||||
verify(account1, times(0)).getUuid();
|
verify(account1, times(0)).getUuid();
|
||||||
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.empty()), eq(Arrays.asList(account1, account2)));
|
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.empty()), eq(Arrays.asList(account1, account2)));
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
|
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
|
||||||
verify(cache, times(1)).isAccelerated();
|
verify(cache, times(1)).isAccelerated();
|
||||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||||
|
|
||||||
|
@ -123,10 +98,8 @@ class AccountDatabaseCrawlerTest {
|
||||||
verifyNoMoreInteractions(cache);
|
verifyNoMoreInteractions(cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testCrawlChunk() throws AccountDatabaseCrawlerRestartException {
|
||||||
void testCrawlChunk(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException {
|
|
||||||
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
|
|
||||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
||||||
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
|
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
|
||||||
|
|
||||||
|
@ -134,18 +107,13 @@ class AccountDatabaseCrawlerTest {
|
||||||
assertThat(accelerated).isFalse();
|
assertThat(accelerated).isFalse();
|
||||||
|
|
||||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||||
verify(cache, times(useDynamo ? 0: 1)).getLastUuid();
|
verify(cache, times(0)).getLastUuid();
|
||||||
verify(cache, times(useDynamo ? 1: 0)).getLastUuidDynamo();
|
verify(cache, times(1)).getLastUuidDynamo();
|
||||||
if (useDynamo) {
|
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
|
||||||
} else {
|
|
||||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
|
||||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
|
||||||
}
|
|
||||||
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
|
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
|
||||||
verify(cache, times(1)).isAccelerated();
|
verify(cache, times(1)).isAccelerated();
|
||||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||||
|
|
||||||
|
@ -157,46 +125,8 @@ class AccountDatabaseCrawlerTest {
|
||||||
verifyNoMoreInteractions(cache);
|
verifyNoMoreInteractions(cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testCrawlChunkAccelerated() throws AccountDatabaseCrawlerRestartException {
|
||||||
void testCrawlChunk_useDynamoDedicatedMigrationCrawler(final boolean dedicatedMigrationCrawler) throws Exception {
|
|
||||||
crawler.setDedicatedDynamoMigrationCrawler(dedicatedMigrationCrawler);
|
|
||||||
|
|
||||||
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(true);
|
|
||||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
|
||||||
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
|
|
||||||
|
|
||||||
boolean accelerated = crawler.doPeriodicWork();
|
|
||||||
assertThat(accelerated).isFalse();
|
|
||||||
|
|
||||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
|
||||||
verify(cache, times(dedicatedMigrationCrawler ? 1 : 0)).getLastUuid();
|
|
||||||
verify(cache, times(dedicatedMigrationCrawler ? 0 : 1)).getLastUuidDynamo();
|
|
||||||
if (dedicatedMigrationCrawler) {
|
|
||||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
|
||||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
|
||||||
} else {
|
|
||||||
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
|
||||||
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
|
||||||
}
|
|
||||||
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
|
||||||
verify(cache, times(dedicatedMigrationCrawler ? 1 : 0)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
|
||||||
verify(cache, times(dedicatedMigrationCrawler ? 0 : 1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
|
|
||||||
verify(cache, times(1)).isAccelerated();
|
|
||||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
|
||||||
|
|
||||||
verifyNoInteractions(account1);
|
|
||||||
|
|
||||||
verifyNoMoreInteractions(account2);
|
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
verifyNoMoreInteractions(listener);
|
|
||||||
verifyNoMoreInteractions(cache);
|
|
||||||
}
|
|
||||||
|
|
||||||
@ParameterizedTest
|
|
||||||
@ValueSource(booleans = {true, false})
|
|
||||||
void testCrawlChunkAccelerated(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException {
|
|
||||||
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
|
|
||||||
when(cache.isAccelerated()).thenReturn(true);
|
when(cache.isAccelerated()).thenReturn(true);
|
||||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
||||||
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
|
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
|
||||||
|
@ -205,22 +135,17 @@ class AccountDatabaseCrawlerTest {
|
||||||
assertThat(accelerated).isTrue();
|
assertThat(accelerated).isTrue();
|
||||||
|
|
||||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).getLastUuid();
|
verify(cache, times(0)).getLastUuid();
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo();
|
verify(cache, times(1)).getLastUuidDynamo();
|
||||||
if (useDynamo) {
|
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
|
||||||
} else {
|
|
||||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
|
||||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
|
||||||
}
|
|
||||||
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
|
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
|
||||||
verify(cache, times(1)).isAccelerated();
|
verify(cache, times(1)).isAccelerated();
|
||||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||||
|
|
||||||
verifyZeroInteractions(account1);
|
verifyNoInteractions(account1);
|
||||||
|
|
||||||
verifyNoMoreInteractions(account2);
|
verifyNoMoreInteractions(account2);
|
||||||
verifyNoMoreInteractions(accounts);
|
verifyNoMoreInteractions(accounts);
|
||||||
|
@ -228,36 +153,30 @@ class AccountDatabaseCrawlerTest {
|
||||||
verifyNoMoreInteractions(cache);
|
verifyNoMoreInteractions(cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testCrawlChunkRestart() throws AccountDatabaseCrawlerRestartException {
|
||||||
void testCrawlChunkRestart(final boolean useDynamo) throws AccountDatabaseCrawlerRestartException {
|
|
||||||
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
|
|
||||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
||||||
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
|
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
|
||||||
doThrow(AccountDatabaseCrawlerRestartException.class).when(listener).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
doThrow(AccountDatabaseCrawlerRestartException.class).when(listener)
|
||||||
|
.timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
||||||
|
|
||||||
boolean accelerated = crawler.doPeriodicWork();
|
boolean accelerated = crawler.doPeriodicWork();
|
||||||
assertThat(accelerated).isFalse();
|
assertThat(accelerated).isFalse();
|
||||||
|
|
||||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).getLastUuid();
|
verify(cache, times(0)).getLastUuid();
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo();
|
verify(cache, times(1)).getLastUuidDynamo();
|
||||||
if (useDynamo) {
|
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
|
||||||
} else {
|
|
||||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
|
||||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
|
||||||
}
|
|
||||||
verify(account2, times(0)).getNumber();
|
verify(account2, times(0)).getNumber();
|
||||||
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.empty()));
|
verify(cache, times(0)).setLastUuid(eq(Optional.empty()));
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.empty()));
|
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.empty()));
|
||||||
verify(cache, times(1)).setAccelerated(false);
|
verify(cache, times(1)).setAccelerated(false);
|
||||||
verify(cache, times(1)).isAccelerated();
|
verify(cache, times(1)).isAccelerated();
|
||||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||||
|
|
||||||
verifyZeroInteractions(account1);
|
verifyNoInteractions(account1);
|
||||||
|
|
||||||
verifyNoMoreInteractions(account2);
|
verifyNoMoreInteractions(account2);
|
||||||
verifyNoMoreInteractions(accounts);
|
verifyNoMoreInteractions(accounts);
|
||||||
|
@ -265,10 +184,8 @@ class AccountDatabaseCrawlerTest {
|
||||||
verifyNoMoreInteractions(cache);
|
verifyNoMoreInteractions(cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testCrawlEnd() {
|
||||||
void testCrawlEnd(final boolean useDynamo) {
|
|
||||||
when(dynamicAccountsDynamoDbMigrationConfiguration.isDynamoCrawlerEnabled()).thenReturn(useDynamo);
|
|
||||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2));
|
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2));
|
||||||
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT2));
|
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT2));
|
||||||
|
|
||||||
|
@ -276,26 +193,21 @@ class AccountDatabaseCrawlerTest {
|
||||||
assertThat(accelerated).isFalse();
|
assertThat(accelerated).isFalse();
|
||||||
|
|
||||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).getLastUuid();
|
verify(cache, times(0)).getLastUuid();
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).getLastUuidDynamo();
|
verify(cache, times(1)).getLastUuidDynamo();
|
||||||
if (useDynamo) {
|
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
|
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE));
|
||||||
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE));
|
|
||||||
} else {
|
|
||||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
|
||||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT2), eq(CHUNK_SIZE));
|
|
||||||
}
|
|
||||||
verify(account1, times(0)).getNumber();
|
verify(account1, times(0)).getNumber();
|
||||||
verify(account2, times(0)).getNumber();
|
verify(account2, times(0)).getNumber();
|
||||||
verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2)));
|
verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2)));
|
||||||
verify(cache, times(useDynamo ? 0 : 1)).setLastUuid(eq(Optional.empty()));
|
verify(cache, times(0)).setLastUuid(eq(Optional.empty()));
|
||||||
verify(cache, times(useDynamo ? 1 : 0)).setLastUuidDynamo(eq(Optional.empty()));
|
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.empty()));
|
||||||
verify(cache, times(1)).setAccelerated(false);
|
verify(cache, times(1)).setAccelerated(false);
|
||||||
verify(cache, times(1)).isAccelerated();
|
verify(cache, times(1)).isAccelerated();
|
||||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||||
|
|
||||||
verifyZeroInteractions(account1);
|
verifyNoInteractions(account1);
|
||||||
verifyZeroInteractions(account2);
|
verifyNoInteractions(account2);
|
||||||
|
|
||||||
verifyNoMoreInteractions(accounts);
|
verifyNoMoreInteractions(accounts);
|
||||||
verifyNoMoreInteractions(listener);
|
verifyNoMoreInteractions(listener);
|
||||||
|
|
|
@ -33,6 +33,7 @@ import java.util.UUID;
|
||||||
import java.util.function.Consumer;
|
import java.util.function.Consumer;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Disabled;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.params.ParameterizedTest;
|
import org.junit.jupiter.params.ParameterizedTest;
|
||||||
import org.junit.jupiter.params.provider.Arguments;
|
import org.junit.jupiter.params.provider.Arguments;
|
||||||
|
@ -40,16 +41,13 @@ import org.junit.jupiter.params.provider.MethodSource;
|
||||||
import org.junit.jupiter.params.provider.ValueSource;
|
import org.junit.jupiter.params.provider.ValueSource;
|
||||||
import org.mockito.ArgumentCaptor;
|
import org.mockito.ArgumentCaptor;
|
||||||
import org.mockito.stubbing.Answer;
|
import org.mockito.stubbing.Answer;
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAccountsDynamoDbMigrationConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
|
||||||
import org.whispersystems.textsecuregcm.entities.AccountAttributes;
|
import org.whispersystems.textsecuregcm.entities.AccountAttributes;
|
||||||
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
||||||
import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
|
|
||||||
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
|
||||||
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
|
||||||
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
||||||
import org.whispersystems.textsecuregcm.storage.Account;
|
import org.whispersystems.textsecuregcm.storage.Account;
|
||||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException;
|
import org.whispersystems.textsecuregcm.storage.ContestedOptimisticLockException;
|
||||||
|
@ -59,7 +57,6 @@ import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities;
|
||||||
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
|
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
|
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
|
||||||
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.MigrationMismatchedAccounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
|
import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
|
||||||
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
||||||
|
@ -68,12 +65,10 @@ import org.whispersystems.textsecuregcm.tests.util.RedisClusterHelper;
|
||||||
|
|
||||||
class AccountsManagerTest {
|
class AccountsManagerTest {
|
||||||
|
|
||||||
private Accounts accounts;
|
|
||||||
private AccountsDynamoDb accountsDynamoDb;
|
private AccountsDynamoDb accountsDynamoDb;
|
||||||
private DeletedAccountsManager deletedAccountsManager;
|
private DeletedAccountsManager deletedAccountsManager;
|
||||||
private DirectoryQueue directoryQueue;
|
private DirectoryQueue directoryQueue;
|
||||||
private DynamicConfigurationManager dynamicConfigurationManager;
|
private DynamicConfigurationManager dynamicConfigurationManager;
|
||||||
private ExperimentEnrollmentManager experimentEnrollmentManager;
|
|
||||||
private KeysDynamoDb keys;
|
private KeysDynamoDb keys;
|
||||||
private MessagesManager messagesManager;
|
private MessagesManager messagesManager;
|
||||||
private ProfilesManager profilesManager;
|
private ProfilesManager profilesManager;
|
||||||
|
@ -91,12 +86,10 @@ class AccountsManagerTest {
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
void setup() throws InterruptedException {
|
void setup() throws InterruptedException {
|
||||||
accounts = mock(Accounts.class);
|
|
||||||
accountsDynamoDb = mock(AccountsDynamoDb.class);
|
accountsDynamoDb = mock(AccountsDynamoDb.class);
|
||||||
deletedAccountsManager = mock(DeletedAccountsManager.class);
|
deletedAccountsManager = mock(DeletedAccountsManager.class);
|
||||||
directoryQueue = mock(DirectoryQueue.class);
|
directoryQueue = mock(DirectoryQueue.class);
|
||||||
dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
|
dynamicConfigurationManager = mock(DynamicConfigurationManager.class);
|
||||||
experimentEnrollmentManager = mock(ExperimentEnrollmentManager.class);
|
|
||||||
keys = mock(KeysDynamoDb.class);
|
keys = mock(KeysDynamoDb.class);
|
||||||
messagesManager = mock(MessagesManager.class);
|
messagesManager = mock(MessagesManager.class);
|
||||||
profilesManager = mock(ProfilesManager.class);
|
profilesManager = mock(ProfilesManager.class);
|
||||||
|
@ -114,30 +107,24 @@ class AccountsManagerTest {
|
||||||
}).when(deletedAccountsManager).lockAndTake(anyString(), any());
|
}).when(deletedAccountsManager).lockAndTake(anyString(), any());
|
||||||
|
|
||||||
accountsManager = new AccountsManager(
|
accountsManager = new AccountsManager(
|
||||||
accounts,
|
|
||||||
accountsDynamoDb,
|
accountsDynamoDb,
|
||||||
RedisClusterHelper.buildMockRedisCluster(commands),
|
RedisClusterHelper.buildMockRedisCluster(commands),
|
||||||
deletedAccountsManager,
|
deletedAccountsManager,
|
||||||
directoryQueue,
|
directoryQueue,
|
||||||
keys,
|
keys,
|
||||||
messagesManager,
|
messagesManager,
|
||||||
mock(MigrationMismatchedAccounts.class),
|
|
||||||
mock(UsernamesManager.class),
|
mock(UsernamesManager.class),
|
||||||
profilesManager,
|
profilesManager,
|
||||||
mock(StoredVerificationCodeManager.class),
|
mock(StoredVerificationCodeManager.class),
|
||||||
mock(SecureStorageClient.class),
|
mock(SecureStorageClient.class),
|
||||||
mock(SecureBackupClient.class),
|
mock(SecureBackupClient.class),
|
||||||
experimentEnrollmentManager,
|
|
||||||
dynamicConfigurationManager);
|
dynamicConfigurationManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testGetAccountByNumberInCache() {
|
||||||
void testGetAccountByNumberInCache(final boolean dynamoEnabled) {
|
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
|
|
||||||
enableDynamo(dynamoEnabled);
|
|
||||||
|
|
||||||
when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(uuid.toString());
|
when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(uuid.toString());
|
||||||
when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}");
|
when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}");
|
||||||
|
|
||||||
|
@ -150,31 +137,14 @@ class AccountsManagerTest {
|
||||||
verify(commands, times(1)).get(eq("AccountMap::+14152222222"));
|
verify(commands, times(1)).get(eq("AccountMap::+14152222222"));
|
||||||
verify(commands, times(1)).get(eq("Account3::" + uuid));
|
verify(commands, times(1)).get(eq("Account3::" + uuid));
|
||||||
verifyNoMoreInteractions(commands);
|
verifyNoMoreInteractions(commands);
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
|
|
||||||
verifyNoInteractions(accountsDynamoDb);
|
verifyNoInteractions(accountsDynamoDb);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void enableDynamo(boolean dynamoEnabled) {
|
@Test
|
||||||
final DynamicAccountsDynamoDbMigrationConfiguration config = dynamicConfigurationManager.getConfiguration()
|
void testGetAccountByUuidInCache() {
|
||||||
.getAccountsDynamoDbMigrationConfiguration();
|
|
||||||
|
|
||||||
config.setDeleteEnabled(dynamoEnabled);
|
|
||||||
config.setReadEnabled(dynamoEnabled);
|
|
||||||
config.setWriteEnabled(dynamoEnabled);
|
|
||||||
|
|
||||||
when(experimentEnrollmentManager.isEnrolled(any(UUID.class), anyString()))
|
|
||||||
.thenReturn(dynamoEnabled);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@ParameterizedTest
|
|
||||||
@ValueSource(booleans = {true, false})
|
|
||||||
void testGetAccountByUuidInCache(boolean dynamoEnabled) {
|
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
|
|
||||||
enableDynamo(dynamoEnabled);
|
|
||||||
|
|
||||||
when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}");
|
when(commands.get(eq("Account3::" + uuid))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}");
|
||||||
|
|
||||||
Optional<Account> account = accountsManager.get(uuid);
|
Optional<Account> account = accountsManager.get(uuid);
|
||||||
|
@ -186,22 +156,19 @@ class AccountsManagerTest {
|
||||||
|
|
||||||
verify(commands, times(1)).get(eq("Account3::" + uuid));
|
verify(commands, times(1)).get(eq("Account3::" + uuid));
|
||||||
verifyNoMoreInteractions(commands);
|
verifyNoMoreInteractions(commands);
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
|
|
||||||
verifyNoInteractions(accountsDynamoDb);
|
verifyNoInteractions(accountsDynamoDb);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testGetAccountByNumberNotInCache() {
|
||||||
void testGetAccountByNumberNotInCache(boolean dynamoEnabled) {
|
final boolean dynamoEnabled = true;
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
|
|
||||||
enableDynamo(dynamoEnabled);
|
|
||||||
|
|
||||||
when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(null);
|
when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(null);
|
||||||
when(accounts.get(eq("+14152222222"))).thenReturn(Optional.of(account));
|
when(accountsDynamoDb.get(eq("+14152222222"))).thenReturn(Optional.of(account));
|
||||||
|
|
||||||
Optional<Account> retrieved = accountsManager.get("+14152222222");
|
Optional<Account> retrieved = accountsManager.get("+14152222222");
|
||||||
|
|
||||||
|
@ -213,24 +180,18 @@ class AccountsManagerTest {
|
||||||
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
|
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
|
||||||
verifyNoMoreInteractions(commands);
|
verifyNoMoreInteractions(commands);
|
||||||
|
|
||||||
verify(accounts, times(1)).get(eq("+14152222222"));
|
verify(accountsDynamoDb, times(1)).get(eq("+14152222222"));
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
|
|
||||||
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never())
|
|
||||||
.get(eq("+14152222222"));
|
|
||||||
verifyNoMoreInteractions(accountsDynamoDb);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testGetAccountByUuidNotInCache() {
|
||||||
void testGetAccountByUuidNotInCache(boolean dynamoEnabled) {
|
final boolean dynamoEnabled = true;
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
|
|
||||||
enableDynamo(dynamoEnabled);
|
|
||||||
|
|
||||||
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
||||||
when(accounts.get(eq(uuid))).thenReturn(Optional.of(account));
|
when(accountsDynamoDb.get(eq(uuid))).thenReturn(Optional.of(account));
|
||||||
|
|
||||||
Optional<Account> retrieved = accountsManager.get(uuid);
|
Optional<Account> retrieved = accountsManager.get(uuid);
|
||||||
|
|
||||||
|
@ -242,25 +203,19 @@ class AccountsManagerTest {
|
||||||
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
|
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
|
||||||
verifyNoMoreInteractions(commands);
|
verifyNoMoreInteractions(commands);
|
||||||
|
|
||||||
verify(accounts, times(1)).get(eq(uuid));
|
verify(accountsDynamoDb, times(1)).get(eq(uuid));
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
|
|
||||||
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq(uuid));
|
|
||||||
verifyNoMoreInteractions(accountsDynamoDb);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testGetAccountByNumberBrokenCache() {
|
||||||
void testGetAccountByNumberBrokenCache(boolean dynamoEnabled) {
|
UUID uuid = UUID.randomUUID();
|
||||||
UUID uuid = UUID.randomUUID();
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
|
||||||
|
|
||||||
enableDynamo(dynamoEnabled);
|
|
||||||
|
|
||||||
when(commands.get(eq("AccountMap::+14152222222"))).thenThrow(new RedisException("Connection lost!"));
|
when(commands.get(eq("AccountMap::+14152222222"))).thenThrow(new RedisException("Connection lost!"));
|
||||||
when(accounts.get(eq("+14152222222"))).thenReturn(Optional.of(account));
|
when(accountsDynamoDb.get(eq("+14152222222"))).thenReturn(Optional.of(account));
|
||||||
|
|
||||||
Optional<Account> retrieved = accountsManager.get("+14152222222");
|
Optional<Account> retrieved = accountsManager.get("+14152222222");
|
||||||
|
|
||||||
assertTrue(retrieved.isPresent());
|
assertTrue(retrieved.isPresent());
|
||||||
assertSame(retrieved.get(), account);
|
assertSame(retrieved.get(), account);
|
||||||
|
@ -270,25 +225,20 @@ class AccountsManagerTest {
|
||||||
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
|
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
|
||||||
verifyNoMoreInteractions(commands);
|
verifyNoMoreInteractions(commands);
|
||||||
|
|
||||||
verify(accounts, times(1)).get(eq("+14152222222"));
|
verify(accountsDynamoDb, times(1)).get(eq("+14152222222"));
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
|
|
||||||
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq("+14152222222"));
|
|
||||||
verifyNoMoreInteractions(accountsDynamoDb);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
@Test
|
||||||
@ValueSource(booleans = {true, false})
|
void testGetAccountByUuidBrokenCache() {
|
||||||
void testGetAccountByUuidBrokenCache(boolean dynamoEnabled) {
|
final boolean dynamoEnabled = true;
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
|
|
||||||
enableDynamo(dynamoEnabled);
|
|
||||||
|
|
||||||
when(commands.get(eq("Account3::" + uuid))).thenThrow(new RedisException("Connection lost!"));
|
when(commands.get(eq("Account3::" + uuid))).thenThrow(new RedisException("Connection lost!"));
|
||||||
when(accounts.get(eq(uuid))).thenReturn(Optional.of(account));
|
when(accountsDynamoDb.get(eq(uuid))).thenReturn(Optional.of(account));
|
||||||
|
|
||||||
Optional<Account> retrieved = accountsManager.get(uuid);
|
Optional<Account> retrieved = accountsManager.get(uuid);
|
||||||
|
|
||||||
assertTrue(retrieved.isPresent());
|
assertTrue(retrieved.isPresent());
|
||||||
assertSame(retrieved.get(), account);
|
assertSame(retrieved.get(), account);
|
||||||
|
@ -298,26 +248,25 @@ class AccountsManagerTest {
|
||||||
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
|
verify(commands, times(1)).set(eq("Account3::" + uuid), anyString());
|
||||||
verifyNoMoreInteractions(commands);
|
verifyNoMoreInteractions(commands);
|
||||||
|
|
||||||
verify(accounts, times(1)).get(eq(uuid));
|
verify(accountsDynamoDb, times(1)).get(eq(uuid));
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
|
|
||||||
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(eq(uuid));
|
|
||||||
verifyNoMoreInteractions(accountsDynamoDb);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ParameterizedTest
|
// TODO delete
|
||||||
@ValueSource(booleans = {true, false})
|
@Disabled("migration specific")
|
||||||
void testUpdate_dynamoDbMigration(boolean dynamoEnabled) throws IOException {
|
@Test
|
||||||
|
void testUpdate_dynamoDbMigration() throws IOException {
|
||||||
|
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
|
|
||||||
enableDynamo(dynamoEnabled);
|
|
||||||
|
|
||||||
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
||||||
// database fetches should always return new instances
|
// database fetches should always return new instances
|
||||||
when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
when(accountsDynamoDb.get(uuid)).thenReturn(
|
||||||
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
||||||
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accounts).update(any(Account.class));
|
when(accountsDynamoDb.get(uuid)).thenReturn(
|
||||||
|
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
||||||
|
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accountsDynamoDb).update(any(Account.class));
|
||||||
|
|
||||||
Account updatedAccount = accountsManager.update(account, a -> a.setProfileName("name"));
|
Account updatedAccount = accountsManager.update(account, a -> a.setProfileName("name"));
|
||||||
|
|
||||||
|
@ -325,17 +274,13 @@ class AccountsManagerTest {
|
||||||
|
|
||||||
assertNotSame(updatedAccount, account);
|
assertNotSame(updatedAccount, account);
|
||||||
|
|
||||||
verify(accounts, times(1)).update(account);
|
verify(accountsDynamoDb, times(1)).update(account);
|
||||||
verifyNoMoreInteractions(accounts);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
|
|
||||||
if (dynamoEnabled) {
|
ArgumentCaptor<Account> argumentCaptor = ArgumentCaptor.forClass(Account.class);
|
||||||
ArgumentCaptor<Account> argumentCaptor = ArgumentCaptor.forClass(Account.class);
|
verify(accountsDynamoDb, times(1)).update(argumentCaptor.capture());
|
||||||
verify(accountsDynamoDb, times(1)).update(argumentCaptor.capture());
|
assertEquals(uuid, argumentCaptor.getValue().getUuid());
|
||||||
assertEquals(uuid, argumentCaptor.getValue().getUuid());
|
verify(accountsDynamoDb, times(1)).get(uuid);
|
||||||
} else {
|
|
||||||
verify(accountsDynamoDb, never()).update(any());
|
|
||||||
}
|
|
||||||
verify(accountsDynamoDb, dynamoEnabled ? times(1) : never()).get(uuid);
|
|
||||||
verifyNoMoreInteractions(accountsDynamoDb);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
|
|
||||||
ArgumentCaptor<String> redisSetArgumentCapture = ArgumentCaptor.forClass(String.class);
|
ArgumentCaptor<String> redisSetArgumentCapture = ArgumentCaptor.forClass(String.class);
|
||||||
|
@ -347,25 +292,26 @@ class AccountsManagerTest {
|
||||||
// uuid is @JsonIgnore, so we need to set it for compareAccounts to work
|
// uuid is @JsonIgnore, so we need to set it for compareAccounts to work
|
||||||
accountCached.setUuid(uuid);
|
accountCached.setUuid(uuid);
|
||||||
|
|
||||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.of(updatedAccount), Optional.of(accountCached)));
|
assertEquals(Optional.empty(),
|
||||||
|
accountsManager.compareAccounts(Optional.of(updatedAccount), Optional.of(accountCached)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO delete
|
||||||
|
@Disabled("migration specific")
|
||||||
@Test
|
@Test
|
||||||
void testUpdate_dynamoMissing() {
|
void testUpdate_dynamoMissing() {
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
|
|
||||||
enableDynamo(true);
|
|
||||||
|
|
||||||
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
||||||
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty());
|
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty());
|
||||||
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accounts).update(any());
|
|
||||||
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accountsDynamoDb).update(any());
|
doAnswer(ACCOUNT_UPDATE_ANSWER).when(accountsDynamoDb).update(any());
|
||||||
|
|
||||||
Account updatedAccount = accountsManager.update(account, a -> {});
|
Account updatedAccount = accountsManager.update(account, a -> {
|
||||||
|
});
|
||||||
|
|
||||||
verify(accounts, times(1)).update(account);
|
verify(accountsDynamoDb, times(1)).update(account);
|
||||||
verifyNoMoreInteractions(accounts);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
|
|
||||||
verify(accountsDynamoDb, never()).update(account);
|
verify(accountsDynamoDb, never()).update(account);
|
||||||
verify(accountsDynamoDb, times(1)).get(uuid);
|
verify(accountsDynamoDb, times(1)).get(uuid);
|
||||||
|
@ -376,19 +322,19 @@ class AccountsManagerTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testUpdate_optimisticLockingFailure() {
|
void testUpdate_optimisticLockingFailure() {
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
|
|
||||||
enableDynamo(true);
|
|
||||||
|
|
||||||
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
||||||
|
|
||||||
when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
when(accountsDynamoDb.get(uuid)).thenReturn(
|
||||||
|
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
||||||
doThrow(ContestedOptimisticLockException.class)
|
doThrow(ContestedOptimisticLockException.class)
|
||||||
.doAnswer(ACCOUNT_UPDATE_ANSWER)
|
.doAnswer(ACCOUNT_UPDATE_ANSWER)
|
||||||
.when(accounts).update(any());
|
.when(accountsDynamoDb).update(any());
|
||||||
|
|
||||||
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
when(accountsDynamoDb.get(uuid)).thenReturn(
|
||||||
|
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
||||||
doThrow(ContestedOptimisticLockException.class)
|
doThrow(ContestedOptimisticLockException.class)
|
||||||
.doAnswer(ACCOUNT_UPDATE_ANSWER)
|
.doAnswer(ACCOUNT_UPDATE_ANSWER)
|
||||||
.when(accountsDynamoDb).update(any());
|
.when(accountsDynamoDb).update(any());
|
||||||
|
@ -398,12 +344,7 @@ class AccountsManagerTest {
|
||||||
assertEquals(1, account.getVersion());
|
assertEquals(1, account.getVersion());
|
||||||
assertEquals("name", account.getProfileName());
|
assertEquals("name", account.getProfileName());
|
||||||
|
|
||||||
verify(accounts, times(1)).get(uuid);
|
verify(accountsDynamoDb, times(1)).get(uuid);
|
||||||
verify(accounts, times(2)).update(any());
|
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
|
|
||||||
// dynamo has an extra get() because the account is fetched before every update
|
|
||||||
verify(accountsDynamoDb, times(2)).get(uuid);
|
|
||||||
verify(accountsDynamoDb, times(2)).update(any());
|
verify(accountsDynamoDb, times(2)).update(any());
|
||||||
verifyNoMoreInteractions(accountsDynamoDb);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
}
|
}
|
||||||
|
@ -413,8 +354,6 @@ class AccountsManagerTest {
|
||||||
UUID uuid = UUID.randomUUID();
|
UUID uuid = UUID.randomUUID();
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
|
|
||||||
enableDynamo(true);
|
|
||||||
|
|
||||||
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
when(commands.get(eq("Account3::" + uuid))).thenReturn(null);
|
||||||
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty())
|
when(accountsDynamoDb.get(uuid)).thenReturn(Optional.empty())
|
||||||
.thenReturn(Optional.of(account));
|
.thenReturn(Optional.of(account));
|
||||||
|
@ -422,10 +361,7 @@ class AccountsManagerTest {
|
||||||
|
|
||||||
accountsManager.update(account, a -> {});
|
accountsManager.update(account, a -> {});
|
||||||
|
|
||||||
verify(accounts, times(1)).update(account);
|
verify(accountsDynamoDb, times(1)).update(account);
|
||||||
verifyNoMoreInteractions(accounts);
|
|
||||||
|
|
||||||
verify(accountsDynamoDb, times(1)).get(uuid);
|
|
||||||
verifyNoMoreInteractions(accountsDynamoDb);
|
verifyNoMoreInteractions(accountsDynamoDb);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -436,7 +372,8 @@ class AccountsManagerTest {
|
||||||
final UUID uuid = UUID.randomUUID();
|
final UUID uuid = UUID.randomUUID();
|
||||||
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
Account account = new Account("+14152222222", uuid, new HashSet<>(), new byte[16]);
|
||||||
|
|
||||||
when(accounts.get(uuid)).thenReturn(Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
when(accountsDynamoDb.get(uuid)).thenReturn(
|
||||||
|
Optional.of(new Account("+14152222222", uuid, new HashSet<>(), new byte[16])));
|
||||||
|
|
||||||
assertTrue(account.getDevices().isEmpty());
|
assertTrue(account.getDevices().isEmpty());
|
||||||
|
|
||||||
|
@ -463,6 +400,8 @@ class AccountsManagerTest {
|
||||||
verify(unknownDeviceUpdater, never()).accept(any(Device.class));
|
verify(unknownDeviceUpdater, never()).accept(any(Device.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO delete
|
||||||
|
@Disabled("migration specific")
|
||||||
@Test
|
@Test
|
||||||
void testCompareAccounts() throws Exception {
|
void testCompareAccounts() throws Exception {
|
||||||
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.empty(), Optional.empty()));
|
assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.empty(), Optional.empty()));
|
||||||
|
@ -538,13 +477,13 @@ class AccountsManagerTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testCreateFreshAccount() throws InterruptedException {
|
void testCreateFreshAccount() throws InterruptedException {
|
||||||
when(accounts.create(any())).thenReturn(true);
|
when(accountsDynamoDb.create(any())).thenReturn(true);
|
||||||
|
|
||||||
final String e164 = "+18005550123";
|
final String e164 = "+18005550123";
|
||||||
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
|
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
|
||||||
accountsManager.create(e164, "password", null, attributes);
|
accountsManager.create(e164, "password", null, attributes);
|
||||||
|
|
||||||
verify(accounts).create(argThat(account -> e164.equals(account.getNumber())));
|
verify(accountsDynamoDb).create(argThat(account -> e164.equals(account.getNumber())));
|
||||||
verifyNoInteractions(keys);
|
verifyNoInteractions(keys);
|
||||||
verifyNoInteractions(messagesManager);
|
verifyNoInteractions(messagesManager);
|
||||||
verifyNoInteractions(profilesManager);
|
verifyNoInteractions(profilesManager);
|
||||||
|
@ -554,7 +493,7 @@ class AccountsManagerTest {
|
||||||
void testReregisterAccount() throws InterruptedException {
|
void testReregisterAccount() throws InterruptedException {
|
||||||
final UUID existingUuid = UUID.randomUUID();
|
final UUID existingUuid = UUID.randomUUID();
|
||||||
|
|
||||||
when(accounts.create(any())).thenAnswer(invocation -> {
|
when(accountsDynamoDb.create(any())).thenAnswer(invocation -> {
|
||||||
invocation.getArgument(0, Account.class).setUuid(existingUuid);
|
invocation.getArgument(0, Account.class).setUuid(existingUuid);
|
||||||
return false;
|
return false;
|
||||||
});
|
});
|
||||||
|
@ -563,7 +502,8 @@ class AccountsManagerTest {
|
||||||
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
|
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
|
||||||
accountsManager.create(e164, "password", null, attributes);
|
accountsManager.create(e164, "password", null, attributes);
|
||||||
|
|
||||||
verify(accounts).create(argThat(account -> e164.equals(account.getNumber()) && existingUuid.equals(account.getUuid())));
|
verify(accountsDynamoDb).create(
|
||||||
|
argThat(account -> e164.equals(account.getNumber()) && existingUuid.equals(account.getUuid())));
|
||||||
verify(keys).delete(existingUuid);
|
verify(keys).delete(existingUuid);
|
||||||
verify(messagesManager).clear(existingUuid);
|
verify(messagesManager).clear(existingUuid);
|
||||||
verify(profilesManager).deleteAll(existingUuid);
|
verify(profilesManager).deleteAll(existingUuid);
|
||||||
|
@ -579,13 +519,14 @@ class AccountsManagerTest {
|
||||||
return null;
|
return null;
|
||||||
}).when(deletedAccountsManager).lockAndTake(anyString(), any());
|
}).when(deletedAccountsManager).lockAndTake(anyString(), any());
|
||||||
|
|
||||||
when(accounts.create(any())).thenReturn(true);
|
when(accountsDynamoDb.create(any())).thenReturn(true);
|
||||||
|
|
||||||
final String e164 = "+18005550123";
|
final String e164 = "+18005550123";
|
||||||
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
|
final AccountAttributes attributes = new AccountAttributes(false, 0, null, null, true, null);
|
||||||
accountsManager.create(e164, "password", null, attributes);
|
accountsManager.create(e164, "password", null, attributes);
|
||||||
|
|
||||||
verify(accounts).create(argThat(account -> e164.equals(account.getNumber()) && recentlyDeletedUuid.equals(account.getUuid())));
|
verify(accountsDynamoDb).create(
|
||||||
|
argThat(account -> e164.equals(account.getNumber()) && recentlyDeletedUuid.equals(account.getUuid())));
|
||||||
verifyNoInteractions(keys);
|
verifyNoInteractions(keys);
|
||||||
verifyNoInteractions(messagesManager);
|
verifyNoInteractions(messagesManager);
|
||||||
verifyNoInteractions(profilesManager);
|
verifyNoInteractions(profilesManager);
|
||||||
|
@ -634,6 +575,7 @@ class AccountsManagerTest {
|
||||||
verify(directoryQueue, times(expectRefresh ? 1 : 0)).refreshAccount(updatedAccount);
|
verify(directoryQueue, times(expectRefresh ? 1 : 0)).refreshAccount(updatedAccount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unused")
|
||||||
private static Stream<Arguments> testUpdateDirectoryQueue() {
|
private static Stream<Arguments> testUpdateDirectoryQueue() {
|
||||||
return Stream.of(
|
return Stream.of(
|
||||||
Arguments.of(false, false, false),
|
Arguments.of(false, false, false),
|
||||||
|
@ -654,7 +596,7 @@ class AccountsManagerTest {
|
||||||
accountsManager.updateDeviceLastSeen(account, device, updatedLastSeen);
|
accountsManager.updateDeviceLastSeen(account, device, updatedLastSeen);
|
||||||
|
|
||||||
assertEquals(expectUpdate ? updatedLastSeen : initialLastSeen, device.getLastSeen());
|
assertEquals(expectUpdate ? updatedLastSeen : initialLastSeen, device.getLastSeen());
|
||||||
verify(accounts, expectUpdate ? times(1) : never()).update(account);
|
verify(accountsDynamoDb, expectUpdate ? times(1) : never()).update(account);
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
|
|
|
@ -1,386 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2013-2020 Signal Messenger, LLC
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.whispersystems.textsecuregcm.tests.storage;
|
|
||||||
|
|
||||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
|
||||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
|
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
|
||||||
import static org.mockito.Mockito.doThrow;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
|
|
||||||
import com.fasterxml.uuid.UUIDComparator;
|
|
||||||
import com.opentable.db.postgres.embedded.LiquibasePreparer;
|
|
||||||
import com.opentable.db.postgres.junit.EmbeddedPostgresRules;
|
|
||||||
import com.opentable.db.postgres.junit.PreparedDbRule;
|
|
||||||
import io.github.resilience4j.circuitbreaker.CallNotPermittedException;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.sql.PreparedStatement;
|
|
||||||
import java.sql.ResultSet;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.Random;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.UUID;
|
|
||||||
import org.jdbi.v3.core.HandleConsumer;
|
|
||||||
import org.jdbi.v3.core.Jdbi;
|
|
||||||
import org.jdbi.v3.core.transaction.TransactionException;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
|
|
||||||
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Account;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.AccountCrawlChunk;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.Device;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
|
|
||||||
import org.whispersystems.textsecuregcm.storage.mappers.AccountRowMapper;
|
|
||||||
|
|
||||||
public class AccountsTest {
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml"));
|
|
||||||
|
|
||||||
private Accounts accounts;
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void setupAccountsDao() {
|
|
||||||
FaultTolerantDatabase faultTolerantDatabase = new FaultTolerantDatabase("accountsTest",
|
|
||||||
Jdbi.create(db.getTestDatabase()),
|
|
||||||
new CircuitBreakerConfiguration());
|
|
||||||
|
|
||||||
this.accounts = new Accounts(faultTolerantDatabase);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testStore() throws SQLException, IOException {
|
|
||||||
Device device = generateDevice (1 );
|
|
||||||
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
|
|
||||||
|
|
||||||
boolean freshUser = accounts.create(account);
|
|
||||||
assertThat(freshUser).isTrue();
|
|
||||||
|
|
||||||
PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?");
|
|
||||||
verifyStoredState(statement, "+14151112222", account.getUuid(), account);
|
|
||||||
|
|
||||||
freshUser = accounts.create(account);
|
|
||||||
assertThat(freshUser).isTrue();
|
|
||||||
|
|
||||||
statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?");
|
|
||||||
verifyStoredState(statement, "+14151112222", account.getUuid(), account);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testStoreMulti() throws SQLException, IOException {
|
|
||||||
Set<Device> devices = new HashSet<>();
|
|
||||||
devices.add(generateDevice(1));
|
|
||||||
devices.add(generateDevice(2));
|
|
||||||
|
|
||||||
Account account = generateAccount("+14151112222", UUID.randomUUID(), devices);
|
|
||||||
|
|
||||||
accounts.create(account);
|
|
||||||
|
|
||||||
PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?");
|
|
||||||
verifyStoredState(statement, "+14151112222", account.getUuid(), account);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testRetrieve() {
|
|
||||||
Set<Device> devicesFirst = new HashSet<>();
|
|
||||||
devicesFirst.add(generateDevice(1));
|
|
||||||
devicesFirst.add(generateDevice(2));
|
|
||||||
|
|
||||||
UUID uuidFirst = UUID.randomUUID();
|
|
||||||
Account accountFirst = generateAccount("+14151112222", uuidFirst, devicesFirst);
|
|
||||||
|
|
||||||
Set<Device> devicesSecond = new HashSet<>();
|
|
||||||
devicesSecond.add(generateDevice(1));
|
|
||||||
devicesSecond.add(generateDevice(2));
|
|
||||||
|
|
||||||
UUID uuidSecond = UUID.randomUUID();
|
|
||||||
Account accountSecond = generateAccount("+14152221111", uuidSecond, devicesSecond);
|
|
||||||
|
|
||||||
accounts.create(accountFirst);
|
|
||||||
accounts.create(accountSecond);
|
|
||||||
|
|
||||||
Optional<Account> retrievedFirst = accounts.get("+14151112222");
|
|
||||||
Optional<Account> retrievedSecond = accounts.get("+14152221111");
|
|
||||||
|
|
||||||
assertThat(retrievedFirst.isPresent()).isTrue();
|
|
||||||
assertThat(retrievedSecond.isPresent()).isTrue();
|
|
||||||
|
|
||||||
verifyStoredState("+14151112222", uuidFirst, retrievedFirst.get(), accountFirst);
|
|
||||||
verifyStoredState("+14152221111", uuidSecond, retrievedSecond.get(), accountSecond);
|
|
||||||
|
|
||||||
retrievedFirst = accounts.get(uuidFirst);
|
|
||||||
retrievedSecond = accounts.get(uuidSecond);
|
|
||||||
|
|
||||||
assertThat(retrievedFirst.isPresent()).isTrue();
|
|
||||||
assertThat(retrievedSecond.isPresent()).isTrue();
|
|
||||||
|
|
||||||
verifyStoredState("+14151112222", uuidFirst, retrievedFirst.get(), accountFirst);
|
|
||||||
verifyStoredState("+14152221111", uuidSecond, retrievedSecond.get(), accountSecond);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testOverwrite() throws Exception {
|
|
||||||
Device device = generateDevice (1 );
|
|
||||||
UUID firstUuid = UUID.randomUUID();
|
|
||||||
Account account = generateAccount("+14151112222", firstUuid, Collections.singleton(device));
|
|
||||||
|
|
||||||
accounts.create(account);
|
|
||||||
|
|
||||||
PreparedStatement statement = db.getTestDatabase().getConnection().prepareStatement("SELECT * FROM accounts WHERE number = ?");
|
|
||||||
verifyStoredState(statement, "+14151112222", account.getUuid(), account);
|
|
||||||
|
|
||||||
UUID secondUuid = UUID.randomUUID();
|
|
||||||
|
|
||||||
device = generateDevice(1);
|
|
||||||
account = generateAccount("+14151112222", secondUuid, Collections.singleton(device));
|
|
||||||
|
|
||||||
final boolean freshUser = accounts.create(account);
|
|
||||||
assertThat(freshUser).isFalse();
|
|
||||||
verifyStoredState(statement, "+14151112222", firstUuid, account);
|
|
||||||
|
|
||||||
device = generateDevice(1);
|
|
||||||
Account invalidAccount = generateAccount("+14151113333", firstUuid, Collections.singleton(device));
|
|
||||||
|
|
||||||
assertThatThrownBy(() -> accounts.create(invalidAccount));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testUpdate() {
|
|
||||||
Device device = generateDevice (1 );
|
|
||||||
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
|
|
||||||
|
|
||||||
accounts.create(account);
|
|
||||||
|
|
||||||
device.setName("foobar");
|
|
||||||
|
|
||||||
accounts.update(account);
|
|
||||||
|
|
||||||
account.setProfileName("profileName");
|
|
||||||
|
|
||||||
accounts.update(account);
|
|
||||||
|
|
||||||
assertThat(account.getVersion()).isEqualTo(2);
|
|
||||||
|
|
||||||
Optional<Account> retrieved = accounts.get("+14151112222");
|
|
||||||
|
|
||||||
assertThat(retrieved.isPresent()).isTrue();
|
|
||||||
verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account);
|
|
||||||
|
|
||||||
retrieved = accounts.get(account.getUuid());
|
|
||||||
|
|
||||||
assertThat(retrieved.isPresent()).isTrue();
|
|
||||||
verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testRetrieveFrom() {
|
|
||||||
List<Account> users = new ArrayList<>();
|
|
||||||
|
|
||||||
for (int i=1;i<=100;i++) {
|
|
||||||
Account account = generateAccount("+1" + String.format("%03d", i), UUID.randomUUID());
|
|
||||||
users.add(account);
|
|
||||||
accounts.create(account);
|
|
||||||
}
|
|
||||||
|
|
||||||
users.sort((account, t1) -> UUIDComparator.staticCompare(account.getUuid(), t1.getUuid()));
|
|
||||||
|
|
||||||
AccountCrawlChunk retrieved = accounts.getAllFrom(10);
|
|
||||||
assertThat(retrieved.getAccounts().size()).isEqualTo(10);
|
|
||||||
|
|
||||||
for (int i=0;i<retrieved.getAccounts().size();i++) {
|
|
||||||
verifyStoredState(users.get(i).getNumber(), users.get(i).getUuid(), retrieved.getAccounts().get(i), users.get(i));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int j=0;j<9;j++) {
|
|
||||||
retrieved = accounts.getAllFrom(retrieved.getLastUuid().orElseThrow(), 10);
|
|
||||||
assertThat(retrieved.getAccounts().size()).isEqualTo(10);
|
|
||||||
|
|
||||||
for (int i=0;i<retrieved.getAccounts().size();i++) {
|
|
||||||
verifyStoredState(users.get(10 + (j * 10) + i).getNumber(), users.get(10 + (j * 10) + i).getUuid(), retrieved.getAccounts().get(i), users.get(10 + (j * 10) + i));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testDelete() {
|
|
||||||
final Device deletedDevice = generateDevice (1);
|
|
||||||
final Account deletedAccount = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(deletedDevice));
|
|
||||||
final Device retainedDevice = generateDevice (1);
|
|
||||||
final Account retainedAccount = generateAccount("+14151112345", UUID.randomUUID(), Collections.singleton(retainedDevice));
|
|
||||||
|
|
||||||
accounts.create(deletedAccount);
|
|
||||||
accounts.create(retainedAccount);
|
|
||||||
|
|
||||||
assertThat(accounts.get(deletedAccount.getUuid())).isPresent();
|
|
||||||
assertThat(accounts.get(retainedAccount.getUuid())).isPresent();
|
|
||||||
|
|
||||||
accounts.delete(deletedAccount.getUuid());
|
|
||||||
|
|
||||||
assertThat(accounts.get(deletedAccount.getUuid())).isNotPresent();
|
|
||||||
|
|
||||||
verifyStoredState(retainedAccount.getNumber(), retainedAccount.getUuid(), accounts.get(retainedAccount.getUuid()).get(), retainedAccount);
|
|
||||||
|
|
||||||
{
|
|
||||||
final Account recreatedAccount = generateAccount(deletedAccount.getNumber(), UUID.randomUUID(),
|
|
||||||
Collections.singleton(generateDevice(1)));
|
|
||||||
|
|
||||||
final boolean freshUser = accounts.create(recreatedAccount);
|
|
||||||
|
|
||||||
assertThat(freshUser).isTrue();
|
|
||||||
|
|
||||||
assertThat(accounts.get(recreatedAccount.getUuid())).isPresent();
|
|
||||||
verifyStoredState(recreatedAccount.getNumber(), recreatedAccount.getUuid(),
|
|
||||||
accounts.get(recreatedAccount.getUuid()).get(), recreatedAccount);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testVacuum() {
|
|
||||||
Device device = generateDevice (1 );
|
|
||||||
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
|
|
||||||
|
|
||||||
accounts.create(account);
|
|
||||||
accounts.vacuum();
|
|
||||||
|
|
||||||
Optional<Account> retrieved = accounts.get("+14151112222");
|
|
||||||
assertThat(retrieved.isPresent()).isTrue();
|
|
||||||
|
|
||||||
verifyStoredState("+14151112222", account.getUuid(), retrieved.get(), account);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testMissing() {
|
|
||||||
Device device = generateDevice (1 );
|
|
||||||
Account account = generateAccount("+14151112222", UUID.randomUUID(), Collections.singleton(device));
|
|
||||||
|
|
||||||
accounts.create(account);
|
|
||||||
|
|
||||||
Optional<Account> retrieved = accounts.get("+11111111");
|
|
||||||
assertThat(retrieved.isPresent()).isFalse();
|
|
||||||
|
|
||||||
retrieved = accounts.get(UUID.randomUUID());
|
|
||||||
assertThat(retrieved.isPresent()).isFalse();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testBreaker() throws InterruptedException {
|
|
||||||
Jdbi jdbi = mock(Jdbi.class);
|
|
||||||
doThrow(new TransactionException("Database error!")).when(jdbi).useHandle(any(HandleConsumer.class));
|
|
||||||
|
|
||||||
CircuitBreakerConfiguration configuration = new CircuitBreakerConfiguration();
|
|
||||||
configuration.setWaitDurationInOpenStateInSeconds(1);
|
|
||||||
configuration.setRingBufferSizeInHalfOpenState(1);
|
|
||||||
configuration.setRingBufferSizeInClosedState(2);
|
|
||||||
configuration.setFailureRateThreshold(50);
|
|
||||||
|
|
||||||
Accounts accounts = new Accounts(new FaultTolerantDatabase("testAccountBreaker", jdbi, configuration));
|
|
||||||
Account account = generateAccount("+14151112222", UUID.randomUUID());
|
|
||||||
|
|
||||||
try {
|
|
||||||
accounts.update(account);
|
|
||||||
throw new AssertionError();
|
|
||||||
} catch (TransactionException e) {
|
|
||||||
// good
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
accounts.update(account);
|
|
||||||
throw new AssertionError();
|
|
||||||
} catch (TransactionException e) {
|
|
||||||
// good
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
accounts.update(account);
|
|
||||||
throw new AssertionError();
|
|
||||||
} catch (CallNotPermittedException e) {
|
|
||||||
// good
|
|
||||||
}
|
|
||||||
|
|
||||||
Thread.sleep(1100);
|
|
||||||
|
|
||||||
try {
|
|
||||||
accounts.update(account);
|
|
||||||
throw new AssertionError();
|
|
||||||
} catch (TransactionException e) {
|
|
||||||
// good
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private Device generateDevice(long id) {
|
|
||||||
Random random = new Random(System.currentTimeMillis());
|
|
||||||
SignedPreKey signedPreKey = new SignedPreKey(random.nextInt(), "testPublicKey-" + random.nextInt(), "testSignature-" + random.nextInt());
|
|
||||||
return new Device(id, "testName-" + random.nextInt(), "testAuthToken-" + random.nextInt(), "testSalt-" + random.nextInt(),
|
|
||||||
"testGcmId-" + random.nextInt(), "testApnId-" + random.nextInt(), "testVoipApnId-" + random.nextInt(), random.nextBoolean(), random.nextInt(), signedPreKey, random.nextInt(), random.nextInt(), "testUserAgent-" + random.nextInt() , 0, new Device.DeviceCapabilities(random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
|
|
||||||
random.nextBoolean(), random.nextBoolean(), random.nextBoolean()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Account generateAccount(String number, UUID uuid) {
|
|
||||||
Device device = generateDevice(1);
|
|
||||||
return generateAccount(number, uuid, Collections.singleton(device));
|
|
||||||
}
|
|
||||||
|
|
||||||
private Account generateAccount(String number, UUID uuid, Set<Device> devices) {
|
|
||||||
byte[] unidentifiedAccessKey = new byte[16];
|
|
||||||
Random random = new Random(System.currentTimeMillis());
|
|
||||||
Arrays.fill(unidentifiedAccessKey, (byte)random.nextInt(255));
|
|
||||||
|
|
||||||
return new Account(number, uuid, devices, unidentifiedAccessKey);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void verifyStoredState(PreparedStatement statement, String number, UUID uuid, Account expecting)
|
|
||||||
throws SQLException, IOException
|
|
||||||
{
|
|
||||||
statement.setString(1, number);
|
|
||||||
|
|
||||||
ResultSet resultSet = statement.executeQuery();
|
|
||||||
|
|
||||||
if (resultSet.next()) {
|
|
||||||
String data = resultSet.getString("data");
|
|
||||||
assertThat(data).isNotEmpty();
|
|
||||||
|
|
||||||
Account result = new AccountRowMapper().map(resultSet, null);
|
|
||||||
verifyStoredState(number, uuid, result, expecting);
|
|
||||||
} else {
|
|
||||||
throw new AssertionError("No data");
|
|
||||||
}
|
|
||||||
|
|
||||||
assertThat(resultSet.next()).isFalse();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void verifyStoredState(String number, UUID uuid, Account result, Account expecting) {
|
|
||||||
assertThat(result.getNumber()).isEqualTo(number);
|
|
||||||
assertThat(result.getLastSeen()).isEqualTo(expecting.getLastSeen());
|
|
||||||
assertThat(result.getUuid()).isEqualTo(uuid);
|
|
||||||
assertThat(result.getVersion()).isEqualTo(expecting.getVersion());
|
|
||||||
assertThat(Arrays.equals(result.getUnidentifiedAccessKey().get(), expecting.getUnidentifiedAccessKey().get())).isTrue();
|
|
||||||
|
|
||||||
for (Device expectingDevice : expecting.getDevices()) {
|
|
||||||
Device resultDevice = result.getDevice(expectingDevice.getId()).get();
|
|
||||||
assertThat(resultDevice.getApnId()).isEqualTo(expectingDevice.getApnId());
|
|
||||||
assertThat(resultDevice.getGcmId()).isEqualTo(expectingDevice.getGcmId());
|
|
||||||
assertThat(resultDevice.getLastSeen()).isEqualTo(expectingDevice.getLastSeen());
|
|
||||||
assertThat(resultDevice.getSignedPreKey().getPublicKey()).isEqualTo(expectingDevice.getSignedPreKey().getPublicKey());
|
|
||||||
assertThat(resultDevice.getSignedPreKey().getKeyId()).isEqualTo(expectingDevice.getSignedPreKey().getKeyId());
|
|
||||||
assertThat(resultDevice.getSignedPreKey().getSignature()).isEqualTo(expectingDevice.getSignedPreKey().getSignature());
|
|
||||||
assertThat(resultDevice.getFetchesMessages()).isEqualTo(expectingDevice.getFetchesMessages());
|
|
||||||
assertThat(resultDevice.getUserAgent()).isEqualTo(expectingDevice.getUserAgent());
|
|
||||||
assertThat(resultDevice.getName()).isEqualTo(expectingDevice.getName());
|
|
||||||
assertThat(resultDevice.getCreated()).isEqualTo(expectingDevice.getCreated());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
Loading…
Reference in New Issue