From 2d1ca986053590eff34a9d8519e0d33fc9615192 Mon Sep 17 00:00:00 2001 From: Jon Chambers Date: Wed, 8 Dec 2021 17:32:19 -0500 Subject: [PATCH] Consolidate DynamoDB clients/configuration --- abusive-message-filter | 2 +- service/config/sample.yml | 84 +++++------ .../WhisperServerConfiguration.java | 103 -------------- .../textsecuregcm/WhisperServerService.java | 93 ++++-------- .../AccountsDynamoDbConfiguration.java | 38 ----- .../AccountsTableConfiguration.java | 49 +++++++ .../DeletedAccountsDynamoDbConfiguration.java | 13 -- .../DeletedAccountsTableConfiguration.java | 25 ++++ .../configuration/DynamoDbConfiguration.java | 44 ------ .../configuration/DynamoDbTables.java | 133 +++++++++++++++--- .../MessageDynamoDbConfiguration.java | 19 --- .../util/DynamoDbFromConfig.java | 12 -- .../workers/AssignUsernameCommand.java | 94 +++++-------- .../workers/DeleteUserCommand.java | 78 ++++------ .../workers/ReserveUsernameCommand.java | 6 +- .../SetUserDiscoverabilityCommand.java | 76 ++++------ 16 files changed, 340 insertions(+), 529 deletions(-) delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/AccountsDynamoDbConfiguration.java create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/AccountsTableConfiguration.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/DeletedAccountsDynamoDbConfiguration.java create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/DeletedAccountsTableConfiguration.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/DynamoDbConfiguration.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/MessageDynamoDbConfiguration.java diff --git a/abusive-message-filter b/abusive-message-filter index 82b7b374c..e24dabb53 160000 --- a/abusive-message-filter +++ b/abusive-message-filter @@ -1 +1 @@ -Subproject commit 82b7b374c57cf993f9a188d7a5ee7e7be83beb36 +Subproject commit e24dabb53940738bdbf67b9b7ac74d39fe876a58 diff --git a/service/config/sample.yml b/service/config/sample.yml index 88c2d1060..1a2e55a36 100644 --- a/service/config/sample.yml +++ b/service/config/sample.yml @@ -13,19 +13,47 @@ dynamoDbClientConfiguration: region: us-west-2 # AWS Region dynamoDbTables: + accounts: + tableName: Example_Accounts + phoneNumberTableName: Example_Accounts_PhoneNumbers + phoneNumberIdentifierTableName: Example_Accounts_PhoneNumberIdentifiers + usernamesTableName: Example_Accounts_Usernames + scanPageSize: 100 + deletedAccounts: + tableName: Example_DeletedAccounts + needsReconciliationIndexName: NeedsReconciliation + deletedAccountsLock: + tableName: Example_DeletedAccountsLock issuedReceipts: tableName: Example_IssuedReceipts expiration: P30D # Duration of time until rows expire generator: abcdefg12345678= # random base64-encoded binary sequence + keys: + tableName: Example_Keys + messages: + tableName: Example_Messages + expiration: P30D # Duration of time until rows expire + pendingAccounts: + tableName: Example_PendingAccounts + pendingDevices: + tableName: Example_PendingDevices + phoneNumberIdentifiers: + tableName: Example_PhoneNumberIdentifiers + profiles: + tableName: Example_Profiles + pushChallenge: + tableName: Example_PushChallenge redeemedReceipts: tableName: Example_RedeemedReceipts expiration: P30D # Duration of time until rows expire - subscriptions: - tableName: Example_Subscriptions - profiles: - tableName: Example_Profiles remoteConfig: tableName: Example_RemoteConfig + reportMessage: + tableName: Example_ReportMessage + reservedUsernames: + tableName: Example_ReservedUsernames + subscriptions: + tableName: Example_Subscriptions twilio: # Twilio gateway configuration accountId: unset @@ -126,54 +154,6 @@ messageCache: # Redis server configuration for message store cache metricsCluster: configurationUri: redis://redis.example.com:6379/ -messageDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_Messages - -keysDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_PreKeys - -accountsDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_Accounts - phoneNumberTableName: Example_Accounts_PhoneNumbers - phoneNumberIdentifierTableName: Example_Accounts_PhoneNumberIdentifiers - usernamesTableName: Example_Accounts_Usernames - -deletedAccountsDynamoDb: # DynamoDb table configuration - region: us-west-2 - tableName: Example_DeletedAccounts - needsReconciliationIndexName: NeedsReconciliation - -deletedAccountsLockDynamoDb: # DynamoDb table configuration - region: us-west-2 - tableName: Example_DeletedAccountsLock - -pendingAccountsDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_PendingAccounts - -pendingDevicesDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_PendingDevices - -reservedUsernamesDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_ReservedUsernames - -phoneNumberIdentifiersDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_PhoneNumberIdentifiers - -pushChallengeDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_PushChallenges - -reportMessageDynamoDb: # DynamoDB table configuration - region: us-west-2 - tableName: Example_ReportedMessages - awsAttachments: # AWS S3 configuration accessKey: test accessSecret: test diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index f60b92628..70897c446 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -15,7 +15,6 @@ import javax.validation.Valid; import javax.validation.constraints.NotNull; import org.whispersystems.textsecuregcm.configuration.AbusiveMessageFilterConfiguration; import org.whispersystems.textsecuregcm.configuration.AccountDatabaseCrawlerConfiguration; -import org.whispersystems.textsecuregcm.configuration.AccountsDynamoDbConfiguration; import org.whispersystems.textsecuregcm.configuration.ApnConfiguration; import org.whispersystems.textsecuregcm.configuration.AppConfigConfiguration; import org.whispersystems.textsecuregcm.configuration.AwsAttachmentsConfiguration; @@ -24,18 +23,15 @@ import org.whispersystems.textsecuregcm.configuration.BoostConfiguration; import org.whispersystems.textsecuregcm.configuration.CdnConfiguration; import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration; import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration; -import org.whispersystems.textsecuregcm.configuration.DeletedAccountsDynamoDbConfiguration; import org.whispersystems.textsecuregcm.configuration.DirectoryConfiguration; import org.whispersystems.textsecuregcm.configuration.DirectoryV2Configuration; import org.whispersystems.textsecuregcm.configuration.DonationConfiguration; import org.whispersystems.textsecuregcm.configuration.DynamoDbClientConfiguration; -import org.whispersystems.textsecuregcm.configuration.DynamoDbConfiguration; import org.whispersystems.textsecuregcm.configuration.DynamoDbTables; import org.whispersystems.textsecuregcm.configuration.GcmConfiguration; import org.whispersystems.textsecuregcm.configuration.GcpAttachmentsConfiguration; import org.whispersystems.textsecuregcm.configuration.MaxDeviceConfiguration; import org.whispersystems.textsecuregcm.configuration.MessageCacheConfiguration; -import org.whispersystems.textsecuregcm.configuration.MessageDynamoDbConfiguration; import org.whispersystems.textsecuregcm.configuration.PaymentsServiceConfiguration; import org.whispersystems.textsecuregcm.configuration.PushConfiguration; import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration; @@ -155,61 +151,6 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private RedisClusterConfiguration clientPresenceCluster; - @Valid - @NotNull - @JsonProperty - private MessageDynamoDbConfiguration messageDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DynamoDbConfiguration keysDynamoDb; - - @Valid - @NotNull - @JsonProperty - private AccountsDynamoDbConfiguration accountsDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DynamoDbConfiguration phoneNumberIdentifiersDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DeletedAccountsDynamoDbConfiguration deletedAccountsDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DynamoDbConfiguration deletedAccountsLockDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DynamoDbConfiguration pushChallengeDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DynamoDbConfiguration reportMessageDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DynamoDbConfiguration pendingAccountsDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DynamoDbConfiguration pendingDevicesDynamoDb; - - @Valid - @NotNull - @JsonProperty - private DynamoDbConfiguration reservedUsernamesDynamoDb; - @Valid @NotNull @JsonProperty @@ -428,30 +369,6 @@ public class WhisperServerConfiguration extends Configuration { return rateLimitersCluster; } - public MessageDynamoDbConfiguration getMessageDynamoDbConfiguration() { - return messageDynamoDb; - } - - public DynamoDbConfiguration getKeysDynamoDbConfiguration() { - return keysDynamoDb; - } - - public AccountsDynamoDbConfiguration getAccountsDynamoDbConfiguration() { - return accountsDynamoDb; - } - - public DynamoDbConfiguration getPhoneNumberIdentifiersDynamoDbConfiguration() { - return phoneNumberIdentifiersDynamoDb; - } - - public DeletedAccountsDynamoDbConfiguration getDeletedAccountsDynamoDbConfiguration() { - return deletedAccountsDynamoDb; - } - - public DynamoDbConfiguration getDeletedAccountsLockDynamoDbConfiguration() { - return deletedAccountsLockDynamoDb; - } - public DatabaseConfiguration getAbuseDatabaseConfiguration() { return abuseDatabase; } @@ -530,26 +447,6 @@ public class WhisperServerConfiguration extends Configuration { return appConfig; } - public DynamoDbConfiguration getPushChallengeDynamoDbConfiguration() { - return pushChallengeDynamoDb; - } - - public DynamoDbConfiguration getReportMessageDynamoDbConfiguration() { - return reportMessageDynamoDb; - } - - public DynamoDbConfiguration getPendingAccountsDynamoDbConfiguration() { - return pendingAccountsDynamoDb; - } - - public DynamoDbConfiguration getPendingDevicesDynamoDbConfiguration() { - return pendingDevicesDynamoDb; - } - - public DynamoDbConfiguration getReservedUsernamesDynamoDbConfiguration() { - return reservedUsernamesDynamoDb; - } - public DonationConfiguration getDonationConfiguration() { return donation; } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index dabc4c373..017ef9415 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -320,53 +320,18 @@ public class WhisperServerService extends Application dynamicConfigurationManager = new DynamicConfigurationManager<>(config.getAppConfig().getApplication(), @@ -374,34 +339,34 @@ public class WhisperServerService extends Application { @@ -95,23 +90,6 @@ public class AssignUsernameCommand extends EnvironmentCommand bootstrap, final Namespace namespace, final WhisperServerConfiguration config) throws Exception { - final DynamoDbClient reservedUsernamesDynamoDbClient = DynamoDbFromConfig.client(config.getReservedUsernamesDynamoDbConfiguration(), + final DynamoDbClient dynamoDbClient = DynamoDbFromConfig.client(config.getDynamoDbClientConfiguration(), software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - final ReservedUsernames reservedUsernames = new ReservedUsernames(reservedUsernamesDynamoDbClient, - config.getReservedUsernamesDynamoDbConfiguration().getTableName()); + final ReservedUsernames reservedUsernames = new ReservedUsernames(dynamoDbClient, + config.getDynamoDbTables().getReservedUsernames().getTableName()); final String pattern = namespace.getString("pattern").trim(); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/SetUserDiscoverabilityCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/SetUserDiscoverabilityCommand.java index ecc40ec1b..bebd7ec33 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/SetUserDiscoverabilityCommand.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/SetUserDiscoverabilityCommand.java @@ -94,26 +94,6 @@ public class SetUserDiscoverabilityCommand extends EnvironmentCommand