From a315c9be9267488993e4106c16c7552f395f8966 Mon Sep 17 00:00:00 2001 From: Chris Eager Date: Fri, 28 May 2021 10:34:28 -0500 Subject: [PATCH] Add DeletedAccounts DynamoDB table --- service/config/sample.yml | 4 ++ .../WhisperServerConfiguration.java | 9 +++ .../textsecuregcm/WhisperServerService.java | 7 +- .../storage/AccountsManager.java | 11 ++- .../storage/DeletedAccounts.java | 71 +++++++++++++++++++ .../workers/DeleteUserCommand.java | 8 ++- .../storage/DeletedAccountsTest.java | 57 +++++++++++++++ .../tests/storage/AccountsManagerTest.java | 37 +++++++--- 8 files changed, 189 insertions(+), 15 deletions(-) create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/DeletedAccounts.java create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/DeletedAccountsTest.java diff --git a/service/config/sample.yml b/service/config/sample.yml index 9aaa5e098..c0de21e48 100644 --- a/service/config/sample.yml +++ b/service/config/sample.yml @@ -89,6 +89,10 @@ accountsDynamoDb: # DynamoDB table configuration tableName: phoneNumberTableName: +deletedAccountsDynamoDb: # DynamoDb table configuration + region: + tableName: + migrationDeletedAccountsDynamoDb: # DynamoDB table configuration region: tableName: diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index 959a7d324..40819a925 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -157,6 +157,11 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private DynamoDbConfiguration migrationRetryAccountsDynamoDb; + @Valid + @NotNull + @JsonProperty + private DynamoDbConfiguration deletedAccountsDynamoDb; + @Valid @NotNull @JsonProperty @@ -381,6 +386,10 @@ public class WhisperServerConfiguration extends Configuration { return migrationRetryAccountsDynamoDb; } + public DynamoDbConfiguration getDeletedAccountsDynamoDbConfiguration() { + return deletedAccountsDynamoDb; + } + public DatabaseConfiguration getAbuseDatabaseConfiguration() { return abuseDatabase; } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index dfae28c7d..a1524c7e5 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -155,6 +155,7 @@ import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDbMigrator; import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.ActiveUserCounter; +import org.whispersystems.textsecuregcm.storage.DeletedAccounts; import org.whispersystems.textsecuregcm.storage.DirectoryReconciler; import org.whispersystems.textsecuregcm.storage.DirectoryReconciliationClient; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; @@ -331,6 +332,9 @@ public class WhisperServerService extends Application> list(final int max) { + + final ScanRequest scanRequest = ScanRequest.builder() + .tableName(tableName) + .limit(max) + .build(); + + return scan(scanRequest, max) + .stream() + .map(item -> new Pair<>( + AttributeValues.getUUID(item, KEY_ACCOUNT_UUID, null), + AttributeValues.getString(item, ATTR_ACCOUNT_E164, null))) + .collect(Collectors.toList()); + } + + public void delete(final List uuidsToDelete) { + + writeInBatches(uuidsToDelete, (uuids -> { + + final List deletes = uuids.stream() + .map(uuid -> WriteRequest.builder().deleteRequest( + DeleteRequest.builder().key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid))).build()).build()) + .collect(Collectors.toList()); + + executeTableWriteItemsUntilComplete(Map.of(tableName, deletes)); + })); + } + +} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java index 60a700b98..c67c2b051 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java @@ -7,8 +7,6 @@ package org.whispersystems.textsecuregcm.workers; import static com.codahale.metrics.MetricRegistry.name; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.fasterxml.jackson.databind.DeserializationFeature; import io.dropwizard.Application; import io.dropwizard.cli.EnvironmentCommand; @@ -39,6 +37,7 @@ import org.whispersystems.textsecuregcm.storage.Accounts; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.AccountsManager.DeletionReason; +import org.whispersystems.textsecuregcm.storage.DeletedAccounts; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; import org.whispersystems.textsecuregcm.storage.KeysDynamoDb; @@ -112,6 +111,8 @@ public class DeleteUserCommand extends EnvironmentCommand account = accountsManager.get(user); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/DeletedAccountsTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/DeletedAccountsTest.java new file mode 100644 index 000000000..7195faacd --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/DeletedAccountsTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2013-2021 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ +package org.whispersystems.textsecuregcm.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.whispersystems.textsecuregcm.util.Pair; +import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; +import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; + +class DeletedAccountsTest { + + @RegisterExtension + static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder() + .tableName("deleted_accounts_test") + .hashKey(DeletedAccounts.KEY_ACCOUNT_UUID) + .attributeDefinition(AttributeDefinition.builder() + .attributeName(DeletedAccounts.KEY_ACCOUNT_UUID) + .attributeType(ScalarAttributeType.B).build()) + .build(); + + @Test + void test() { + + final DeletedAccounts deletedAccounts = new DeletedAccounts(dynamoDbExtension.getDynamoDbClient(), + dynamoDbExtension.getTableName()); + + UUID firstUuid = UUID.randomUUID(); + UUID secondUuid = UUID.randomUUID(); + + String firstNumber = "+14152221234"; + String secondNumber = "+14152225678"; + + assertTrue(deletedAccounts.list(1).isEmpty()); + + deletedAccounts.put(firstUuid, firstNumber); + deletedAccounts.put(secondUuid, secondNumber); + + assertEquals(1, deletedAccounts.list(1).size()); + + assertTrue(deletedAccounts.list(10).containsAll( + List.of( + new Pair<>(firstUuid, firstNumber), + new Pair<>(secondUuid, secondNumber)))); + + deletedAccounts.delete(List.of(firstUuid, secondUuid)); + + assertTrue(deletedAccounts.list(10).isEmpty()); + } +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsManagerTest.java index b6dcae6db..4b8483579 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsManagerTest.java @@ -41,6 +41,7 @@ import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.Accounts; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsManager; +import org.whispersystems.textsecuregcm.storage.DeletedAccounts; import org.whispersystems.textsecuregcm.storage.Device; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.storage.KeysDynamoDb; @@ -70,6 +71,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -85,7 +87,8 @@ class AccountsManagerTest { when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(uuid.toString()); when(commands.get(eq("Account3::" + uuid.toString()))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}"); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); Optional account = accountsManager.get("+14152222222"); assertTrue(account.isPresent()); @@ -120,6 +123,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -134,7 +138,8 @@ class AccountsManagerTest { when(commands.get(eq("Account3::" + uuid.toString()))).thenReturn("{\"number\": \"+14152222222\", \"name\": \"test\"}"); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); Optional account = accountsManager.get(uuid); assertTrue(account.isPresent()); @@ -157,6 +162,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -172,7 +178,8 @@ class AccountsManagerTest { when(commands.get(eq("AccountMap::+14152222222"))).thenReturn(null); when(accounts.get(eq("+14152222222"))).thenReturn(Optional.of(account)); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); Optional retrieved = accountsManager.get("+14152222222"); assertTrue(retrieved.isPresent()); @@ -198,6 +205,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -213,7 +221,8 @@ class AccountsManagerTest { when(commands.get(eq("Account3::" + uuid))).thenReturn(null); when(accounts.get(eq(uuid))).thenReturn(Optional.of(account)); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); Optional retrieved = accountsManager.get(uuid); assertTrue(retrieved.isPresent()); @@ -238,6 +247,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -253,7 +263,8 @@ class AccountsManagerTest { when(commands.get(eq("AccountMap::+14152222222"))).thenThrow(new RedisException("Connection lost!")); when(accounts.get(eq("+14152222222"))).thenReturn(Optional.of(account)); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); Optional retrieved = accountsManager.get("+14152222222"); assertTrue(retrieved.isPresent()); @@ -278,6 +289,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -293,7 +305,8 @@ class AccountsManagerTest { when(commands.get(eq("Account3::" + uuid))).thenThrow(new RedisException("Connection lost!")); when(accounts.get(eq(uuid))).thenReturn(Optional.of(account)); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); Optional retrieved = accountsManager.get(uuid); assertTrue(retrieved.isPresent()); @@ -318,6 +331,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -332,7 +346,8 @@ class AccountsManagerTest { when(commands.get(eq("Account3::" + uuid))).thenReturn(null); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); assertEquals(0, account.getDynamoDbMigrationVersion()); @@ -353,6 +368,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -368,7 +384,8 @@ class AccountsManagerTest { when(commands.get(eq("Account3::" + uuid))).thenReturn(null); doThrow(ConditionalCheckFailedException.class).when(accountsDynamoDb).update(any(Account.class)); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); assertEquals(0, account.getDynamoDbMigrationVersion()); @@ -390,6 +407,7 @@ class AccountsManagerTest { FaultTolerantRedisCluster cacheCluster = RedisClusterHelper.buildMockRedisCluster(commands); Accounts accounts = mock(Accounts.class); AccountsDynamoDb accountsDynamoDb = mock(AccountsDynamoDb.class); + DeletedAccounts deletedAccounts = mock(DeletedAccounts.class); DirectoryQueue directoryQueue = mock(DirectoryQueue.class); KeysDynamoDb keysDynamoDb = mock(KeysDynamoDb.class); MessagesManager messagesManager = mock(MessagesManager.class); @@ -398,7 +416,8 @@ class AccountsManagerTest { SecureBackupClient secureBackupClient = mock(SecureBackupClient.class); SecureStorageClient secureStorageClient = mock(SecureStorageClient.class); - AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); + AccountsManager accountsManager = new AccountsManager(accounts, accountsDynamoDb, cacheCluster, deletedAccounts, + directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager, secureStorageClient, secureBackupClient, experimentEnrollmentManager, dynamicConfigurationManager); assertEquals(Optional.empty(), accountsManager.compareAccounts(Optional.empty(), Optional.empty()));