diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 8f1d13d87..8212086e8 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -171,7 +171,6 @@ import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache; import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener; import org.whispersystems.textsecuregcm.storage.Accounts; import org.whispersystems.textsecuregcm.storage.AccountsManager; -import org.whispersystems.textsecuregcm.storage.AssignPhoneNumberIdentifierCrawlerListener; import org.whispersystems.textsecuregcm.storage.ContactDiscoveryWriter; import org.whispersystems.textsecuregcm.storage.DeletedAccounts; import org.whispersystems.textsecuregcm.storage.DeletedAccountsDirectoryReconciler; @@ -215,7 +214,6 @@ import org.whispersystems.textsecuregcm.websocket.AuthenticatedConnectListener; import org.whispersystems.textsecuregcm.websocket.DeadLetterHandler; import org.whispersystems.textsecuregcm.websocket.ProvisioningConnectListener; import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator; -import org.whispersystems.textsecuregcm.workers.AssignPniCommand; import org.whispersystems.textsecuregcm.workers.CertificateCommand; import org.whispersystems.textsecuregcm.workers.CheckDynamicConfigurationCommand; import org.whispersystems.textsecuregcm.workers.DeleteUserCommand; @@ -245,7 +243,6 @@ public class WhisperServerService extends Application("accountdb", "accountsdb.xml") { @Override @@ -549,7 +546,6 @@ public class WhisperServerService extends Application accountDatabaseCrawlerListeners = List.of( new NonNormalizedAccountCrawlerListener(accountsManager, metricsCluster), new ContactDiscoveryWriter(accountsManager), - new AssignPhoneNumberIdentifierCrawlerListener(accountsManager, phoneNumberIdentifiers), // PushFeedbackProcessor may update device properties new PushFeedbackProcessor(accountsManager), // delete accounts last diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AssignPhoneNumberIdentifierCrawlerListener.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AssignPhoneNumberIdentifierCrawlerListener.java deleted file mode 100644 index 4490dc827..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AssignPhoneNumberIdentifierCrawlerListener.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2013-2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Metrics; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name; - -// TODO Remove this crawler when PNIs have been assigned to all existing accounts -public class AssignPhoneNumberIdentifierCrawlerListener extends AccountDatabaseCrawlerListener { - - private final AccountsManager accountsManager; - private final PhoneNumberIdentifiers phoneNumberIdentifiers; - - private final ExecutorService updateExecutor = Executors.newFixedThreadPool(16); - - private static final Counter ASSIGNED_PNI_COUNTER = - Metrics.counter(name(AssignPhoneNumberIdentifierCrawlerListener.class, "assignPni")); - - public AssignPhoneNumberIdentifierCrawlerListener(final AccountsManager accountsManager, - final PhoneNumberIdentifiers phoneNumberIdentifiers) { - - this.accountsManager = accountsManager; - this.phoneNumberIdentifiers = phoneNumberIdentifiers; - } - - @Override - public void onCrawlStart() { - } - - @Override - public void onCrawlEnd(final Optional fromUuid) { - } - - @Override - protected void onCrawlChunk(final Optional fromUuid, final List chunkAccounts) { - // There are exactly two ways an account can get a phone number identifier (PNI): - // - // 1. It's assigned at construction time for accounts created after the introduction of PNIs - // 2. It's assigned by this crawler - // - // That means that we don't need to worry about accidentally overwriting a PNI assigned by another source; if an - // account doesn't have a PNI when it winds up in a crawled chunk, there's no danger that it will have one after a - // refresh, and so we can blindly assign a random PNI. - CompletableFuture.allOf(chunkAccounts.stream() - .filter(account -> account.getPhoneNumberIdentifier().isEmpty()) - .map(Account::getUuid) - .map(accountsManager::getByAccountIdentifier) - .filter(Optional::isPresent) - .map(Optional::get) - .map(accountWithoutPni -> CompletableFuture.runAsync(() -> { - final String number = accountWithoutPni.getNumber(); - final UUID phoneNumberIdentifier = phoneNumberIdentifiers.getPhoneNumberIdentifier(number); - - accountsManager.update(accountWithoutPni, a -> a.setNumber(number, phoneNumberIdentifier)); - - ASSIGNED_PNI_COUNTER.increment(); - }, updateExecutor)) - .toArray(CompletableFuture[]::new)).join(); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignPniCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignPniCommand.java deleted file mode 100644 index 03e31f868..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignPniCommand.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright 2013-2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.workers; - -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; -import com.fasterxml.jackson.databind.DeserializationFeature; -import io.dropwizard.Application; -import io.dropwizard.cli.EnvironmentCommand; -import io.dropwizard.jdbi3.JdbiFactory; -import io.dropwizard.setup.Environment; -import io.lettuce.core.resource.ClientResources; -import io.micrometer.core.instrument.Metrics; -import net.sourceforge.argparse4j.inf.Namespace; -import net.sourceforge.argparse4j.inf.Subparser; -import org.jdbi.v3.core.Jdbi; -import org.whispersystems.textsecuregcm.WhisperServerConfiguration; -import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; -import org.whispersystems.textsecuregcm.metrics.PushLatencyManager; -import org.whispersystems.textsecuregcm.push.ClientPresenceManager; -import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; -import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; -import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; -import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; -import org.whispersystems.textsecuregcm.storage.Account; -import org.whispersystems.textsecuregcm.storage.Accounts; -import org.whispersystems.textsecuregcm.storage.AccountsManager; -import org.whispersystems.textsecuregcm.storage.DeletedAccounts; -import org.whispersystems.textsecuregcm.storage.DeletedAccountsManager; -import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; -import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; -import org.whispersystems.textsecuregcm.storage.Keys; -import org.whispersystems.textsecuregcm.storage.MessagesCache; -import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb; -import org.whispersystems.textsecuregcm.storage.MessagesManager; -import org.whispersystems.textsecuregcm.storage.PhoneNumberIdentifiers; -import org.whispersystems.textsecuregcm.storage.Profiles; -import org.whispersystems.textsecuregcm.storage.ProfilesManager; -import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb; -import org.whispersystems.textsecuregcm.storage.ReportMessageManager; -import org.whispersystems.textsecuregcm.storage.ReservedUsernames; -import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager; -import org.whispersystems.textsecuregcm.storage.Usernames; -import org.whispersystems.textsecuregcm.storage.UsernamesManager; -import org.whispersystems.textsecuregcm.storage.VerificationCodeStore; -import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import java.nio.ByteBuffer; -import java.time.Clock; -import java.util.Base64; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static com.codahale.metrics.MetricRegistry.name; - -public class AssignPniCommand extends EnvironmentCommand { - - public AssignPniCommand() { - - super(new Application<>() { - @Override - public void run(final WhisperServerConfiguration whisperServerConfiguration, final Environment environment) { - } - }, "assign-pni", "Assigns a PNI to one or more users if they don't already have one"); - } - - @Override - public void configure(final Subparser subparser) { - super.configure(subparser); - - subparser.addArgument("-b") - .dest("base64EncodedUuid") - .type(String.class) - .required(true) - .nargs("*") - .help("the base64-encoded UUID (ACI) for the user to whom to assign a PNI"); - } - - @Override - protected void run(final Environment environment, - final Namespace namespace, - final WhisperServerConfiguration configuration) throws Exception { - - Clock clock = Clock.systemUTC(); - environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - JdbiFactory jdbiFactory = new JdbiFactory(); - Jdbi accountJdbi = jdbiFactory.build(environment, configuration.getAccountsDatabaseConfiguration(), "accountdb"); - FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_delete_user", accountJdbi, - configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration()); - ClientResources redisClusterClientResources = ClientResources.builder().build(); - - DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig - .client(configuration.getReportMessageDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - DynamoDbClient messageDynamoDb = DynamoDbFromConfig.client(configuration.getMessageDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - DynamoDbClient preKeysDynamoDb = DynamoDbFromConfig.client(configuration.getKeysDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig - .client(configuration.getAccountsDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig - .client(configuration.getDeletedAccountsDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - DynamoDbClient phoneNumberIdentifiersDynamoDbClient = - DynamoDbFromConfig.client(configuration.getPhoneNumberIdentifiersDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - - FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", - configuration.getCacheClusterConfiguration(), redisClusterClientResources); - FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters", - configuration.getRateLimitersCluster(), redisClusterClientResources); - - ExecutorService keyspaceNotificationDispatchExecutor = environment.lifecycle() - .executorService(name(getClass(), "keyspaceNotification-%d")).maxThreads(4).build(); - ExecutorService backupServiceExecutor = environment.lifecycle() - .executorService(name(getClass(), "backupService-%d")).maxThreads(8).minThreads(1).build(); - ExecutorService storageServiceExecutor = environment.lifecycle() - .executorService(name(getClass(), "storageService-%d")).maxThreads(8).minThreads(1).build(); - - ExternalServiceCredentialGenerator backupCredentialsGenerator = new ExternalServiceCredentialGenerator( - configuration.getSecureBackupServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true); - ExternalServiceCredentialGenerator storageCredentialsGenerator = new ExternalServiceCredentialGenerator( - configuration.getSecureStorageServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true); - - DynamicConfigurationManager dynamicConfigurationManager = new DynamicConfigurationManager<>( - configuration.getAppConfig().getApplication(), configuration.getAppConfig().getEnvironment(), - configuration.getAppConfig().getConfigurationName(), DynamicConfiguration.class); - dynamicConfigurationManager.start(); - - DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig - .client(configuration.getPendingAccountsDynamoDbConfiguration(), - software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create()); - - AmazonDynamoDB deletedAccountsLockDynamoDbClient = AmazonDynamoDBClientBuilder.standard() - .withRegion(configuration.getDeletedAccountsLockDynamoDbConfiguration().getRegion()) - .withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout( - ((int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientExecutionTimeout() - .toMillis())) - .withRequestTimeout( - (int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientRequestTimeout() - .toMillis())) - .withCredentials(InstanceProfileCredentialsProvider.getInstance()) - .build(); - - DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient, - configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(), - configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName()); - VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient, - configuration.getPendingAccountsDynamoDbConfiguration().getTableName()); - - Accounts accounts = new Accounts(accountsDynamoDbClient, - configuration.getAccountsDynamoDbConfiguration().getTableName(), - configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(), - configuration.getAccountsDynamoDbConfiguration().getPhoneNumberIdentifierTableName(), - configuration.getAccountsDynamoDbConfiguration().getScanPageSize()); - PhoneNumberIdentifiers phoneNumberIdentifiers = new PhoneNumberIdentifiers(phoneNumberIdentifiersDynamoDbClient, - configuration.getPhoneNumberIdentifiersDynamoDbConfiguration().getTableName()); - Usernames usernames = new Usernames(accountDatabase); - Profiles profiles = new Profiles(accountDatabase); - ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase); - Keys keys = new Keys(preKeysDynamoDb, - configuration.getKeysDynamoDbConfiguration().getTableName()); - MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb, - configuration.getMessageDynamoDbConfiguration().getTableName(), - configuration.getMessageDynamoDbConfiguration().getTimeToLive()); - FaultTolerantRedisCluster messageInsertCacheCluster = new FaultTolerantRedisCluster("message_insert_cluster", - configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources); - FaultTolerantRedisCluster messageReadDeleteCluster = new FaultTolerantRedisCluster("message_read_delete_cluster", - configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources); - FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", - configuration.getMetricsClusterConfiguration(), redisClusterClientResources); - FaultTolerantRedisCluster clientPresenceCluster = new FaultTolerantRedisCluster("client_presence", - configuration.getClientPresenceClusterConfiguration(), redisClusterClientResources); - SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor, - configuration.getSecureBackupServiceConfiguration()); - SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator, - storageServiceExecutor, configuration.getSecureStorageServiceConfiguration()); - ClientPresenceManager clientPresenceManager = new ClientPresenceManager(clientPresenceCluster, - Executors.newSingleThreadScheduledExecutor(), keyspaceNotificationDispatchExecutor); - MessagesCache messagesCache = new MessagesCache(messageInsertCacheCluster, messageReadDeleteCluster, - keyspaceNotificationDispatchExecutor); - PushLatencyManager pushLatencyManager = new PushLatencyManager(metricsCluster, dynamicConfigurationManager); - DirectoryQueue directoryQueue = new DirectoryQueue( - configuration.getDirectoryConfiguration().getSqsConfiguration()); - UsernamesManager usernamesManager = new UsernamesManager(usernames, reservedUsernames, cacheCluster); - ProfilesManager profilesManager = new ProfilesManager(profiles, cacheCluster); - ReportMessageDynamoDb reportMessageDynamoDb = new ReportMessageDynamoDb(reportMessagesDynamoDb, - configuration.getReportMessageDynamoDbConfiguration().getTableName(), - configuration.getReportMessageConfiguration().getReportTtl()); - ReportMessageManager reportMessageManager = new ReportMessageManager(reportMessageDynamoDb, rateLimitersCluster, - Metrics.globalRegistry, configuration.getReportMessageConfiguration().getCounterTtl()); - MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager, - reportMessageManager); - DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts, - deletedAccountsLockDynamoDbClient, - configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName()); - StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts); - AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster, - deletedAccountsManager, directoryQueue, keys, messagesManager, usernamesManager, profilesManager, - pendingAccountsManager, secureStorageClient, secureBackupClient, clientPresenceManager, clock); - - final List base64EncodedUuids = namespace.getList("base64EncodedUuid"); - - for (final String base64EncodedUuid : base64EncodedUuids) { - final ByteBuffer uuidBuffer = ByteBuffer.wrap(Base64.getDecoder().decode(base64EncodedUuid)); - final UUID aci = new UUID(uuidBuffer.getLong(), uuidBuffer.getLong()); - - accountsManager.getByAccountIdentifier(aci).ifPresentOrElse(account -> { - if (account.getPhoneNumberIdentifier().isEmpty()) { - final String number = account.getNumber(); - final UUID phoneNumberIdentifier = phoneNumberIdentifiers.getPhoneNumberIdentifier(number); - - accountsManager.update(account, a -> a.setNumber(number, phoneNumberIdentifier)); - - System.out.println("Assigned PNI to account: " + aci); - } else { - System.out.println("Account already had PNI: " + aci); - } - }, - () -> System.out.println("Account not found: " + aci)); - } - } -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AssignPhoneNumberIdentifierCrawlerListenerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AssignPhoneNumberIdentifierCrawlerListenerTest.java deleted file mode 100644 index aabcac82b..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AssignPhoneNumberIdentifierCrawlerListenerTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2013-2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.function.Consumer; -import org.junit.jupiter.api.Test; -import org.mockito.stubbing.Answer; - -class AssignPhoneNumberIdentifierCrawlerListenerTest { - - @Test - void onCrawlChunk() { - final UUID accountIdentifierWithPni = UUID.randomUUID(); - final UUID accountIdentifierWithoutPni = UUID.randomUUID(); - - final String numberWithPni = "+18005551111"; - final String numberWithoutPni = "+18005552222"; - - final Account accountWithPni = mock(Account.class); - when(accountWithPni.getUuid()).thenReturn(accountIdentifierWithPni); - when(accountWithPni.getNumber()).thenReturn(numberWithPni); - when(accountWithPni.getPhoneNumberIdentifier()).thenReturn(Optional.of(UUID.randomUUID())); - - final Account accountWithoutPni = mock(Account.class); - when(accountWithoutPni.getUuid()).thenReturn(accountIdentifierWithoutPni); - when(accountWithoutPni.getNumber()).thenReturn(numberWithoutPni); - when(accountWithoutPni.getPhoneNumberIdentifier()).thenReturn(Optional.empty()); - - final AccountsManager accountsManager = mock(AccountsManager.class); - when(accountsManager.getByAccountIdentifier(accountIdentifierWithPni)).thenReturn(Optional.of(accountWithPni)); - when(accountsManager.getByAccountIdentifier(accountIdentifierWithoutPni)).thenReturn(Optional.of(accountWithoutPni)); - - when(accountsManager.update(any(), any())).thenAnswer((Answer) invocation -> { - final Account account = invocation.getArgument(0, Account.class); - final Consumer updater = invocation.getArgument(1, Consumer.class); - - updater.accept(account); - - return account; - }); - - final PhoneNumberIdentifiers phoneNumberIdentifiers = mock(PhoneNumberIdentifiers.class); - when(phoneNumberIdentifiers.getPhoneNumberIdentifier(anyString())).thenAnswer( - (Answer) invocation -> UUID.randomUUID()); - - final AssignPhoneNumberIdentifierCrawlerListener crawler = - new AssignPhoneNumberIdentifierCrawlerListener(accountsManager, phoneNumberIdentifiers); - - crawler.onCrawlChunk(Optional.empty(), List.of(accountWithPni, accountWithoutPni)); - - verify(accountsManager).update(eq(accountWithoutPni), any()); - verify(accountWithoutPni).setNumber(eq(numberWithoutPni), any()); - - verify(accountsManager, never()).update(eq(accountWithPni), any()); - verify(accountWithPni, never()).setNumber(any(), any()); - } -}