Drop PNI migration tools

This commit is contained in:
Jon Chambers 2021-11-09 18:24:47 -05:00 committed by Jon Chambers
parent f42fd8a840
commit 069ffa9921
4 changed files with 0 additions and 381 deletions

View File

@ -171,7 +171,6 @@ import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.AssignPhoneNumberIdentifierCrawlerListener;
import org.whispersystems.textsecuregcm.storage.ContactDiscoveryWriter;
import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
import org.whispersystems.textsecuregcm.storage.DeletedAccountsDirectoryReconciler;
@ -215,7 +214,6 @@ import org.whispersystems.textsecuregcm.websocket.AuthenticatedConnectListener;
import org.whispersystems.textsecuregcm.websocket.DeadLetterHandler;
import org.whispersystems.textsecuregcm.websocket.ProvisioningConnectListener;
import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator;
import org.whispersystems.textsecuregcm.workers.AssignPniCommand;
import org.whispersystems.textsecuregcm.workers.CertificateCommand;
import org.whispersystems.textsecuregcm.workers.CheckDynamicConfigurationCommand;
import org.whispersystems.textsecuregcm.workers.DeleteUserCommand;
@ -245,7 +243,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
bootstrap.addCommand(new ServerVersionCommand());
bootstrap.addCommand(new CheckDynamicConfigurationCommand());
bootstrap.addCommand(new SetUserDiscoverabilityCommand());
bootstrap.addCommand(new AssignPniCommand());
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("accountdb", "accountsdb.xml") {
@Override
@ -549,7 +546,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
final List<AccountDatabaseCrawlerListener> accountDatabaseCrawlerListeners = List.of(
new NonNormalizedAccountCrawlerListener(accountsManager, metricsCluster),
new ContactDiscoveryWriter(accountsManager),
new AssignPhoneNumberIdentifierCrawlerListener(accountsManager, phoneNumberIdentifiers),
// PushFeedbackProcessor may update device properties
new PushFeedbackProcessor(accountsManager),
// delete accounts last

View File

@ -1,72 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
// TODO Remove this crawler when PNIs have been assigned to all existing accounts
public class AssignPhoneNumberIdentifierCrawlerListener extends AccountDatabaseCrawlerListener {
private final AccountsManager accountsManager;
private final PhoneNumberIdentifiers phoneNumberIdentifiers;
private final ExecutorService updateExecutor = Executors.newFixedThreadPool(16);
private static final Counter ASSIGNED_PNI_COUNTER =
Metrics.counter(name(AssignPhoneNumberIdentifierCrawlerListener.class, "assignPni"));
public AssignPhoneNumberIdentifierCrawlerListener(final AccountsManager accountsManager,
final PhoneNumberIdentifiers phoneNumberIdentifiers) {
this.accountsManager = accountsManager;
this.phoneNumberIdentifiers = phoneNumberIdentifiers;
}
@Override
public void onCrawlStart() {
}
@Override
public void onCrawlEnd(final Optional<UUID> fromUuid) {
}
@Override
protected void onCrawlChunk(final Optional<UUID> fromUuid, final List<Account> chunkAccounts) {
// There are exactly two ways an account can get a phone number identifier (PNI):
//
// 1. It's assigned at construction time for accounts created after the introduction of PNIs
// 2. It's assigned by this crawler
//
// That means that we don't need to worry about accidentally overwriting a PNI assigned by another source; if an
// account doesn't have a PNI when it winds up in a crawled chunk, there's no danger that it will have one after a
// refresh, and so we can blindly assign a random PNI.
CompletableFuture.allOf(chunkAccounts.stream()
.filter(account -> account.getPhoneNumberIdentifier().isEmpty())
.map(Account::getUuid)
.map(accountsManager::getByAccountIdentifier)
.filter(Optional::isPresent)
.map(Optional::get)
.map(accountWithoutPni -> CompletableFuture.runAsync(() -> {
final String number = accountWithoutPni.getNumber();
final UUID phoneNumberIdentifier = phoneNumberIdentifiers.getPhoneNumberIdentifier(number);
accountsManager.update(accountWithoutPni, a -> a.setNumber(number, phoneNumberIdentifier));
ASSIGNED_PNI_COUNTER.increment();
}, updateExecutor))
.toArray(CompletableFuture[]::new)).join();
}
}

View File

@ -1,234 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.workers;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.fasterxml.jackson.databind.DeserializationFeature;
import io.dropwizard.Application;
import io.dropwizard.cli.EnvironmentCommand;
import io.dropwizard.jdbi3.JdbiFactory;
import io.dropwizard.setup.Environment;
import io.lettuce.core.resource.ClientResources;
import io.micrometer.core.instrument.Metrics;
import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser;
import org.jdbi.v3.core.Jdbi;
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient;
import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.Accounts;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.DeletedAccounts;
import org.whispersystems.textsecuregcm.storage.DeletedAccountsManager;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
import org.whispersystems.textsecuregcm.storage.Keys;
import org.whispersystems.textsecuregcm.storage.MessagesCache;
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.PhoneNumberIdentifiers;
import org.whispersystems.textsecuregcm.storage.Profiles;
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb;
import org.whispersystems.textsecuregcm.storage.ReportMessageManager;
import org.whispersystems.textsecuregcm.storage.ReservedUsernames;
import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
import org.whispersystems.textsecuregcm.storage.Usernames;
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
import org.whispersystems.textsecuregcm.util.DynamoDbFromConfig;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import java.nio.ByteBuffer;
import java.time.Clock;
import java.util.Base64;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static com.codahale.metrics.MetricRegistry.name;
public class AssignPniCommand extends EnvironmentCommand<WhisperServerConfiguration> {
public AssignPniCommand() {
super(new Application<>() {
@Override
public void run(final WhisperServerConfiguration whisperServerConfiguration, final Environment environment) {
}
}, "assign-pni", "Assigns a PNI to one or more users if they don't already have one");
}
@Override
public void configure(final Subparser subparser) {
super.configure(subparser);
subparser.addArgument("-b")
.dest("base64EncodedUuid")
.type(String.class)
.required(true)
.nargs("*")
.help("the base64-encoded UUID (ACI) for the user to whom to assign a PNI");
}
@Override
protected void run(final Environment environment,
final Namespace namespace,
final WhisperServerConfiguration configuration) throws Exception {
Clock clock = Clock.systemUTC();
environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
JdbiFactory jdbiFactory = new JdbiFactory();
Jdbi accountJdbi = jdbiFactory.build(environment, configuration.getAccountsDatabaseConfiguration(), "accountdb");
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("account_database_delete_user", accountJdbi,
configuration.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
ClientResources redisClusterClientResources = ClientResources.builder().build();
DynamoDbClient reportMessagesDynamoDb = DynamoDbFromConfig
.client(configuration.getReportMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient messageDynamoDb = DynamoDbFromConfig.client(configuration.getMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient preKeysDynamoDb = DynamoDbFromConfig.client(configuration.getKeysDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig
.client(configuration.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig
.client(configuration.getDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient phoneNumberIdentifiersDynamoDbClient =
DynamoDbFromConfig.client(configuration.getPhoneNumberIdentifiersDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster",
configuration.getCacheClusterConfiguration(), redisClusterClientResources);
FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters",
configuration.getRateLimitersCluster(), redisClusterClientResources);
ExecutorService keyspaceNotificationDispatchExecutor = environment.lifecycle()
.executorService(name(getClass(), "keyspaceNotification-%d")).maxThreads(4).build();
ExecutorService backupServiceExecutor = environment.lifecycle()
.executorService(name(getClass(), "backupService-%d")).maxThreads(8).minThreads(1).build();
ExecutorService storageServiceExecutor = environment.lifecycle()
.executorService(name(getClass(), "storageService-%d")).maxThreads(8).minThreads(1).build();
ExternalServiceCredentialGenerator backupCredentialsGenerator = new ExternalServiceCredentialGenerator(
configuration.getSecureBackupServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
ExternalServiceCredentialGenerator storageCredentialsGenerator = new ExternalServiceCredentialGenerator(
configuration.getSecureStorageServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager = new DynamicConfigurationManager<>(
configuration.getAppConfig().getApplication(), configuration.getAppConfig().getEnvironment(),
configuration.getAppConfig().getConfigurationName(), DynamicConfiguration.class);
dynamicConfigurationManager.start();
DynamoDbClient pendingAccountsDynamoDbClient = DynamoDbFromConfig
.client(configuration.getPendingAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
AmazonDynamoDB deletedAccountsLockDynamoDbClient = AmazonDynamoDBClientBuilder.standard()
.withRegion(configuration.getDeletedAccountsLockDynamoDbConfiguration().getRegion())
.withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(
((int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientExecutionTimeout()
.toMillis()))
.withRequestTimeout(
(int) configuration.getDeletedAccountsLockDynamoDbConfiguration().getClientRequestTimeout()
.toMillis()))
.withCredentials(InstanceProfileCredentialsProvider.getInstance())
.build();
DeletedAccounts deletedAccounts = new DeletedAccounts(deletedAccountsDynamoDbClient,
configuration.getDeletedAccountsDynamoDbConfiguration().getTableName(),
configuration.getDeletedAccountsDynamoDbConfiguration().getNeedsReconciliationIndexName());
VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient,
configuration.getPendingAccountsDynamoDbConfiguration().getTableName());
Accounts accounts = new Accounts(accountsDynamoDbClient,
configuration.getAccountsDynamoDbConfiguration().getTableName(),
configuration.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(),
configuration.getAccountsDynamoDbConfiguration().getPhoneNumberIdentifierTableName(),
configuration.getAccountsDynamoDbConfiguration().getScanPageSize());
PhoneNumberIdentifiers phoneNumberIdentifiers = new PhoneNumberIdentifiers(phoneNumberIdentifiersDynamoDbClient,
configuration.getPhoneNumberIdentifiersDynamoDbConfiguration().getTableName());
Usernames usernames = new Usernames(accountDatabase);
Profiles profiles = new Profiles(accountDatabase);
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
Keys keys = new Keys(preKeysDynamoDb,
configuration.getKeysDynamoDbConfiguration().getTableName());
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb,
configuration.getMessageDynamoDbConfiguration().getTableName(),
configuration.getMessageDynamoDbConfiguration().getTimeToLive());
FaultTolerantRedisCluster messageInsertCacheCluster = new FaultTolerantRedisCluster("message_insert_cluster",
configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
FaultTolerantRedisCluster messageReadDeleteCluster = new FaultTolerantRedisCluster("message_read_delete_cluster",
configuration.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClusterClientResources);
FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster",
configuration.getMetricsClusterConfiguration(), redisClusterClientResources);
FaultTolerantRedisCluster clientPresenceCluster = new FaultTolerantRedisCluster("client_presence",
configuration.getClientPresenceClusterConfiguration(), redisClusterClientResources);
SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor,
configuration.getSecureBackupServiceConfiguration());
SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator,
storageServiceExecutor, configuration.getSecureStorageServiceConfiguration());
ClientPresenceManager clientPresenceManager = new ClientPresenceManager(clientPresenceCluster,
Executors.newSingleThreadScheduledExecutor(), keyspaceNotificationDispatchExecutor);
MessagesCache messagesCache = new MessagesCache(messageInsertCacheCluster, messageReadDeleteCluster,
keyspaceNotificationDispatchExecutor);
PushLatencyManager pushLatencyManager = new PushLatencyManager(metricsCluster, dynamicConfigurationManager);
DirectoryQueue directoryQueue = new DirectoryQueue(
configuration.getDirectoryConfiguration().getSqsConfiguration());
UsernamesManager usernamesManager = new UsernamesManager(usernames, reservedUsernames, cacheCluster);
ProfilesManager profilesManager = new ProfilesManager(profiles, cacheCluster);
ReportMessageDynamoDb reportMessageDynamoDb = new ReportMessageDynamoDb(reportMessagesDynamoDb,
configuration.getReportMessageDynamoDbConfiguration().getTableName(),
configuration.getReportMessageConfiguration().getReportTtl());
ReportMessageManager reportMessageManager = new ReportMessageManager(reportMessageDynamoDb, rateLimitersCluster,
Metrics.globalRegistry, configuration.getReportMessageConfiguration().getCounterTtl());
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager,
reportMessageManager);
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
deletedAccountsLockDynamoDbClient,
configuration.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster,
deletedAccountsManager, directoryQueue, keys, messagesManager, usernamesManager, profilesManager,
pendingAccountsManager, secureStorageClient, secureBackupClient, clientPresenceManager, clock);
final List<String> base64EncodedUuids = namespace.getList("base64EncodedUuid");
for (final String base64EncodedUuid : base64EncodedUuids) {
final ByteBuffer uuidBuffer = ByteBuffer.wrap(Base64.getDecoder().decode(base64EncodedUuid));
final UUID aci = new UUID(uuidBuffer.getLong(), uuidBuffer.getLong());
accountsManager.getByAccountIdentifier(aci).ifPresentOrElse(account -> {
if (account.getPhoneNumberIdentifier().isEmpty()) {
final String number = account.getNumber();
final UUID phoneNumberIdentifier = phoneNumberIdentifiers.getPhoneNumberIdentifier(number);
accountsManager.update(account, a -> a.setNumber(number, phoneNumberIdentifier));
System.out.println("Assigned PNI to account: " + aci);
} else {
System.out.println("Account already had PNI: " + aci);
}
},
() -> System.out.println("Account not found: " + aci));
}
}
}

View File

@ -1,71 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Consumer;
import org.junit.jupiter.api.Test;
import org.mockito.stubbing.Answer;
class AssignPhoneNumberIdentifierCrawlerListenerTest {
@Test
void onCrawlChunk() {
final UUID accountIdentifierWithPni = UUID.randomUUID();
final UUID accountIdentifierWithoutPni = UUID.randomUUID();
final String numberWithPni = "+18005551111";
final String numberWithoutPni = "+18005552222";
final Account accountWithPni = mock(Account.class);
when(accountWithPni.getUuid()).thenReturn(accountIdentifierWithPni);
when(accountWithPni.getNumber()).thenReturn(numberWithPni);
when(accountWithPni.getPhoneNumberIdentifier()).thenReturn(Optional.of(UUID.randomUUID()));
final Account accountWithoutPni = mock(Account.class);
when(accountWithoutPni.getUuid()).thenReturn(accountIdentifierWithoutPni);
when(accountWithoutPni.getNumber()).thenReturn(numberWithoutPni);
when(accountWithoutPni.getPhoneNumberIdentifier()).thenReturn(Optional.empty());
final AccountsManager accountsManager = mock(AccountsManager.class);
when(accountsManager.getByAccountIdentifier(accountIdentifierWithPni)).thenReturn(Optional.of(accountWithPni));
when(accountsManager.getByAccountIdentifier(accountIdentifierWithoutPni)).thenReturn(Optional.of(accountWithoutPni));
when(accountsManager.update(any(), any())).thenAnswer((Answer<Account>) invocation -> {
final Account account = invocation.getArgument(0, Account.class);
final Consumer<Account> updater = invocation.getArgument(1, Consumer.class);
updater.accept(account);
return account;
});
final PhoneNumberIdentifiers phoneNumberIdentifiers = mock(PhoneNumberIdentifiers.class);
when(phoneNumberIdentifiers.getPhoneNumberIdentifier(anyString())).thenAnswer(
(Answer<UUID>) invocation -> UUID.randomUUID());
final AssignPhoneNumberIdentifierCrawlerListener crawler =
new AssignPhoneNumberIdentifierCrawlerListener(accountsManager, phoneNumberIdentifiers);
crawler.onCrawlChunk(Optional.empty(), List.of(accountWithPni, accountWithoutPni));
verify(accountsManager).update(eq(accountWithoutPni), any());
verify(accountWithoutPni).setNumber(eq(numberWithoutPni), any());
verify(accountsManager, never()).update(eq(accountWithPni), any());
verify(accountWithPni, never()).setNumber(any(), any());
}
}