From 744eb5807148c4a4f39b60a53c975a9c15fc2bbc Mon Sep 17 00:00:00 2001 From: Jon Chambers Date: Thu, 19 Oct 2023 12:21:17 -0400 Subject: [PATCH] Discard old chunk-based account crawler machinery --- service/config/sample.yml | 4 - .../WhisperServerConfiguration.java | 10 -- .../textsecuregcm/WhisperServerService.java | 5 +- .../AccountDatabaseCrawlerConfiguration.java | 18 -- .../AccountsTableConfiguration.java | 9 +- .../storage/AccountCrawlChunk.java | 30 ---- .../storage/AccountDatabaseCrawler.java | 111 ------------ .../storage/AccountDatabaseCrawlerCache.java | 74 -------- .../AccountDatabaseCrawlerListener.java | 37 ---- .../textsecuregcm/storage/Accounts.java | 39 +---- .../storage/AccountsManager.java | 8 - .../storage/PushFeedbackProcessor.java | 132 --------------- .../workers/AssignUsernameCommand.java | 3 +- .../workers/CommandDependencies.java | 3 +- .../workers/CrawlAccountsCommand.java | 152 ----------------- .../filters/RemoteDeprecationFilterTest.java | 17 +- ...AccountDatabaseCrawlerIntegrationTest.java | 78 --------- .../storage/AccountDatabaseCrawlerTest.java | 82 --------- ...ntsManagerChangeNumberIntegrationTest.java | 5 +- ...ConcurrentModificationIntegrationTest.java | 5 +- ...ccountsManagerUsernameIntegrationTest.java | 4 +- .../textsecuregcm/storage/AccountsTest.java | 57 +------ .../storage/PushFeedbackProcessorTest.java | 159 ------------------ 23 files changed, 18 insertions(+), 1024 deletions(-) delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/AccountDatabaseCrawlerConfiguration.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountCrawlChunk.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerListener.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessor.java delete mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/workers/CrawlAccountsCommand.java delete mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java delete mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerTest.java delete mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessorTest.java diff --git a/service/config/sample.yml b/service/config/sample.yml index 38c21ce1b..810d2ea7f 100644 --- a/service/config/sample.yml +++ b/service/config/sample.yml @@ -85,7 +85,6 @@ dynamoDbTables: phoneNumberTableName: Example_Accounts_PhoneNumbers phoneNumberIdentifierTableName: Example_Accounts_PhoneNumberIdentifiers usernamesTableName: Example_Accounts_Usernames - scanPageSize: 100 clientReleases: tableName: Example_ClientReleases deletedAccounts: @@ -202,9 +201,6 @@ tus: uploadUri: https://example.org/upload userAuthenticationTokenSharedSecret: secret://tus.userAuthenticationTokenSharedSecret -accountDatabaseCrawler: - chunkSize: 10 # accounts per run - apn: # Apple Push Notifications configuration sandbox: true bundleId: com.example.textsecuregcm diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index 58f13a318..4116cb9cb 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -16,7 +16,6 @@ import java.util.Set; import javax.validation.Valid; import javax.validation.constraints.NotNull; import org.whispersystems.textsecuregcm.attachments.TusConfiguration; -import org.whispersystems.textsecuregcm.configuration.AccountDatabaseCrawlerConfiguration; import org.whispersystems.textsecuregcm.configuration.AdminEventLoggingConfiguration; import org.whispersystems.textsecuregcm.configuration.ApnConfiguration; import org.whispersystems.textsecuregcm.configuration.AppConfigConfiguration; @@ -132,11 +131,6 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private SecureValueRecovery2Configuration svr2; - @NotNull - @Valid - @JsonProperty - private AccountDatabaseCrawlerConfiguration accountDatabaseCrawler; - @NotNull @Valid @JsonProperty @@ -374,10 +368,6 @@ public class WhisperServerConfiguration extends Configuration { return storageService; } - public AccountDatabaseCrawlerConfiguration getAccountDatabaseCrawlerConfiguration() { - return accountDatabaseCrawler; - } - public MessageCacheConfiguration getMessageCacheConfiguration() { return messageCache; } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 6cb0314af..90cc45cf0 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -212,7 +212,6 @@ import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator; import org.whispersystems.textsecuregcm.workers.AssignUsernameCommand; import org.whispersystems.textsecuregcm.workers.CertificateCommand; import org.whispersystems.textsecuregcm.workers.CheckDynamicConfigurationCommand; -import org.whispersystems.textsecuregcm.workers.CrawlAccountsCommand; import org.whispersystems.textsecuregcm.workers.DeleteUserCommand; import org.whispersystems.textsecuregcm.workers.MessagePersisterServiceCommand; import org.whispersystems.textsecuregcm.workers.MigrateSignedECPreKeysCommand; @@ -270,7 +269,6 @@ public class WhisperServerService extends Application accounts; - @Nullable - private final UUID lastUuid; - - public AccountCrawlChunk(final List accounts, @Nullable final UUID lastUuid) { - this.accounts = accounts; - this.lastUuid = lastUuid; - } - - public List getAccounts() { - return accounts; - } - - public Optional getLastUuid() { - return Optional.ofNullable(lastUuid); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java deleted file mode 100644 index e934ea07a..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ -package org.whispersystems.textsecuregcm.storage; - -import static com.codahale.metrics.MetricRegistry.name; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.SharedMetricRegistries; -import com.codahale.metrics.Timer; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.whispersystems.textsecuregcm.util.Constants; - -@SuppressWarnings("OptionalUsedAsFieldOrParameterType") -public class AccountDatabaseCrawler { - - private static final Logger logger = LoggerFactory.getLogger(AccountDatabaseCrawler.class); - private static final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME); - private static final Timer readChunkTimer = metricRegistry.timer(name(AccountDatabaseCrawler.class, "readChunk")); - private static final Timer processChunkTimer = metricRegistry.timer( - name(AccountDatabaseCrawler.class, "processChunk")); - - private static final long WORKER_TTL_MS = 120_000L; - - private final String name; - private final AccountsManager accounts; - private final int chunkSize; - private final String workerId; - private final AccountDatabaseCrawlerCache cache; - private final List listeners; - - public AccountDatabaseCrawler(final String name, - AccountsManager accounts, - AccountDatabaseCrawlerCache cache, - List listeners, - int chunkSize) { - this.name = name; - this.accounts = accounts; - this.chunkSize = chunkSize; - this.workerId = UUID.randomUUID().toString(); - this.cache = cache; - this.listeners = listeners; - } - - public void crawlAllAccounts() { - if (!cache.claimActiveWork(workerId, WORKER_TTL_MS)) { - logger.info("Did not claim active work"); - return; - } - try { - Optional fromUuid = getLastUuid(); - - if (fromUuid.isEmpty()) { - logger.info("{}: Started crawl", name); - listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart); - } else { - logger.info("{}: Resuming crawl", name); - } - - AccountCrawlChunk chunkAccounts; - do { - try (Timer.Context timer = processChunkTimer.time()) { - logger.debug("{}: Processing chunk", name); - chunkAccounts = readChunk(fromUuid, chunkSize); - - for (AccountDatabaseCrawlerListener listener : listeners) { - listener.timeAndProcessCrawlChunk(fromUuid, chunkAccounts.getAccounts()); - } - fromUuid = chunkAccounts.getLastUuid(); - cacheLastUuid(fromUuid); - } - - } while (!chunkAccounts.getAccounts().isEmpty()); - - logger.info("{}: Finished crawl", name); - listeners.forEach(AccountDatabaseCrawlerListener::onCrawlEnd); - - } finally { - cache.releaseActiveWork(workerId); - } - } - - private AccountCrawlChunk readChunk(Optional fromUuid, int chunkSize) { - return readChunk(fromUuid, chunkSize, readChunkTimer); - } - - private AccountCrawlChunk readChunk(Optional fromUuid, int chunkSize, Timer readTimer) { - try (Timer.Context timer = readTimer.time()) { - - if (fromUuid.isPresent()) { - return accounts.getAllFromDynamo(fromUuid.get(), chunkSize); - } - - return accounts.getAllFromDynamo(chunkSize); - } - } - - private Optional getLastUuid() { - return cache.getLastUuid(); - } - - private void cacheLastUuid(final Optional lastUuid) { - cache.setLastUuid(lastUuid); - } - -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java deleted file mode 100644 index fb76a86d1..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2013-2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ -package org.whispersystems.textsecuregcm.storage; - -import io.lettuce.core.ScriptOutputType; -import io.lettuce.core.SetArgs; -import java.io.IOException; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import org.whispersystems.textsecuregcm.redis.ClusterLuaScript; -import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; - -@SuppressWarnings("OptionalUsedAsFieldOrParameterType") -public class AccountDatabaseCrawlerCache { - - public static final String GENERAL_PURPOSE_PREFIX = ""; - public static final String ACCOUNT_CLEANER_PREFIX = "account-cleaner"; - - private static final String ACTIVE_WORKER_KEY = "account_database_crawler_cache_active_worker"; - private static final String LAST_UUID_DYNAMO_KEY = "account_database_crawler_cache_last_uuid_dynamo"; - - private static final long LAST_NUMBER_TTL_MS = 86400_000L; - - private final FaultTolerantRedisCluster cacheCluster; - private final ClusterLuaScript unlockClusterScript; - - private final String prefix; - - public AccountDatabaseCrawlerCache(FaultTolerantRedisCluster cacheCluster, String prefix) throws IOException { - this.cacheCluster = cacheCluster; - this.unlockClusterScript = ClusterLuaScript.fromResource(cacheCluster, "lua/account_database_crawler/unlock.lua", - ScriptOutputType.INTEGER); - - this.prefix = prefix + "::"; - } - - public boolean claimActiveWork(String workerId, long ttlMs) { - return "OK".equals(cacheCluster.withCluster(connection -> connection.sync() - .set(getPrefixedKey(ACTIVE_WORKER_KEY), workerId, SetArgs.Builder.nx().px(ttlMs)))); - } - - public void releaseActiveWork(String workerId) { - unlockClusterScript.execute(List.of(getPrefixedKey(ACTIVE_WORKER_KEY)), List.of(workerId)); - } - - public Optional getLastUuid() { - final String lastUuidString = cacheCluster.withCluster( - connection -> connection.sync().get(getPrefixedKey(LAST_UUID_DYNAMO_KEY))); - - if (lastUuidString == null) { - return Optional.empty(); - } else { - return Optional.of(UUID.fromString(lastUuidString)); - } - } - - public void setLastUuid(Optional lastUuid) { - if (lastUuid.isPresent()) { - cacheCluster.useCluster( - connection -> connection.sync() - .psetex(getPrefixedKey(LAST_UUID_DYNAMO_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString())); - } else { - cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_DYNAMO_KEY))); - } - } - - private String getPrefixedKey(final String key) { - return prefix + key; - } - -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerListener.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerListener.java deleted file mode 100644 index ff497dae5..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerListener.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ -package org.whispersystems.textsecuregcm.storage; - -import static com.codahale.metrics.MetricRegistry.name; - -import com.codahale.metrics.SharedMetricRegistries; -import com.codahale.metrics.Timer; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import org.whispersystems.textsecuregcm.util.Constants; - -@SuppressWarnings("OptionalUsedAsFieldOrParameterType") -public abstract class AccountDatabaseCrawlerListener { - - private final Timer processChunkTimer; - - abstract public void onCrawlStart(); - - abstract public void onCrawlEnd(); - - abstract protected void onCrawlChunk(Optional fromUuid, List chunkAccounts); - - public AccountDatabaseCrawlerListener() { - processChunkTimer = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME).timer(name(AccountDatabaseCrawlerListener.class, "processChunk", getClass().getSimpleName())); - } - - public void timeAndProcessCrawlChunk(Optional fromUuid, List chunkAccounts) { - try (Timer.Context timer = processChunkTimer.time()) { - onCrawlChunk(fromUuid, chunkAccounts); - } - } - -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java index e8a2670df..7b86260f5 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java @@ -94,8 +94,6 @@ public class Accounts extends AbstractDynamoDbStore { private static final Timer GET_BY_USERNAME_LINK_HANDLE_TIMER = Metrics.timer(name(Accounts.class, "getByUsernameLinkHandle")); private static final Timer GET_BY_PNI_TIMER = Metrics.timer(name(Accounts.class, "getByPni")); private static final Timer GET_BY_UUID_TIMER = Metrics.timer(name(Accounts.class, "getByUuid")); - private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(Accounts.class, "getAllFrom")); - private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(Accounts.class, "getAllFromOffset")); private static final Timer DELETE_TIMER = Metrics.timer(name(Accounts.class, "delete")); private static final String CONDITIONAL_CHECK_FAILED = "ConditionalCheckFailed"; @@ -144,9 +142,6 @@ public class Accounts extends AbstractDynamoDbStore { private final String deletedAccountsTableName; private final String accountsTableName; - private final int scanPageSize; - - @VisibleForTesting public Accounts( final Clock clock, @@ -156,8 +151,7 @@ public class Accounts extends AbstractDynamoDbStore { final String phoneNumberConstraintTableName, final String phoneNumberIdentifierConstraintTableName, final String usernamesConstraintTableName, - final String deletedAccountsTableName, - final int scanPageSize) { + final String deletedAccountsTableName) { super(client); this.clock = clock; this.asyncClient = asyncClient; @@ -166,7 +160,6 @@ public class Accounts extends AbstractDynamoDbStore { this.accountsTableName = accountsTableName; this.usernamesConstraintTableName = usernamesConstraintTableName; this.deletedAccountsTableName = deletedAccountsTableName; - this.scanPageSize = scanPageSize; } public Accounts( @@ -176,11 +169,10 @@ public class Accounts extends AbstractDynamoDbStore { final String phoneNumberConstraintTableName, final String phoneNumberIdentifierConstraintTableName, final String usernamesConstraintTableName, - final String deletedAccountsTableName, - final int scanPageSize) { + final String deletedAccountsTableName) { this(Clock.systemUTC(), client, asyncClient, accountsTableName, phoneNumberConstraintTableName, phoneNumberIdentifierConstraintTableName, usernamesConstraintTableName, - deletedAccountsTableName, scanPageSize); + deletedAccountsTableName); } public boolean create(final Account account) { @@ -856,23 +848,6 @@ public class Accounts extends AbstractDynamoDbStore { .map(Accounts::fromItem)); } - @Nonnull - public AccountCrawlChunk getAllFrom(final UUID from, final int maxCount) { - final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder() - .limit(scanPageSize) - .exclusiveStartKey(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(from))); - - return scanForChunk(scanRequestBuilder, maxCount, GET_ALL_FROM_OFFSET_TIMER); - } - - @Nonnull - public AccountCrawlChunk getAllFromStart(final int maxCount) { - final ScanRequest.Builder scanRequestBuilder = ScanRequest.builder() - .limit(scanPageSize); - - return scanForChunk(scanRequestBuilder, maxCount, GET_ALL_FROM_START_TIMER); - } - @Nonnull private Optional getByIndirectLookup( final Timer timer, @@ -1103,14 +1078,6 @@ public class Accounts extends AbstractDynamoDbStore { .build(); } - @Nonnull - private AccountCrawlChunk scanForChunk(final ScanRequest.Builder scanRequestBuilder, final int maxCount, final Timer timer) { - scanRequestBuilder.tableName(accountsTableName); - final List> items = requireNonNull(timer.record(() -> scan(scanRequestBuilder.build(), maxCount))); - final List accounts = items.stream().map(Accounts::fromItem).toList(); - return new AccountCrawlChunk(accounts, accounts.size() > 0 ? accounts.get(accounts.size() - 1).getUuid() : null); - } - @Nonnull private static String extractCancellationReasonCodes(final TransactionCanceledException exception) { return exception.cancellationReasons().stream() diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java index 427dfbc56..7ed59ea0d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java @@ -837,14 +837,6 @@ public class AccountsManager { return accounts.findRecentlyDeletedE164(uuid); } - public AccountCrawlChunk getAllFromDynamo(int length) { - return accounts.getAllFromStart(length); - } - - public AccountCrawlChunk getAllFromDynamo(UUID uuid, int length) { - return accounts.getAllFrom(uuid, length); - } - public ParallelFlux streamAllFromDynamo(final int segments, final Scheduler scheduler) { return accounts.getAll(segments, scheduler); } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessor.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessor.java deleted file mode 100644 index ae1875143..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessor.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import static com.codahale.metrics.MetricRegistry.name; - -import com.codahale.metrics.Meter; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.SharedMetricRegistries; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Metrics; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.whispersystems.textsecuregcm.metrics.MetricsUtil; -import org.whispersystems.textsecuregcm.util.Constants; -import org.whispersystems.textsecuregcm.util.Util; - -public class PushFeedbackProcessor extends AccountDatabaseCrawlerListener { - - private static final Logger log = LoggerFactory.getLogger(PushFeedbackProcessor.class); - - private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME); - private final Meter expired = metricRegistry.meter(name(getClass(), "unregistered", "expired")); - private final Meter recovered = metricRegistry.meter(name(getClass(), "unregistered", "recovered")); - - private static final Counter UPDATED_ACCOUNT_COUNTER = Metrics.counter( - MetricsUtil.name(PushFeedbackProcessor.class, "updatedAccounts")); - - - private final AccountsManager accountsManager; - private final ExecutorService updateExecutor; - - public PushFeedbackProcessor(AccountsManager accountsManager, ExecutorService updateExecutor) { - this.accountsManager = accountsManager; - this.updateExecutor = updateExecutor; - } - - @Override - public void onCrawlStart() {} - - @Override - public void onCrawlEnd() { - } - - @Override - protected void onCrawlChunk(Optional fromUuid, List chunkAccounts) { - - final List> updateFutures = chunkAccounts.stream() - .filter(account -> { - boolean update = false; - - for (Device device : account.getDevices()) { - if (deviceNeedsUpdate(device)) { - if (deviceExpired(device)) { - if (device.isEnabled()) { - expired.mark(); - update = true; - } - } else { - recovered.mark(); - update = true; - } - } - } - - return update; - }) - .map(account -> CompletableFuture.runAsync(() -> { - // fetch a new version, since the chunk is shared and implicitly read-only - accountsManager.getByAccountIdentifier(account.getUuid()).ifPresent(accountToUpdate -> { - accountsManager.update(accountToUpdate, a -> { - for (Device device : a.getDevices()) { - if (deviceNeedsUpdate(device)) { - if (deviceExpired(device)) { - if (StringUtils.isNotEmpty(device.getApnId())) { - if (device.getId() == 1) { - device.setUserAgent("OWI"); - } else { - device.setUserAgent("OWP"); - } - } else if (StringUtils.isNotEmpty(device.getGcmId())) { - device.setUserAgent("OWA"); - } - device.setGcmId(null); - device.setApnId(null); - device.setVoipApnId(null); - device.setFetchesMessages(false); - } else { - device.setUninstalledFeedbackTimestamp(0); - } - } - } - }); - }); - }, updateExecutor) - .whenComplete((ignored, throwable) -> { - if (throwable != null) { - log.warn("Failed to update account {}", account.getUuid(), throwable); - } else { - UPDATED_ACCOUNT_COUNTER.increment(); - } - })) - .toList(); - - try { - CompletableFuture.allOf(updateFutures.toArray(new CompletableFuture[0])) - .orTimeout(10, TimeUnit.MINUTES) - .join(); - } catch (final Exception e) { - log.debug("Failed to update one or more accounts in chunk", e); - } - } - - private boolean deviceNeedsUpdate(final Device device) { - return device.getUninstalledFeedbackTimestamp() != 0 && - device.getUninstalledFeedbackTimestamp() + TimeUnit.DAYS.toMillis(2) <= Util.todayInMillis(); - } - - private boolean deviceExpired(final Device device) { - return device.getLastSeen() + TimeUnit.DAYS.toMillis(2) <= Util.todayInMillis(); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignUsernameCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignUsernameCommand.java index 0b899071f..12549f5de 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignUsernameCommand.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/AssignUsernameCommand.java @@ -156,8 +156,7 @@ public class AssignUsernameCommand extends EnvironmentCommand { - - private static final String CRAWL_TYPE = "crawlType"; - private static final String WORKER_COUNT = "workers"; - - private static final Logger logger = LoggerFactory.getLogger(CrawlAccountsCommand.class); - - public enum CrawlType implements ArgumentType { - GENERAL_PURPOSE, - ; - - @Override - public CrawlType convert(final ArgumentParser parser, final Argument arg, final String value) - throws ArgumentParserException { - return CrawlType.valueOf(value); - } - } - - public CrawlAccountsCommand() { - super(new Application<>() { - @Override - public void run(final WhisperServerConfiguration configuration, final Environment environment) throws Exception { - - } - }, "crawl-accounts", "Runs account crawler tasks"); - } - - @Override - public void configure(final Subparser subparser) { - super.configure(subparser); - subparser.addArgument("--crawl-type") - .type(CrawlType.class) - .dest(CRAWL_TYPE) - .required(true) - .help("The type of crawl to perform"); - - subparser.addArgument("--workers") - .type(Integer.class) - .dest(WORKER_COUNT) - .required(true) - .help("The number of worker threads"); - } - - @Override - protected void run(final Environment environment, final Namespace namespace, - final WhisperServerConfiguration configuration) throws Exception { - - UncaughtExceptionHandler.register(); - - MetricsUtil.configureRegistries(configuration, environment); - - final CommandDependencies deps = CommandDependencies.build("account-crawler", environment, configuration); - final AccountsManager accountsManager = deps.accountsManager(); - - final FaultTolerantRedisCluster cacheCluster = deps.cacheCluster(); - final FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", - configuration.getMetricsClusterConfiguration(), deps.redisClusterClientResources()); - - final DynamicConfigurationManager dynamicConfigurationManager = - new DynamicConfigurationManager<>(configuration.getAppConfig().getApplication(), - configuration.getAppConfig().getEnvironment(), - configuration.getAppConfig().getConfigurationName(), - DynamicConfiguration.class); - - dynamicConfigurationManager.start(); - MetricsUtil.registerSystemResourceMetrics(environment); - - final int workers = Objects.requireNonNull(namespace.getInt(WORKER_COUNT)); - - final AccountDatabaseCrawler crawler = switch ((CrawlType) namespace.get(CRAWL_TYPE)) { - case GENERAL_PURPOSE -> { - final ExecutorService pushFeedbackUpdateExecutor = environment.lifecycle() - .executorService(name(getClass(), "pushFeedback-%d")).maxThreads(workers).minThreads(workers).build(); - - // TODO listeners must be ordered so that ones that directly update accounts come last, so that read-only ones are not working with stale data - final List accountDatabaseCrawlerListeners = List.of( - // PushFeedbackProcessor may update device properties - new PushFeedbackProcessor(accountsManager, pushFeedbackUpdateExecutor)); - - final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache( - cacheCluster, - AccountDatabaseCrawlerCache.GENERAL_PURPOSE_PREFIX); - - yield new AccountDatabaseCrawler("General-purpose account crawler", - accountsManager, - accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners, - configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize() - ); - } - }; - - environment.lifecycle().manage(new CommandStopListener(configuration.getCommandStopListener())); - - environment.lifecycle().getManagedObjects().forEach(managedObject -> { - try { - managedObject.start(); - } catch (final Exception e) { - logger.error("Failed to start managed object", e); - throw new RuntimeException(e); - } - }); - - try { - crawler.crawlAllAccounts(); - } catch (final Exception e) { - LoggerFactory.getLogger(CrawlAccountsCommand.class).error("Error crawling accounts", e); - } - - environment.lifecycle().getManagedObjects().forEach(managedObject -> { - try { - managedObject.stop(); - } catch (final Exception e) { - logger.error("Failed to stop managed object", e); - } - }); - } -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/filters/RemoteDeprecationFilterTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/filters/RemoteDeprecationFilterTest.java index 48572f294..11db14dca 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/filters/RemoteDeprecationFilterTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/filters/RemoteDeprecationFilterTest.java @@ -17,11 +17,15 @@ import static org.mockito.Mockito.when; import com.google.common.net.HttpHeaders; import com.google.protobuf.ByteString; import com.vdurmont.semver4j.Semver; +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.StatusRuntimeException; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; import java.io.IOException; import java.util.EnumMap; import java.util.Set; import java.util.stream.Stream; - import javax.servlet.FilterChain; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -30,8 +34,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.signal.chat.rpc.EchoServiceGrpc; import org.signal.chat.rpc.EchoRequest; +import org.signal.chat.rpc.EchoServiceGrpc; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicRemoteDeprecationConfiguration; import org.whispersystems.textsecuregcm.grpc.EchoServiceImpl; @@ -40,15 +44,6 @@ import org.whispersystems.textsecuregcm.grpc.UserAgentInterceptor; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.util.ua.ClientPlatform; -import io.grpc.Metadata; -import io.grpc.ManagedChannel; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.StatusRuntimeException; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.stub.MetadataUtils; - class RemoteDeprecationFilterTest { @Test diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java deleted file mode 100644 index be7073460..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerIntegrationTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.whispersystems.textsecuregcm.redis.RedisClusterExtension; - -class AccountDatabaseCrawlerIntegrationTest { - - @RegisterExtension - static final RedisClusterExtension REDIS_CLUSTER_EXTENSION = RedisClusterExtension.builder().build(); - - private static final UUID FIRST_UUID = UUID.fromString("82339e80-81cd-48e2-9ed2-ccd5dd262ad9"); - private static final UUID SECOND_UUID = UUID.fromString("cc705c84-33cf-456b-8239-a6a34e2f561a"); - - private Account firstAccount; - private Account secondAccount; - - private AccountsManager accountsManager; - private AccountDatabaseCrawlerListener listener; - - private AccountDatabaseCrawler accountDatabaseCrawler; - - private static final int CHUNK_SIZE = 1; - - @BeforeEach - void setUp() throws Exception { - - firstAccount = mock(Account.class); - secondAccount = mock(Account.class); - - accountsManager = mock(AccountsManager.class); - listener = mock(AccountDatabaseCrawlerListener.class); - - when(firstAccount.getUuid()).thenReturn(FIRST_UUID); - when(secondAccount.getUuid()).thenReturn(SECOND_UUID); - - when(accountsManager.getAllFromDynamo(CHUNK_SIZE)).thenReturn( - new AccountCrawlChunk(List.of(firstAccount), FIRST_UUID)); - when(accountsManager.getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE))) - .thenReturn(new AccountCrawlChunk(List.of(secondAccount), SECOND_UUID)) - .thenReturn(new AccountCrawlChunk(Collections.emptyList(), null)); - - final AccountDatabaseCrawlerCache crawlerCache = new AccountDatabaseCrawlerCache( - REDIS_CLUSTER_EXTENSION.getRedisCluster(), "test"); - accountDatabaseCrawler = new AccountDatabaseCrawler("test", accountsManager, crawlerCache, List.of(listener), - CHUNK_SIZE); - } - - @Test - void testCrawlAllAccounts() throws Exception { - accountDatabaseCrawler.crawlAllAccounts(); - - verify(accountsManager).getAllFromDynamo(CHUNK_SIZE); - verify(accountsManager).getAllFromDynamo(FIRST_UUID, CHUNK_SIZE); - verify(accountsManager).getAllFromDynamo(SECOND_UUID, CHUNK_SIZE); - - verify(listener).onCrawlStart(); - verify(listener).timeAndProcessCrawlChunk(Optional.empty(), List.of(firstAccount)); - verify(listener).timeAndProcessCrawlChunk(Optional.of(FIRST_UUID), List.of(secondAccount)); - verify(listener).onCrawlEnd(); - } -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerTest.java deleted file mode 100644 index f17a13b61..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2013 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class AccountDatabaseCrawlerTest { - - private static final UUID ACCOUNT1 = UUID.randomUUID(); - private static final UUID ACCOUNT2 = UUID.randomUUID(); - - private static final int CHUNK_SIZE = 1000; - - private final Account account1 = mock(Account.class); - private final Account account2 = mock(Account.class); - - private final AccountsManager accounts = mock(AccountsManager.class); - private final AccountDatabaseCrawlerListener listener = mock(AccountDatabaseCrawlerListener.class); - private final AccountDatabaseCrawlerCache cache = mock(AccountDatabaseCrawlerCache.class); - - private final AccountDatabaseCrawler crawler = - new AccountDatabaseCrawler("test", accounts, cache, List.of(listener), CHUNK_SIZE); - - @BeforeEach - void setup() { - when(account1.getUuid()).thenReturn(ACCOUNT1); - when(account2.getUuid()).thenReturn(ACCOUNT2); - - when(accounts.getAllFromDynamo(anyInt())).thenReturn( - new AccountCrawlChunk(List.of(account1, account2), ACCOUNT2)); - when(accounts.getAllFromDynamo(eq(ACCOUNT1), anyInt())).thenReturn( - new AccountCrawlChunk(List.of(account2), ACCOUNT2)); - when(accounts.getAllFromDynamo(eq(ACCOUNT2), anyInt())).thenReturn( - new AccountCrawlChunk(Collections.emptyList(), null)); - - when(cache.claimActiveWork(any(), anyLong())).thenReturn(true); - } - - @Test - void testCrawlAllAccounts() { - when(cache.getLastUuid()) - .thenReturn(Optional.empty()); - - crawler.crawlAllAccounts(); - - verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(1)).getLastUuid(); - verify(listener, times(1)).onCrawlStart(); - verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE)); - verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE)); - verify(listener, times(1)).timeAndProcessCrawlChunk(Optional.empty(), List.of(account1, account2)); - verify(listener, times(1)).timeAndProcessCrawlChunk(Optional.of(ACCOUNT2), Collections.emptyList()); - verify(listener, times(1)).onCrawlEnd(); - verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2))); - // times(2) because empty() will get cached on the last run of loop and then again at the end - verify(cache, times(1)).setLastUuid(eq(Optional.empty())); - verify(cache, times(1)).releaseActiveWork(any(String.class)); - - verifyNoMoreInteractions(accounts); - verifyNoMoreInteractions(listener); - verifyNoMoreInteractions(cache); - } - -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java index d455df7fd..8b879c4be 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java @@ -45,8 +45,6 @@ import org.whispersystems.textsecuregcm.tests.util.KeysHelper; class AccountsManagerChangeNumberIntegrationTest { - private static final int SCAN_PAGE_SIZE = 1; - @RegisterExtension static final DynamoDbExtension DYNAMO_DB_EXTENSION = new DynamoDbExtension( Tables.ACCOUNTS, @@ -82,8 +80,7 @@ class AccountsManagerChangeNumberIntegrationTest { Tables.NUMBERS.tableName(), Tables.PNI_ASSIGNMENTS.tableName(), Tables.USERNAMES.tableName(), - Tables.DELETED_ACCOUNTS.tableName(), - SCAN_PAGE_SIZE); + Tables.DELETED_ACCOUNTS.tableName()); accountLockExecutor = Executors.newSingleThreadExecutor(); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java index d3fd89e4c..0f12d3315 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java @@ -57,8 +57,6 @@ import org.whispersystems.textsecuregcm.util.Pair; class AccountsManagerConcurrentModificationIntegrationTest { - private static final int SCAN_PAGE_SIZE = 1; - @RegisterExtension static final DynamoDbExtension DYNAMO_DB_EXTENSION = new DynamoDbExtension( Tables.ACCOUNTS, @@ -89,8 +87,7 @@ class AccountsManagerConcurrentModificationIntegrationTest { Tables.NUMBERS.tableName(), Tables.PNI_ASSIGNMENTS.tableName(), Tables.USERNAMES.tableName(), - Tables.DELETED_ACCOUNTS.tableName(), - SCAN_PAGE_SIZE); + Tables.DELETED_ACCOUNTS.tableName()); { //noinspection unchecked diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java index 4b95d27cd..22719f827 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java @@ -58,7 +58,6 @@ class AccountsManagerUsernameIntegrationTest { private static final String BASE_64_URL_USERNAME_HASH_2 = "NLUom-CHwtemcdvOTTXdmXmzRIV7F05leS8lwkVK_vc"; private static final String BASE_64_URL_ENCRYPTED_USERNAME_1 = "md1votbj9r794DsqTNrBqA"; private static final String BASE_64_URL_ENCRYPTED_USERNAME_2 = "9hrqVLy59bzgPse-S9NUsA"; - private static final int SCAN_PAGE_SIZE = 1; private static final byte[] USERNAME_HASH_1 = Base64.getUrlDecoder().decode(BASE_64_URL_USERNAME_HASH_1); private static final byte[] USERNAME_HASH_2 = Base64.getUrlDecoder().decode(BASE_64_URL_USERNAME_HASH_2); private static final byte[] ENCRYPTED_USERNAME_1 = Base64.getUrlDecoder().decode(BASE_64_URL_ENCRYPTED_USERNAME_1); @@ -99,8 +98,7 @@ class AccountsManagerUsernameIntegrationTest { Tables.NUMBERS.tableName(), Tables.PNI_ASSIGNMENTS.tableName(), Tables.USERNAMES.tableName(), - Tables.DELETED_ACCOUNTS.tableName(), - SCAN_PAGE_SIZE)); + Tables.DELETED_ACCOUNTS.tableName())); final AccountLockManager accountLockManager = mock(AccountLockManager.class); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsTest.java index 0b5973bca..8b677ce9b 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsTest.java @@ -19,7 +19,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.uuid.UUIDComparator; import java.nio.charset.StandardCharsets; import java.security.SecureRandom; import java.time.Duration; @@ -85,8 +84,6 @@ class AccountsTest { private static final byte[] ENCRYPTED_USERNAME_1 = Base64.getUrlDecoder().decode(BASE_64_URL_ENCRYPTED_USERNAME_1); private static final byte[] ENCRYPTED_USERNAME_2 = Base64.getUrlDecoder().decode(BASE_64_URL_ENCRYPTED_USERNAME_2); - private static final int SCAN_PAGE_SIZE = 1; - private static final AtomicInteger ACCOUNT_COUNTER = new AtomicInteger(1); @@ -119,8 +116,7 @@ class AccountsTest { Tables.NUMBERS.tableName(), Tables.PNI_ASSIGNMENTS.tableName(), Tables.USERNAMES.tableName(), - Tables.DELETED_ACCOUNTS.tableName(), - SCAN_PAGE_SIZE); + Tables.DELETED_ACCOUNTS.tableName()); } @Test @@ -423,7 +419,7 @@ class AccountsTest { accounts = new Accounts(mock(DynamoDbClient.class), dynamoDbAsyncClient, Tables.ACCOUNTS.tableName(), Tables.NUMBERS.tableName(), Tables.PNI_ASSIGNMENTS.tableName(), Tables.USERNAMES.tableName(), - Tables.DELETED_ACCOUNTS.tableName(), SCAN_PAGE_SIZE); + Tables.DELETED_ACCOUNTS.tableName()); Exception e = TransactionConflictException.builder().build(); e = wrapException ? new CompletionException(e) : e; @@ -436,55 +432,6 @@ class AccountsTest { assertThatThrownBy(() -> accounts.update(account)).isInstanceOfAny(ContestedOptimisticLockException.class); } - @Test - void testRetrieveFrom() { - List users = new ArrayList<>(); - - for (int i = 1; i <= 100; i++) { - Account account = generateAccount("+1" + String.format("%03d", i), UUID.randomUUID(), UUID.randomUUID()); - users.add(account); - accounts.create(account); - } - - users.sort((account, t1) -> UUIDComparator.staticCompare(account.getUuid(), t1.getUuid())); - - AccountCrawlChunk retrieved = accounts.getAllFromStart(10); - assertThat(retrieved.getAccounts().size()).isEqualTo(10); - - for (int i = 0; i < retrieved.getAccounts().size(); i++) { - final Account retrievedAccount = retrieved.getAccounts().get(i); - - final Account expectedAccount = users.stream() - .filter(account -> account.getUuid().equals(retrievedAccount.getUuid())) - .findAny() - .orElseThrow(); - - verifyStoredState(expectedAccount.getNumber(), expectedAccount.getUuid(), expectedAccount.getPhoneNumberIdentifier(), null, retrievedAccount, expectedAccount); - - users.remove(expectedAccount); - } - - for (int j = 0; j < 9; j++) { - retrieved = accounts.getAllFrom(retrieved.getLastUuid().orElseThrow(), 10); - assertThat(retrieved.getAccounts().size()).isEqualTo(10); - - for (int i = 0; i < retrieved.getAccounts().size(); i++) { - final Account retrievedAccount = retrieved.getAccounts().get(i); - - final Account expectedAccount = users.stream() - .filter(account -> account.getUuid().equals(retrievedAccount.getUuid())) - .findAny() - .orElseThrow(); - - verifyStoredState(expectedAccount.getNumber(), expectedAccount.getUuid(), expectedAccount.getPhoneNumberIdentifier(), null, retrievedAccount, expectedAccount); - - users.remove(expectedAccount); - } - } - - assertThat(users).isEmpty(); - } - @Test void testGetAll() { final List expectedAccounts = new ArrayList<>(); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessorTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessorTest.java deleted file mode 100644 index 62bd69c64..000000000 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessorTest.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright 2013 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyBoolean; -import static org.mockito.Mockito.clearInvocations; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.isNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; -import static org.whispersystems.textsecuregcm.tests.util.AccountsHelper.eqUuid; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.whispersystems.textsecuregcm.tests.util.AccountsHelper; -import org.whispersystems.textsecuregcm.util.Util; - -class PushFeedbackProcessorTest { - - private final AccountsManager accountsManager = mock(AccountsManager.class); - - private Account uninstalledAccount = mock(Account.class); - private Account mixedAccount = mock(Account.class); - private Account freshAccount = mock(Account.class); - private Account cleanAccount = mock(Account.class); - private Account stillActiveAccount = mock(Account.class); - - private Device uninstalledDevice = mock(Device.class); - private Device uninstalledDeviceTwo = mock(Device.class); - private Device installedDevice = mock(Device.class); - private Device installedDeviceTwo = mock(Device.class); - private Device recentUninstalledDevice = mock(Device.class); - private Device stillActiveDevice = mock(Device.class); - - @BeforeEach - void setup() { - AccountsHelper.setupMockUpdate(accountsManager); - - when(uninstalledDevice.getUninstalledFeedbackTimestamp()).thenReturn( - Util.todayInMillis() - TimeUnit.DAYS.toMillis(2)); - when(uninstalledDevice.getLastSeen()).thenReturn(Util.todayInMillis() - TimeUnit.DAYS.toMillis(2)); - when(uninstalledDevice.isEnabled()).thenReturn(true); - when(uninstalledDeviceTwo.getUninstalledFeedbackTimestamp()).thenReturn( - Util.todayInMillis() - TimeUnit.DAYS.toMillis(3)); - when(uninstalledDeviceTwo.getLastSeen()).thenReturn(Util.todayInMillis() - TimeUnit.DAYS.toMillis(3)); - when(uninstalledDeviceTwo.isEnabled()).thenReturn(true); - - when(installedDevice.getUninstalledFeedbackTimestamp()).thenReturn(0L); - when(installedDevice.isEnabled()).thenReturn(true); - when(installedDeviceTwo.getUninstalledFeedbackTimestamp()).thenReturn(0L); - when(installedDeviceTwo.isEnabled()).thenReturn(true); - - when(recentUninstalledDevice.getUninstalledFeedbackTimestamp()).thenReturn( - Util.todayInMillis() - TimeUnit.DAYS.toMillis(1)); - when(recentUninstalledDevice.getLastSeen()).thenReturn(Util.todayInMillis()); - when(recentUninstalledDevice.isEnabled()).thenReturn(true); - - when(stillActiveDevice.getUninstalledFeedbackTimestamp()).thenReturn( - Util.todayInMillis() - TimeUnit.DAYS.toMillis(2)); - when(stillActiveDevice.getLastSeen()).thenReturn(Util.todayInMillis()); - when(stillActiveDevice.isEnabled()).thenReturn(true); - - when(uninstalledAccount.getDevices()).thenReturn(List.of(uninstalledDevice)); - when(mixedAccount.getDevices()).thenReturn(List.of(installedDevice, uninstalledDeviceTwo)); - when(freshAccount.getDevices()).thenReturn(List.of(recentUninstalledDevice)); - when(cleanAccount.getDevices()).thenReturn(List.of(installedDeviceTwo)); - when(stillActiveAccount.getDevices()).thenReturn(List.of(stillActiveDevice)); - - when(mixedAccount.getUuid()).thenReturn(UUID.randomUUID()); - when(freshAccount.getUuid()).thenReturn(UUID.randomUUID()); - when(cleanAccount.getUuid()).thenReturn(UUID.randomUUID()); - when(stillActiveAccount.getUuid()).thenReturn(UUID.randomUUID()); - - when(uninstalledAccount.isEnabled()).thenReturn(true); - when(uninstalledAccount.isDiscoverableByPhoneNumber()).thenReturn(true); - when(uninstalledAccount.getUuid()).thenReturn(UUID.randomUUID()); - when(uninstalledAccount.getNumber()).thenReturn("+18005551234"); - - AccountsHelper.setupMockGet(accountsManager, - Set.of(uninstalledAccount, mixedAccount, freshAccount, cleanAccount, stillActiveAccount)); - } - - @Test - void testEmpty() { - PushFeedbackProcessor processor = new PushFeedbackProcessor(accountsManager, Executors.newSingleThreadExecutor()); - processor.timeAndProcessCrawlChunk(Optional.of(UUID.randomUUID()), Collections.emptyList()); - - verifyNoInteractions(accountsManager); - } - - @Test - void testUpdate() { - PushFeedbackProcessor processor = new PushFeedbackProcessor(accountsManager, Executors.newSingleThreadExecutor()); - processor.timeAndProcessCrawlChunk(Optional.of(UUID.randomUUID()), - List.of(uninstalledAccount, mixedAccount, stillActiveAccount, freshAccount, cleanAccount)); - - verify(uninstalledDevice).setApnId(isNull()); - verify(uninstalledDevice).setGcmId(isNull()); - verify(uninstalledDevice).setFetchesMessages(eq(false)); - when(uninstalledDevice.isEnabled()).thenReturn(false); - - verify(accountsManager).update(eqUuid(uninstalledAccount), any()); - - verify(uninstalledDeviceTwo).setApnId(isNull()); - verify(uninstalledDeviceTwo).setGcmId(isNull()); - verify(uninstalledDeviceTwo).setFetchesMessages(eq(false)); - when(uninstalledDeviceTwo.isEnabled()).thenReturn(false); - - verify(installedDevice, never()).setApnId(any()); - verify(installedDevice, never()).setGcmId(any()); - verify(installedDevice, never()).setFetchesMessages(anyBoolean()); - - verify(accountsManager).update(eqUuid(mixedAccount), any()); - - verify(recentUninstalledDevice, never()).setApnId(any()); - verify(recentUninstalledDevice, never()).setGcmId(any()); - verify(recentUninstalledDevice, never()).setFetchesMessages(anyBoolean()); - - verify(accountsManager, never()).update(eqUuid(freshAccount), any()); - - verify(installedDeviceTwo, never()).setApnId(any()); - verify(installedDeviceTwo, never()).setGcmId(any()); - verify(installedDeviceTwo, never()).setFetchesMessages(anyBoolean()); - - verify(accountsManager, never()).update(eqUuid(cleanAccount), any()); - - verify(stillActiveDevice).setUninstalledFeedbackTimestamp(eq(0L)); - verify(stillActiveDevice, never()).setApnId(any()); - verify(stillActiveDevice, never()).setGcmId(any()); - verify(stillActiveDevice, never()).setFetchesMessages(anyBoolean()); - when(stillActiveDevice.getUninstalledFeedbackTimestamp()).thenReturn(0L); - - verify(accountsManager).update(eqUuid(stillActiveAccount), any()); - - // there are un-verified calls to updateDevice - clearInvocations(accountsManager); - - // a second crawl should not make any further updates - processor.timeAndProcessCrawlChunk(Optional.of(UUID.randomUUID()), - List.of(uninstalledAccount, mixedAccount, stillActiveAccount, freshAccount, cleanAccount)); - - verify(accountsManager, never()).update(any(Account.class), any()); - } - -}