From 69742839c045710fb48ebfd8447a57d13183b2d5 Mon Sep 17 00:00:00 2001 From: Jeffrey Griffin Date: Mon, 26 Aug 2019 14:11:23 -0700 Subject: [PATCH] uuid-based account crawler --- .../entities/ActiveUserTally.java | 15 ++-- .../DirectoryReconciliationRequest.java | 72 +++++++++++++++---- .../textsecuregcm/storage/AccountCleaner.java | 5 +- .../storage/AccountDatabaseCrawler.java | 24 +++---- .../storage/AccountDatabaseCrawlerCache.java | 18 +++-- .../AccountDatabaseCrawlerListener.java | 5 +- .../textsecuregcm/storage/Accounts.java | 6 +- .../storage/AccountsManager.java | 4 +- .../storage/ActiveUserCounter.java | 19 +++-- .../storage/DirectoryReconciler.java | 35 +++++---- .../DirectoryReconciliationClient.java | 2 +- .../storage/PushFeedbackProcessor.java | 5 +- .../storage/AccountDatabaseCrawlerTest.java | 49 ++++++------- .../tests/storage/AccountsTest.java | 4 +- .../tests/storage/ActiveUserCounterTest.java | 32 +++++---- .../storage/DirectoryReconcilerTest.java | 27 ++++--- .../storage/PushFeedbackProcessorTest.java | 5 +- 17 files changed, 202 insertions(+), 125 deletions(-) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/entities/ActiveUserTally.java b/service/src/main/java/org/whispersystems/textsecuregcm/entities/ActiveUserTally.java index 5976db2c4..5cb88e7c0 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/entities/ActiveUserTally.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/entities/ActiveUserTally.java @@ -19,10 +19,11 @@ package org.whispersystems.textsecuregcm.entities; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Map; import java.util.HashMap; +import java.util.UUID; public class ActiveUserTally { @JsonProperty - private String fromNumber; + private UUID fromUuid; @JsonProperty private Map platforms; @@ -32,14 +33,14 @@ public class ActiveUserTally { public ActiveUserTally() {} - public ActiveUserTally(String fromNumber, Map platforms, Map countries) { - this.fromNumber = fromNumber; + public ActiveUserTally(UUID fromUuid, Map platforms, Map countries) { + this.fromUuid = fromUuid; this.platforms = platforms; this.countries = countries; } - public String getFromNumber() { - return this.fromNumber; + public UUID getFromUuid() { + return this.fromUuid; } public Map getPlatforms() { @@ -50,8 +51,8 @@ public class ActiveUserTally { return this.countries; } - public void setFromNumber(String fromNumber) { - this.fromNumber = fromNumber; + public void setFromUuid(UUID fromUuid) { + this.fromUuid = fromUuid; } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/entities/DirectoryReconciliationRequest.java b/service/src/main/java/org/whispersystems/textsecuregcm/entities/DirectoryReconciliationRequest.java index 2731347eb..b1224d1b3 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/entities/DirectoryReconciliationRequest.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/entities/DirectoryReconciliationRequest.java @@ -18,38 +18,84 @@ package org.whispersystems.textsecuregcm.entities; import com.fasterxml.jackson.annotation.JsonProperty; +import javax.validation.constraints.NotNull; import java.util.List; +import java.util.UUID; public class DirectoryReconciliationRequest { @JsonProperty - private String fromNumber; + private UUID fromUuid; @JsonProperty - private String toNumber; + private UUID toUuid; @JsonProperty - private List numbers; + private List users; public DirectoryReconciliationRequest() { } - public DirectoryReconciliationRequest(String fromNumber, String toNumber, List numbers) { - this.fromNumber = fromNumber; - this.toNumber = toNumber; - this.numbers = numbers; + public DirectoryReconciliationRequest(UUID fromUuid, UUID toUuid, List users) { + this.fromUuid = fromUuid; + this.toUuid = toUuid; + this.users = users; } - public String getFromNumber() { - return fromNumber; + public UUID getFromUuid() { + return fromUuid; } - public String getToNumber() { - return toNumber; + public UUID getToUuid() { + return toUuid; } - public List getNumbers() { - return numbers; + public List getUsers() { + return users; } + public static class User { + + @JsonProperty + private UUID uuid; + + @JsonProperty + private String number; + + public User() { + } + + public User(UUID uuid, String number) { + this.uuid = uuid; + this.number = number; + } + + public UUID getUuid() { + return uuid; + } + + public String getNumber() { + return number; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + User user = (User) o; + + if (uuid != null ? !uuid.equals(user.uuid) : user.uuid != null) return false; + if (number != null ? !number.equals(user.number) : user.number != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = uuid != null ? uuid.hashCode() : 0; + result = 31 * result + (number != null ? number.hashCode() : 0); + return result; + } + } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountCleaner.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountCleaner.java index 066ecb222..cf68231aa 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountCleaner.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountCleaner.java @@ -26,6 +26,7 @@ import org.whispersystems.textsecuregcm.util.Util; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static com.codahale.metrics.MetricRegistry.name; @@ -51,7 +52,7 @@ public class AccountCleaner implements AccountDatabaseCrawlerListener { } @Override - public void onCrawlChunk(Optional fromNumber, List chunkAccounts) { + public void onCrawlChunk(Optional fromUuid, List chunkAccounts) { int accountUpdateCount = 0; for (Account account : chunkAccounts) { if (needsExplicitRemoval(account)) { @@ -74,7 +75,7 @@ public class AccountCleaner implements AccountDatabaseCrawlerListener { } @Override - public void onCrawlEnd(Optional fromNumber) { + public void onCrawlEnd(Optional fromUuid) { } private boolean needsExplicitRemoval(Account account) { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java index 80cf9cc74..93d08d679 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java @@ -122,26 +122,26 @@ public class AccountDatabaseCrawler implements Managed, Runnable { } private void processChunk() { - Optional fromNumber = cache.getLastNumber(); + Optional fromUuid = cache.getLastUuid(); - if (!fromNumber.isPresent()) { - listeners.forEach(listener -> { listener.onCrawlStart(); }); + if (!fromUuid.isPresent()) { + listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart); } - List chunkAccounts = readChunk(fromNumber, chunkSize); + List chunkAccounts = readChunk(fromUuid, chunkSize); if (chunkAccounts.isEmpty()) { - listeners.forEach(listener -> { listener.onCrawlEnd(fromNumber); }); - cache.setLastNumber(Optional.empty()); + listeners.forEach(listener -> listener.onCrawlEnd(fromUuid)); + cache.setLastUuid(Optional.empty()); cache.clearAccelerate(); } else { try { for (AccountDatabaseCrawlerListener listener : listeners) { - listener.onCrawlChunk(fromNumber, chunkAccounts); + listener.onCrawlChunk(fromUuid, chunkAccounts); } - cache.setLastNumber(Optional.of(chunkAccounts.get(chunkAccounts.size() - 1).getNumber())); + cache.setLastUuid(Optional.of(chunkAccounts.get(chunkAccounts.size() - 1).getUuid())); } catch (AccountDatabaseCrawlerRestartException e) { - cache.setLastNumber(Optional.empty()); + cache.setLastUuid(Optional.empty()); cache.clearAccelerate(); } @@ -149,12 +149,12 @@ public class AccountDatabaseCrawler implements Managed, Runnable { } - private List readChunk(Optional fromNumber, int chunkSize) { + private List readChunk(Optional fromUuid, int chunkSize) { try (Timer.Context timer = readChunkTimer.time()) { List chunkAccounts; - if (fromNumber.isPresent()) { - chunkAccounts = accounts.getAllFrom(fromNumber.get(), chunkSize); + if (fromUuid.isPresent()) { + chunkAccounts = accounts.getAllFrom(fromUuid.get(), chunkSize); } else { chunkAccounts = accounts.getAllFrom(chunkSize); } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java index abd578c12..4b1402b1a 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.UUID; import redis.clients.jedis.Jedis; @@ -30,7 +31,7 @@ import redis.clients.jedis.Jedis; public class AccountDatabaseCrawlerCache { private static final String ACTIVE_WORKER_KEY = "account_database_crawler_cache_active_worker"; - private static final String LAST_NUMBER_KEY = "account_database_crawler_cache_last_number"; + private static final String LAST_UUID_KEY = "account_database_crawler_cache_last_uuid"; private static final String ACCELERATE_KEY = "account_database_crawler_cache_accelerate"; private static final long LAST_NUMBER_TTL_MS = 86400_000L; @@ -67,18 +68,21 @@ public class AccountDatabaseCrawlerCache { luaScript.execute(keys, args); } - public Optional getLastNumber() { + public Optional getLastUuid() { try (Jedis jedis = jedisPool.getWriteResource()) { - return Optional.ofNullable(jedis.get(LAST_NUMBER_KEY)); + String lastUuidString = jedis.get(LAST_UUID_KEY); + + if (lastUuidString == null) return Optional.empty(); + else return Optional.of(UUID.fromString(lastUuidString)); } } - public void setLastNumber(Optional lastNumber) { + public void setLastUuid(Optional lastUuid) { try (Jedis jedis = jedisPool.getWriteResource()) { - if (lastNumber.isPresent()) { - jedis.psetex(LAST_NUMBER_KEY, LAST_NUMBER_TTL_MS, lastNumber.get()); + if (lastUuid.isPresent()) { + jedis.psetex(LAST_UUID_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString()); } else { - jedis.del(LAST_NUMBER_KEY); + jedis.del(LAST_UUID_KEY); } } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerListener.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerListener.java index 0ffb673a1..590f61c38 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerListener.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerListener.java @@ -18,10 +18,11 @@ package org.whispersystems.textsecuregcm.storage; import java.util.List; import java.util.Optional; +import java.util.UUID; @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public interface AccountDatabaseCrawlerListener { void onCrawlStart(); - void onCrawlChunk(Optional fromNumber, List chunkAccounts) throws AccountDatabaseCrawlerRestartException; - void onCrawlEnd(Optional fromNumber); + void onCrawlChunk(Optional fromUuid, List chunkAccounts) throws AccountDatabaseCrawlerRestartException; + void onCrawlEnd(Optional fromUuid); } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java index c84e646ab..36b25db51 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java @@ -111,10 +111,10 @@ public class Accounts { })); } - public List getAllFrom(String from, int length) { + public List getAllFrom(UUID from, int length) { return database.with(jdbi -> jdbi.withHandle(handle -> { try (Timer.Context ignored = getAllFromOffsetTimer.time()) { - return handle.createQuery("SELECT * FROM accounts WHERE " + NUMBER + " > :from ORDER BY " + NUMBER + " LIMIT :limit") + return handle.createQuery("SELECT * FROM accounts WHERE " + UID + " > :from ORDER BY " + UID + " LIMIT :limit") .bind("from", from) .bind("limit", length) .mapTo(Account.class) @@ -126,7 +126,7 @@ public class Accounts { public List getAllFrom(int length) { return database.with(jdbi -> jdbi.withHandle(handle -> { try (Timer.Context ignored = getAllFromTimer.time()) { - return handle.createQuery("SELECT * FROM accounts ORDER BY " + NUMBER + " LIMIT :limit") + return handle.createQuery("SELECT * FROM accounts ORDER BY " + UID + " LIMIT :limit") .bind("limit", length) .mapTo(Account.class) .list(); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java index dcdd46445..6e94b9a6f 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java @@ -121,8 +121,8 @@ public class AccountsManager { return accounts.getAllFrom(length); } - public List getAllFrom(String number, int length) { - return accounts.getAllFrom(number, length); + public List getAllFrom(UUID uuid, int length) { + return accounts.getAllFrom(uuid, length); } private void updateDirectory(Account account) { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/ActiveUserCounter.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/ActiveUserCounter.java index 5c84baf7d..099cae5d5 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/ActiveUserCounter.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/ActiveUserCounter.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.TimeUnit; import io.dropwizard.metrics.MetricsFactory; @@ -43,8 +44,6 @@ public class ActiveUserCounter implements AccountDatabaseCrawlerListener { private static final String PLATFORM_IOS = "ios"; private static final String PLATFORM_ANDROID = "android"; - private static final String FIRST_FROM_NUMBER = "+"; - private static final String INTERVALS[] = {"daily", "weekly", "monthly", "quarterly", "yearly"}; private final MetricsFactory metricsFactory; @@ -64,7 +63,7 @@ public class ActiveUserCounter implements AccountDatabaseCrawlerListener { } @Override - public void onCrawlChunk(Optional fromNumber, List chunkAccounts) { + public void onCrawlChunk(Optional fromNumber, List chunkAccounts) { long nowDays = TimeUnit.MILLISECONDS.toDays(System.currentTimeMillis()); long agoMs[] = {TimeUnit.DAYS.toMillis(nowDays - 1), TimeUnit.DAYS.toMillis(nowDays - 7), @@ -107,12 +106,11 @@ public class ActiveUserCounter implements AccountDatabaseCrawlerListener { } } - incrementTallies(fromNumber.orElse(FIRST_FROM_NUMBER), platformIncrements, countryIncrements); - + incrementTallies(fromNumber.orElse(UUID.randomUUID()), platformIncrements, countryIncrements); } @Override - public void onCrawlEnd(Optional fromNumber) { + public void onCrawlEnd(Optional fromNumber) { MetricRegistry metrics = new MetricRegistry(); long intervalTallies[] = new long[INTERVALS.length]; ActiveUserTally activeUserTally = getFinalTallies(); @@ -156,17 +154,18 @@ public class ActiveUserCounter implements AccountDatabaseCrawlerListener { return tally; } - private void incrementTallies(String fromNumber, Map platformIncrements, Map countryIncrements) { + private void incrementTallies(UUID fromUuid, Map platformIncrements, Map countryIncrements) { try (Jedis jedis = jedisPool.getWriteResource()) { String tallyValue = jedis.get(TALLY_KEY); ActiveUserTally activeUserTally; if (tallyValue == null) { - activeUserTally = new ActiveUserTally(fromNumber, platformIncrements, countryIncrements); + activeUserTally = new ActiveUserTally(fromUuid, platformIncrements, countryIncrements); } else { activeUserTally = mapper.readValue(tallyValue, ActiveUserTally.class); - if (activeUserTally.getFromNumber() != fromNumber) { - activeUserTally.setFromNumber(fromNumber); + + if (!fromUuid.equals(activeUserTally.getFromUuid())) { + activeUserTally.setFromUuid(fromUuid); Map platformTallies = activeUserTally.getPlatforms(); addTallyMaps(platformTallies, platformIncrements); Map countryTallies = activeUserTally.getCountries(); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryReconciler.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryReconciler.java index e83c0f7e0..e3e5aee3d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryReconciler.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryReconciler.java @@ -30,9 +30,12 @@ import org.whispersystems.textsecuregcm.util.Constants; import org.whispersystems.textsecuregcm.util.Util; import javax.ws.rs.ProcessingException; + +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; import static com.codahale.metrics.MetricRegistry.name; @@ -55,16 +58,16 @@ public class DirectoryReconciler implements AccountDatabaseCrawlerListener { public void onCrawlStart() { } - public void onCrawlEnd(Optional fromNumber) { - DirectoryReconciliationRequest request = new DirectoryReconciliationRequest(fromNumber.orElse(null), null, Collections.emptyList()); + public void onCrawlEnd(Optional fromUuid) { + DirectoryReconciliationRequest request = new DirectoryReconciliationRequest(fromUuid.orElse(null), null, Collections.emptyList()); DirectoryReconciliationResponse response = sendChunk(request); } - public void onCrawlChunk(Optional fromNumber, List chunkAccounts) throws AccountDatabaseCrawlerRestartException { + public void onCrawlChunk(Optional fromUuid, List chunkAccounts) throws AccountDatabaseCrawlerRestartException { updateDirectoryCache(chunkAccounts); - DirectoryReconciliationRequest request = createChunkRequest(fromNumber, chunkAccounts); + DirectoryReconciliationRequest request = createChunkRequest(fromUuid, chunkAccounts); DirectoryReconciliationResponse response = sendChunk(request); if (response.getStatus() == DirectoryReconciliationResponse.Status.MISSING) { throw new AccountDatabaseCrawlerRestartException("directory reconciler missing"); @@ -91,19 +94,21 @@ public class DirectoryReconciler implements AccountDatabaseCrawlerListener { } @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private DirectoryReconciliationRequest createChunkRequest(Optional fromNumber, List accounts) { - List numbers = accounts.stream() - .filter(Account::isEnabled) - .map(Account::getNumber) - .collect(Collectors.toList()); - - Optional toNumber = Optional.empty(); - - if (!accounts.isEmpty()) { - toNumber = Optional.of(accounts.get(accounts.size() - 1).getNumber()); + private DirectoryReconciliationRequest createChunkRequest(Optional fromUuid, List accounts) { + List users = new ArrayList<>(accounts.size()); + for (Account account : accounts) { + if (account.isEnabled()) { + users.add(new DirectoryReconciliationRequest.User(account.getUuid(), account.getNumber())); + } } - return new DirectoryReconciliationRequest(fromNumber.orElse(null), toNumber.orElse(null), numbers); + Optional toUuid = Optional.empty(); + + if (!accounts.isEmpty()) { + toUuid = Optional.of(accounts.get(accounts.size() - 1).getUuid()); + } + + return new DirectoryReconciliationRequest(fromUuid.orElse(null), toUuid.orElse(null), users); } private DirectoryReconciliationResponse sendChunk(DirectoryReconciliationRequest request) { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryReconciliationClient.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryReconciliationClient.java index 441dd4745..e2c085e90 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryReconciliationClient.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/DirectoryReconciliationClient.java @@ -51,7 +51,7 @@ public class DirectoryReconciliationClient { public DirectoryReconciliationResponse sendChunk(DirectoryReconciliationRequest request) { return client.target(replicationUrl) - .path("/v1/directory/reconcile") + .path("/v2/directory/reconcile") .request(MediaType.APPLICATION_JSON_TYPE) .put(Entity.json(request), DirectoryReconciliationResponse.class); } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessor.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessor.java index 1464dff66..fd3d6ceb8 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessor.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/PushFeedbackProcessor.java @@ -9,6 +9,7 @@ import org.whispersystems.textsecuregcm.util.Util; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static com.codahale.metrics.MetricRegistry.name; @@ -31,7 +32,7 @@ public class PushFeedbackProcessor implements AccountDatabaseCrawlerListener { public void onCrawlStart() {} @Override - public void onCrawlChunk(Optional fromNumber, List chunkAccounts) { + public void onCrawlChunk(Optional fromUuid, List chunkAccounts) { for (Account account : chunkAccounts) { boolean update = false; @@ -65,5 +66,5 @@ public class PushFeedbackProcessor implements AccountDatabaseCrawlerListener { } @Override - public void onCrawlEnd(Optional fromNumber) {} + public void onCrawlEnd(Optional toUuid) {} } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java index 8e31a6c19..1ce7ef08a 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java @@ -29,6 +29,7 @@ import org.whispersystems.textsecuregcm.storage.AccountsManager; import java.util.Arrays; import java.util.Collections; import java.util.Optional; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -39,8 +40,8 @@ import static org.mockito.Mockito.*; public class AccountDatabaseCrawlerTest { - private static final String ACCOUNT1 = "+1"; - private static final String ACCOUNT2 = "+2"; + private static final UUID ACCOUNT1 = UUID.randomUUID(); + private static final UUID ACCOUNT2 = UUID.randomUUID(); private static final int CHUNK_SIZE = 1000; private static final long CHUNK_INTERVAL_MS = 30_000L; @@ -56,8 +57,8 @@ public class AccountDatabaseCrawlerTest { @Before public void setup() { - when(account1.getNumber()).thenReturn(ACCOUNT1); - when(account2.getNumber()).thenReturn(ACCOUNT2); + when(account1.getUuid()).thenReturn(ACCOUNT1); + when(account2.getUuid()).thenReturn(ACCOUNT2); when(accounts.getAllFrom(anyInt())).thenReturn(Arrays.asList(account1, account2)); when(accounts.getAllFrom(eq(ACCOUNT1), anyInt())).thenReturn(Arrays.asList(account2)); @@ -69,20 +70,20 @@ public class AccountDatabaseCrawlerTest { @Test public void testCrawlStart() throws AccountDatabaseCrawlerRestartException { - when(cache.getLastNumber()).thenReturn(Optional.empty()); + when(cache.getLastUuid()).thenReturn(Optional.empty()); boolean accelerated = crawler.doPeriodicWork(); assertThat(accelerated).isFalse(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(1)).getLastNumber(); + verify(cache, times(1)).getLastUuid(); verify(listener, times(1)).onCrawlStart(); verify(accounts, times(1)).getAllFrom(eq(CHUNK_SIZE)); - verify(accounts, times(0)).getAllFrom(any(String.class), eq(CHUNK_SIZE)); - verify(account1, times(0)).getNumber(); - verify(account2, times(1)).getNumber(); + verify(accounts, times(0)).getAllFrom(any(UUID.class), eq(CHUNK_SIZE)); + verify(account1, times(0)).getUuid(); + verify(account2, times(1)).getUuid(); verify(listener, times(1)).onCrawlChunk(eq(Optional.empty()), eq(Arrays.asList(account1, account2))); - verify(cache, times(1)).setLastNumber(eq(Optional.of(ACCOUNT2))); + verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); @@ -95,18 +96,18 @@ public class AccountDatabaseCrawlerTest { @Test public void testCrawlChunk() throws AccountDatabaseCrawlerRestartException { - when(cache.getLastNumber()).thenReturn(Optional.of(ACCOUNT1)); + when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); boolean accelerated = crawler.doPeriodicWork(); assertThat(accelerated).isFalse(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(1)).getLastNumber(); + verify(cache, times(1)).getLastUuid(); verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE)); - verify(account2, times(1)).getNumber(); + verify(account2, times(1)).getUuid(); verify(listener, times(1)).onCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); - verify(cache, times(1)).setLastNumber(eq(Optional.of(ACCOUNT2))); + verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); @@ -121,18 +122,18 @@ public class AccountDatabaseCrawlerTest { @Test public void testCrawlChunkAccelerated() throws AccountDatabaseCrawlerRestartException { when(cache.isAccelerated()).thenReturn(true); - when(cache.getLastNumber()).thenReturn(Optional.of(ACCOUNT1)); + when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); boolean accelerated = crawler.doPeriodicWork(); assertThat(accelerated).isTrue(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(1)).getLastNumber(); + verify(cache, times(1)).getLastUuid(); verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE)); - verify(account2, times(1)).getNumber(); + verify(account2, times(1)).getUuid(); verify(listener, times(1)).onCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); - verify(cache, times(1)).setLastNumber(eq(Optional.of(ACCOUNT2))); + verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); @@ -146,19 +147,19 @@ public class AccountDatabaseCrawlerTest { @Test public void testCrawlChunkRestart() throws AccountDatabaseCrawlerRestartException { - when(cache.getLastNumber()).thenReturn(Optional.of(ACCOUNT1)); + when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); doThrow(AccountDatabaseCrawlerRestartException.class).when(listener).onCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); boolean accelerated = crawler.doPeriodicWork(); assertThat(accelerated).isFalse(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(1)).getLastNumber(); + verify(cache, times(1)).getLastUuid(); verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE)); verify(account2, times(0)).getNumber(); verify(listener, times(1)).onCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2))); - verify(cache, times(1)).setLastNumber(eq(Optional.empty())); + verify(cache, times(1)).setLastUuid(eq(Optional.empty())); verify(cache, times(1)).clearAccelerate(); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); @@ -173,19 +174,19 @@ public class AccountDatabaseCrawlerTest { @Test public void testCrawlEnd() { - when(cache.getLastNumber()).thenReturn(Optional.of(ACCOUNT2)); + when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2)); boolean accelerated = crawler.doPeriodicWork(); assertThat(accelerated).isFalse(); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); - verify(cache, times(1)).getLastNumber(); + verify(cache, times(1)).getLastUuid(); verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFrom(eq(ACCOUNT2), eq(CHUNK_SIZE)); verify(account1, times(0)).getNumber(); verify(account2, times(0)).getNumber(); verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2))); - verify(cache, times(1)).setLastNumber(eq(Optional.empty())); + verify(cache, times(1)).setLastUuid(eq(Optional.empty())); verify(cache, times(1)).clearAccelerate(); verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).releaseActiveWork(any(String.class)); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsTest.java index bb1664cf4..c8c893d28 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountsTest.java @@ -173,6 +173,8 @@ public class AccountsTest { accounts.create(account); } + users.sort((account, t1) -> UUIDComparator.staticCompare(account.getUuid(), t1.getUuid())); + List retrieved = accounts.getAllFrom(10); assertThat(retrieved.size()).isEqualTo(10); @@ -181,7 +183,7 @@ public class AccountsTest { } for (int j=0;j<9;j++) { - retrieved = accounts.getAllFrom(retrieved.get(9).getNumber(), 10); + retrieved = accounts.getAllFrom(retrieved.get(9).getUuid(), 10); assertThat(retrieved.size()).isEqualTo(10); for (int i=0;i request = ArgumentCaptor.forClass(DirectoryReconciliationRequest.class); verify(reconciliationClient, times(1)).sendChunk(request.capture()); - assertThat(request.getValue().getFromNumber()).isEqualTo(VALID_NUMBER); - assertThat(request.getValue().getToNumber()).isEqualTo(INACTIVE_NUMBER); - assertThat(request.getValue().getNumbers()).isEqualTo(Arrays.asList(VALID_NUMBER)); + assertThat(request.getValue().getFromUuid()).isEqualTo(VALID_UUID); + assertThat(request.getValue().getToUuid()).isEqualTo(INACTIVE_UUID); + assertThat(request.getValue().getUsers()).isEqualTo(Arrays.asList(new DirectoryReconciliationRequest.User(VALID_UUID, VALID_NUMBERRR))); ArgumentCaptor addedContact = ArgumentCaptor.forClass(ClientContact.class); verify(directoryManager, times(1)).startBatchOperation(); verify(directoryManager, times(1)).add(eq(batchOperationHandle), addedContact.capture()); - verify(directoryManager, times(1)).remove(eq(batchOperationHandle), eq(INACTIVE_NUMBER)); + verify(directoryManager, times(1)).remove(eq(batchOperationHandle), eq(INACTIVE_NUMBERRR)); verify(directoryManager, times(1)).stopBatchOperation(eq(batchOperationHandle)); - assertThat(addedContact.getValue().getToken()).isEqualTo(Util.getContactToken(VALID_NUMBER)); + assertThat(addedContact.getValue().getToken()).isEqualTo(Util.getContactToken(VALID_NUMBERRR)); verifyNoMoreInteractions(activeAccount); verifyNoMoreInteractions(inactiveAccount); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/PushFeedbackProcessorTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/PushFeedbackProcessorTest.java index 87a59d1f0..8a39a7c4a 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/PushFeedbackProcessorTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/PushFeedbackProcessorTest.java @@ -13,6 +13,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.mockito.Mockito.*; @@ -62,7 +63,7 @@ public class PushFeedbackProcessorTest { @Test public void testEmpty() { PushFeedbackProcessor processor = new PushFeedbackProcessor(accountsManager, directoryQueue); - processor.onCrawlChunk(Optional.of("+14152222222"), Collections.emptyList()); + processor.onCrawlChunk(Optional.of(UUID.randomUUID()), Collections.emptyList()); verifyZeroInteractions(accountsManager); verifyZeroInteractions(directoryQueue); @@ -71,7 +72,7 @@ public class PushFeedbackProcessorTest { @Test public void testUpdate() { PushFeedbackProcessor processor = new PushFeedbackProcessor(accountsManager, directoryQueue); - processor.onCrawlChunk(Optional.of("+14153333333"), List.of(uninstalledAccount, mixedAccount, stillActiveAccount, freshAccount, cleanAccount)); + processor.onCrawlChunk(Optional.of(UUID.randomUUID()), List.of(uninstalledAccount, mixedAccount, stillActiveAccount, freshAccount, cleanAccount)); verify(uninstalledDevice).setApnId(isNull()); verify(uninstalledDevice).setGcmId(isNull());