revert to phone number-based account crawler
This commit is contained in:
Jeffrey Griffin
parent
284428a45a
commit
cf78047830
|
|
@ -19,11 +19,10 @@ package org.whispersystems.textsecuregcm.entities;
|
|||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import java.util.Map;
|
||||
import java.util.HashMap;
|
||||
import java.util.UUID;
|
||||
|
||||
public class ActiveUserTally {
|
||||
@JsonProperty
|
||||
private UUID fromUuid;
|
||||
private String fromNumber;
|
||||
|
||||
@JsonProperty
|
||||
private Map<String, long[]> platforms;
|
||||
|
|
@ -33,14 +32,14 @@ public class ActiveUserTally {
|
|||
|
||||
public ActiveUserTally() {}
|
||||
|
||||
public ActiveUserTally(UUID fromUuid, Map<String, long[]> platforms, Map<String, long[]> countries) {
|
||||
this.fromUuid = fromUuid;
|
||||
public ActiveUserTally(String fromNumber, Map<String, long[]> platforms, Map<String, long[]> countries) {
|
||||
this.fromNumber = fromNumber;
|
||||
this.platforms = platforms;
|
||||
this.countries = countries;
|
||||
}
|
||||
|
||||
public UUID getFromUuid() {
|
||||
return this.fromUuid;
|
||||
public String getFromNumber() {
|
||||
return this.fromNumber;
|
||||
}
|
||||
|
||||
public Map<String, long[]> getPlatforms() {
|
||||
|
|
@ -51,8 +50,8 @@ public class ActiveUserTally {
|
|||
return this.countries;
|
||||
}
|
||||
|
||||
public void setFromUuid(UUID fromUuid) {
|
||||
this.fromUuid = fromUuid;
|
||||
public void setFromNumber(String fromNumber) {
|
||||
this.fromNumber = fromNumber;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,18 +19,14 @@ package org.whispersystems.textsecuregcm.entities;
|
|||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
public class DirectoryReconciliationRequest {
|
||||
|
||||
@JsonProperty
|
||||
private UUID fromUuid;
|
||||
private String fromNumber;
|
||||
|
||||
@JsonProperty
|
||||
private UUID toUuid;
|
||||
|
||||
@JsonProperty
|
||||
private List<UUID> uuids;
|
||||
private String toNumber;
|
||||
|
||||
@JsonProperty
|
||||
private List<String> numbers;
|
||||
|
|
@ -38,23 +34,18 @@ public class DirectoryReconciliationRequest {
|
|||
public DirectoryReconciliationRequest() {
|
||||
}
|
||||
|
||||
public DirectoryReconciliationRequest(UUID fromUuid, UUID toUuid, List<UUID> uuids, List<String> numbers) {
|
||||
this.fromUuid = fromUuid;
|
||||
this.toUuid = toUuid;
|
||||
this.uuids = uuids;
|
||||
this.numbers = numbers;
|
||||
public DirectoryReconciliationRequest(String fromNumber, String toNumber, List<String> numbers) {
|
||||
this.fromNumber = fromNumber;
|
||||
this.toNumber = toNumber;
|
||||
this.numbers = numbers;
|
||||
}
|
||||
|
||||
public UUID getFromUuid() {
|
||||
return fromUuid;
|
||||
public String getFromNumber() {
|
||||
return fromNumber;
|
||||
}
|
||||
|
||||
public UUID getToUuid() {
|
||||
return toUuid;
|
||||
}
|
||||
|
||||
public List<UUID> getUuids() {
|
||||
return uuids;
|
||||
public String getToNumber() {
|
||||
return toNumber;
|
||||
}
|
||||
|
||||
public List<String> getNumbers() {
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import org.whispersystems.textsecuregcm.util.Util;
|
|||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
|
@ -52,7 +51,7 @@ public class AccountCleaner implements AccountDatabaseCrawlerListener {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void onCrawlChunk(Optional<UUID> fromUuid, List<Account> chunkAccounts) {
|
||||
public void onCrawlChunk(Optional<String> fromNumber, List<Account> chunkAccounts) {
|
||||
int accountUpdateCount = 0;
|
||||
for (Account account : chunkAccounts) {
|
||||
if (needsExplicitRemoval(account)) {
|
||||
|
|
@ -75,7 +74,7 @@ public class AccountCleaner implements AccountDatabaseCrawlerListener {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void onCrawlEnd(Optional<UUID> fromUuid) {
|
||||
public void onCrawlEnd(Optional<String> fromNumber) {
|
||||
}
|
||||
|
||||
private boolean needsExplicitRemoval(Account account) {
|
||||
|
|
|
|||
|
|
@ -122,26 +122,26 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
|
|||
}
|
||||
|
||||
private void processChunk() {
|
||||
Optional<UUID> fromUuid = cache.getLastUuid();
|
||||
Optional<String> fromNumber = cache.getLastNumber();
|
||||
|
||||
if (!fromUuid.isPresent()) {
|
||||
listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart);
|
||||
if (!fromNumber.isPresent()) {
|
||||
listeners.forEach(listener -> { listener.onCrawlStart(); });
|
||||
}
|
||||
|
||||
List<Account> chunkAccounts = readChunk(fromUuid, chunkSize);
|
||||
List<Account> chunkAccounts = readChunk(fromNumber, chunkSize);
|
||||
|
||||
if (chunkAccounts.isEmpty()) {
|
||||
listeners.forEach(listener -> listener.onCrawlEnd(fromUuid));
|
||||
cache.setLastUuid(Optional.empty());
|
||||
listeners.forEach(listener -> { listener.onCrawlEnd(fromNumber); });
|
||||
cache.setLastNumber(Optional.empty());
|
||||
cache.clearAccelerate();
|
||||
} else {
|
||||
try {
|
||||
for (AccountDatabaseCrawlerListener listener : listeners) {
|
||||
listener.onCrawlChunk(fromUuid, chunkAccounts);
|
||||
listener.onCrawlChunk(fromNumber, chunkAccounts);
|
||||
}
|
||||
cache.setLastUuid(Optional.of(chunkAccounts.get(chunkAccounts.size() - 1).getUuid()));
|
||||
cache.setLastNumber(Optional.of(chunkAccounts.get(chunkAccounts.size() - 1).getNumber()));
|
||||
} catch (AccountDatabaseCrawlerRestartException e) {
|
||||
cache.setLastUuid(Optional.empty());
|
||||
cache.setLastNumber(Optional.empty());
|
||||
cache.clearAccelerate();
|
||||
}
|
||||
|
||||
|
|
@ -149,12 +149,12 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
|
|||
|
||||
}
|
||||
|
||||
private List<Account> readChunk(Optional<UUID> fromUuid, int chunkSize) {
|
||||
private List<Account> readChunk(Optional<String> fromNumber, int chunkSize) {
|
||||
try (Timer.Context timer = readChunkTimer.time()) {
|
||||
List<Account> chunkAccounts;
|
||||
|
||||
if (fromUuid.isPresent()) {
|
||||
chunkAccounts = accounts.getAllFrom(fromUuid.get(), chunkSize);
|
||||
if (fromNumber.isPresent()) {
|
||||
chunkAccounts = accounts.getAllFrom(fromNumber.get(), chunkSize);
|
||||
} else {
|
||||
chunkAccounts = accounts.getAllFrom(chunkSize);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import java.io.IOException;
|
|||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import redis.clients.jedis.Jedis;
|
||||
|
||||
|
|
@ -31,7 +30,7 @@ import redis.clients.jedis.Jedis;
|
|||
public class AccountDatabaseCrawlerCache {
|
||||
|
||||
private static final String ACTIVE_WORKER_KEY = "account_database_crawler_cache_active_worker";
|
||||
private static final String LAST_UUID_KEY = "account_database_crawler_cache_last_uuid";
|
||||
private static final String LAST_NUMBER_KEY = "account_database_crawler_cache_last_number";
|
||||
private static final String ACCELERATE_KEY = "account_database_crawler_cache_accelerate";
|
||||
|
||||
private static final long LAST_NUMBER_TTL_MS = 86400_000L;
|
||||
|
|
@ -68,21 +67,18 @@ public class AccountDatabaseCrawlerCache {
|
|||
luaScript.execute(keys, args);
|
||||
}
|
||||
|
||||
public Optional<UUID> getLastUuid() {
|
||||
public Optional<String> getLastNumber() {
|
||||
try (Jedis jedis = jedisPool.getWriteResource()) {
|
||||
String lastUuidString = jedis.get(LAST_UUID_KEY);
|
||||
|
||||
if (lastUuidString == null) return Optional.empty();
|
||||
else return Optional.of(UUID.fromString(lastUuidString));
|
||||
return Optional.ofNullable(jedis.get(LAST_NUMBER_KEY));
|
||||
}
|
||||
}
|
||||
|
||||
public void setLastUuid(Optional<UUID> lastUuid) {
|
||||
public void setLastNumber(Optional<String> lastNumber) {
|
||||
try (Jedis jedis = jedisPool.getWriteResource()) {
|
||||
if (lastUuid.isPresent()) {
|
||||
jedis.psetex(LAST_UUID_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString());
|
||||
if (lastNumber.isPresent()) {
|
||||
jedis.psetex(LAST_NUMBER_KEY, LAST_NUMBER_TTL_MS, lastNumber.get());
|
||||
} else {
|
||||
jedis.del(LAST_UUID_KEY);
|
||||
jedis.del(LAST_NUMBER_KEY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,11 +18,10 @@ package org.whispersystems.textsecuregcm.storage;
|
|||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
||||
public interface AccountDatabaseCrawlerListener {
|
||||
void onCrawlStart();
|
||||
void onCrawlChunk(Optional<UUID> fromUuid, List<Account> chunkAccounts) throws AccountDatabaseCrawlerRestartException;
|
||||
void onCrawlEnd(Optional<UUID> fromUuid);
|
||||
void onCrawlChunk(Optional<String> fromNumber, List<Account> chunkAccounts) throws AccountDatabaseCrawlerRestartException;
|
||||
void onCrawlEnd(Optional<String> fromNumber);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -110,10 +110,10 @@ public class Accounts {
|
|||
}));
|
||||
}
|
||||
|
||||
public List<Account> getAllFrom(UUID from, int length) {
|
||||
public List<Account> getAllFrom(String from, int length) {
|
||||
return database.with(jdbi -> jdbi.withHandle(handle -> {
|
||||
try (Timer.Context ignored = getAllFromOffsetTimer.time()) {
|
||||
return handle.createQuery("SELECT * FROM accounts WHERE " + UID + " > :from ORDER BY " + UID + " LIMIT :limit")
|
||||
return handle.createQuery("SELECT * FROM accounts WHERE " + NUMBER + " > :from ORDER BY " + NUMBER + " LIMIT :limit")
|
||||
.bind("from", from)
|
||||
.bind("limit", length)
|
||||
.mapTo(Account.class)
|
||||
|
|
@ -125,7 +125,7 @@ public class Accounts {
|
|||
public List<Account> getAllFrom(int length) {
|
||||
return database.with(jdbi -> jdbi.withHandle(handle -> {
|
||||
try (Timer.Context ignored = getAllFromTimer.time()) {
|
||||
return handle.createQuery("SELECT * FROM accounts ORDER BY " + UID + " LIMIT :limit")
|
||||
return handle.createQuery("SELECT * FROM accounts ORDER BY " + NUMBER + " LIMIT :limit")
|
||||
.bind("limit", length)
|
||||
.mapTo(Account.class)
|
||||
.list();
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ public class AccountsManager {
|
|||
return accounts.getAllFrom(length);
|
||||
}
|
||||
|
||||
public List<Account> getAllFrom(UUID uuid, int length) {
|
||||
return accounts.getAllFrom(uuid, length);
|
||||
public List<Account> getAllFrom(String number, int length) {
|
||||
return accounts.getAllFrom(number, length);
|
||||
}
|
||||
|
||||
private void updateDirectory(Account account) {
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import java.util.HashMap;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import io.dropwizard.metrics.MetricsFactory;
|
||||
|
|
@ -44,6 +43,8 @@ public class ActiveUserCounter implements AccountDatabaseCrawlerListener {
|
|||
private static final String PLATFORM_IOS = "ios";
|
||||
private static final String PLATFORM_ANDROID = "android";
|
||||
|
||||
private static final String FIRST_FROM_NUMBER = "+";
|
||||
|
||||
private static final String INTERVALS[] = {"daily", "weekly", "monthly", "quarterly", "yearly"};
|
||||
|
||||
private final MetricsFactory metricsFactory;
|
||||
|
|
@ -63,7 +64,7 @@ public class ActiveUserCounter implements AccountDatabaseCrawlerListener {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void onCrawlChunk(Optional<UUID> fromNumber, List<Account> chunkAccounts) {
|
||||
public void onCrawlChunk(Optional<String> fromNumber, List<Account> chunkAccounts) {
|
||||
long nowDays = TimeUnit.MILLISECONDS.toDays(System.currentTimeMillis());
|
||||
long agoMs[] = {TimeUnit.DAYS.toMillis(nowDays - 1),
|
||||
TimeUnit.DAYS.toMillis(nowDays - 7),
|
||||
|
|
@ -106,11 +107,12 @@ public class ActiveUserCounter implements AccountDatabaseCrawlerListener {
|
|||
}
|
||||
}
|
||||
|
||||
incrementTallies(fromNumber.orElse(UUID.randomUUID()), platformIncrements, countryIncrements);
|
||||
incrementTallies(fromNumber.orElse(FIRST_FROM_NUMBER), platformIncrements, countryIncrements);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCrawlEnd(Optional<UUID> fromNumber) {
|
||||
public void onCrawlEnd(Optional<String> fromNumber) {
|
||||
MetricRegistry metrics = new MetricRegistry();
|
||||
long intervalTallies[] = new long[INTERVALS.length];
|
||||
ActiveUserTally activeUserTally = getFinalTallies();
|
||||
|
|
@ -154,18 +156,17 @@ public class ActiveUserCounter implements AccountDatabaseCrawlerListener {
|
|||
return tally;
|
||||
}
|
||||
|
||||
private void incrementTallies(UUID fromUuid, Map<String, long[]> platformIncrements, Map<String, long[]> countryIncrements) {
|
||||
private void incrementTallies(String fromNumber, Map<String, long[]> platformIncrements, Map<String, long[]> countryIncrements) {
|
||||
try (Jedis jedis = jedisPool.getWriteResource()) {
|
||||
String tallyValue = jedis.get(TALLY_KEY);
|
||||
ActiveUserTally activeUserTally;
|
||||
|
||||
if (tallyValue == null) {
|
||||
activeUserTally = new ActiveUserTally(fromUuid, platformIncrements, countryIncrements);
|
||||
activeUserTally = new ActiveUserTally(fromNumber, platformIncrements, countryIncrements);
|
||||
} else {
|
||||
activeUserTally = mapper.readValue(tallyValue, ActiveUserTally.class);
|
||||
|
||||
if (!fromUuid.equals(activeUserTally.getFromUuid())) {
|
||||
activeUserTally.setFromUuid(fromUuid);
|
||||
if (activeUserTally.getFromNumber() != fromNumber) {
|
||||
activeUserTally.setFromNumber(fromNumber);
|
||||
Map<String, long[]> platformTallies = activeUserTally.getPlatforms();
|
||||
addTallyMaps(platformTallies, platformIncrements);
|
||||
Map<String, long[]> countryTallies = activeUserTally.getCountries();
|
||||
|
|
|
|||
|
|
@ -30,12 +30,9 @@ import org.whispersystems.textsecuregcm.util.Constants;
|
|||
import org.whispersystems.textsecuregcm.util.Util;
|
||||
|
||||
import javax.ws.rs.ProcessingException;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
|
@ -58,16 +55,16 @@ public class DirectoryReconciler implements AccountDatabaseCrawlerListener {
|
|||
|
||||
public void onCrawlStart() { }
|
||||
|
||||
public void onCrawlEnd(Optional<UUID> fromUuid) {
|
||||
DirectoryReconciliationRequest request = new DirectoryReconciliationRequest(fromUuid.orElse(null), null, Collections.emptyList(), Collections.emptyList());
|
||||
public void onCrawlEnd(Optional<String> fromNumber) {
|
||||
DirectoryReconciliationRequest request = new DirectoryReconciliationRequest(fromNumber.orElse(null), null, Collections.emptyList());
|
||||
DirectoryReconciliationResponse response = sendChunk(request);
|
||||
}
|
||||
|
||||
public void onCrawlChunk(Optional<UUID> fromUuid, List<Account> chunkAccounts) throws AccountDatabaseCrawlerRestartException {
|
||||
public void onCrawlChunk(Optional<String> fromNumber, List<Account> chunkAccounts) throws AccountDatabaseCrawlerRestartException {
|
||||
|
||||
updateDirectoryCache(chunkAccounts);
|
||||
|
||||
DirectoryReconciliationRequest request = createChunkRequest(fromUuid, chunkAccounts);
|
||||
DirectoryReconciliationRequest request = createChunkRequest(fromNumber, chunkAccounts);
|
||||
DirectoryReconciliationResponse response = sendChunk(request);
|
||||
if (response.getStatus() == DirectoryReconciliationResponse.Status.MISSING) {
|
||||
throw new AccountDatabaseCrawlerRestartException("directory reconciler missing");
|
||||
|
|
@ -94,23 +91,19 @@ public class DirectoryReconciler implements AccountDatabaseCrawlerListener {
|
|||
}
|
||||
|
||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
||||
private DirectoryReconciliationRequest createChunkRequest(Optional<UUID> fromUuid, List<Account> accounts) {
|
||||
List<UUID> uuids = new ArrayList<>(accounts.size());
|
||||
List<String> numbers = new ArrayList<>(accounts.size());
|
||||
for (Account account : accounts) {
|
||||
if (account.isEnabled()) {
|
||||
uuids.add(account.getUuid());
|
||||
numbers.add(account.getNumber());
|
||||
}
|
||||
}
|
||||
private DirectoryReconciliationRequest createChunkRequest(Optional<String> fromNumber, List<Account> accounts) {
|
||||
List<String> numbers = accounts.stream()
|
||||
.filter(Account::isEnabled)
|
||||
.map(Account::getNumber)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
Optional<UUID> toUuid = Optional.empty();
|
||||
Optional<String> toNumber = Optional.empty();
|
||||
|
||||
if (!accounts.isEmpty()) {
|
||||
toUuid = Optional.of(accounts.get(accounts.size() - 1).getUuid());
|
||||
toNumber = Optional.of(accounts.get(accounts.size() - 1).getNumber());
|
||||
}
|
||||
|
||||
return new DirectoryReconciliationRequest(fromUuid.orElse(null), toUuid.orElse(null), uuids, numbers);
|
||||
return new DirectoryReconciliationRequest(fromNumber.orElse(null), toNumber.orElse(null), numbers);
|
||||
}
|
||||
|
||||
private DirectoryReconciliationResponse sendChunk(DirectoryReconciliationRequest request) {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import org.whispersystems.textsecuregcm.util.Util;
|
|||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
|
|
@ -32,7 +31,7 @@ public class PushFeedbackProcessor implements AccountDatabaseCrawlerListener {
|
|||
public void onCrawlStart() {}
|
||||
|
||||
@Override
|
||||
public void onCrawlChunk(Optional<UUID> fromUuid, List<Account> chunkAccounts) {
|
||||
public void onCrawlChunk(Optional<String> fromNumber, List<Account> chunkAccounts) {
|
||||
for (Account account : chunkAccounts) {
|
||||
boolean update = false;
|
||||
|
||||
|
|
@ -66,5 +65,5 @@ public class PushFeedbackProcessor implements AccountDatabaseCrawlerListener {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void onCrawlEnd(Optional<UUID> toUuid) {}
|
||||
public void onCrawlEnd(Optional<String> fromNumber) {}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
|||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
|
|
@ -40,8 +39,8 @@ import static org.mockito.Mockito.*;
|
|||
|
||||
public class AccountDatabaseCrawlerTest {
|
||||
|
||||
private static final UUID ACCOUNT1 = UUID.randomUUID();
|
||||
private static final UUID ACCOUNT2 = UUID.randomUUID();
|
||||
private static final String ACCOUNT1 = "+1";
|
||||
private static final String ACCOUNT2 = "+2";
|
||||
|
||||
private static final int CHUNK_SIZE = 1000;
|
||||
private static final long CHUNK_INTERVAL_MS = 30_000L;
|
||||
|
|
@ -57,8 +56,8 @@ public class AccountDatabaseCrawlerTest {
|
|||
|
||||
@Before
|
||||
public void setup() {
|
||||
when(account1.getUuid()).thenReturn(ACCOUNT1);
|
||||
when(account2.getUuid()).thenReturn(ACCOUNT2);
|
||||
when(account1.getNumber()).thenReturn(ACCOUNT1);
|
||||
when(account2.getNumber()).thenReturn(ACCOUNT2);
|
||||
|
||||
when(accounts.getAllFrom(anyInt())).thenReturn(Arrays.asList(account1, account2));
|
||||
when(accounts.getAllFrom(eq(ACCOUNT1), anyInt())).thenReturn(Arrays.asList(account2));
|
||||
|
|
@ -70,20 +69,20 @@ public class AccountDatabaseCrawlerTest {
|
|||
|
||||
@Test
|
||||
public void testCrawlStart() throws AccountDatabaseCrawlerRestartException {
|
||||
when(cache.getLastUuid()).thenReturn(Optional.empty());
|
||||
when(cache.getLastNumber()).thenReturn(Optional.empty());
|
||||
|
||||
boolean accelerated = crawler.doPeriodicWork();
|
||||
assertThat(accelerated).isFalse();
|
||||
|
||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||
verify(cache, times(1)).getLastUuid();
|
||||
verify(cache, times(1)).getLastNumber();
|
||||
verify(listener, times(1)).onCrawlStart();
|
||||
verify(accounts, times(1)).getAllFrom(eq(CHUNK_SIZE));
|
||||
verify(accounts, times(0)).getAllFrom(any(UUID.class), eq(CHUNK_SIZE));
|
||||
verify(account1, times(0)).getUuid();
|
||||
verify(account2, times(1)).getUuid();
|
||||
verify(accounts, times(0)).getAllFrom(any(String.class), eq(CHUNK_SIZE));
|
||||
verify(account1, times(0)).getNumber();
|
||||
verify(account2, times(1)).getNumber();
|
||||
verify(listener, times(1)).onCrawlChunk(eq(Optional.empty()), eq(Arrays.asList(account1, account2)));
|
||||
verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
||||
verify(cache, times(1)).setLastNumber(eq(Optional.of(ACCOUNT2)));
|
||||
verify(cache, times(1)).isAccelerated();
|
||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||
|
||||
|
|
@ -96,18 +95,18 @@ public class AccountDatabaseCrawlerTest {
|
|||
|
||||
@Test
|
||||
public void testCrawlChunk() throws AccountDatabaseCrawlerRestartException {
|
||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
||||
when(cache.getLastNumber()).thenReturn(Optional.of(ACCOUNT1));
|
||||
|
||||
boolean accelerated = crawler.doPeriodicWork();
|
||||
assertThat(accelerated).isFalse();
|
||||
|
||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||
verify(cache, times(1)).getLastUuid();
|
||||
verify(cache, times(1)).getLastNumber();
|
||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
||||
verify(account2, times(1)).getUuid();
|
||||
verify(account2, times(1)).getNumber();
|
||||
verify(listener, times(1)).onCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
||||
verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
||||
verify(cache, times(1)).setLastNumber(eq(Optional.of(ACCOUNT2)));
|
||||
verify(cache, times(1)).isAccelerated();
|
||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||
|
||||
|
|
@ -122,18 +121,18 @@ public class AccountDatabaseCrawlerTest {
|
|||
@Test
|
||||
public void testCrawlChunkAccelerated() throws AccountDatabaseCrawlerRestartException {
|
||||
when(cache.isAccelerated()).thenReturn(true);
|
||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
||||
when(cache.getLastNumber()).thenReturn(Optional.of(ACCOUNT1));
|
||||
|
||||
boolean accelerated = crawler.doPeriodicWork();
|
||||
assertThat(accelerated).isTrue();
|
||||
|
||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||
verify(cache, times(1)).getLastUuid();
|
||||
verify(cache, times(1)).getLastNumber();
|
||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
||||
verify(account2, times(1)).getUuid();
|
||||
verify(account2, times(1)).getNumber();
|
||||
verify(listener, times(1)).onCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
||||
verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
|
||||
verify(cache, times(1)).setLastNumber(eq(Optional.of(ACCOUNT2)));
|
||||
verify(cache, times(1)).isAccelerated();
|
||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||
|
||||
|
|
@ -147,19 +146,19 @@ public class AccountDatabaseCrawlerTest {
|
|||
|
||||
@Test
|
||||
public void testCrawlChunkRestart() throws AccountDatabaseCrawlerRestartException {
|
||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
|
||||
when(cache.getLastNumber()).thenReturn(Optional.of(ACCOUNT1));
|
||||
doThrow(AccountDatabaseCrawlerRestartException.class).when(listener).onCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
||||
|
||||
boolean accelerated = crawler.doPeriodicWork();
|
||||
assertThat(accelerated).isFalse();
|
||||
|
||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||
verify(cache, times(1)).getLastUuid();
|
||||
verify(cache, times(1)).getLastNumber();
|
||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT1), eq(CHUNK_SIZE));
|
||||
verify(account2, times(0)).getNumber();
|
||||
verify(listener, times(1)).onCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(Arrays.asList(account2)));
|
||||
verify(cache, times(1)).setLastUuid(eq(Optional.empty()));
|
||||
verify(cache, times(1)).setLastNumber(eq(Optional.empty()));
|
||||
verify(cache, times(1)).clearAccelerate();
|
||||
verify(cache, times(1)).isAccelerated();
|
||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||
|
|
@ -174,19 +173,19 @@ public class AccountDatabaseCrawlerTest {
|
|||
|
||||
@Test
|
||||
public void testCrawlEnd() {
|
||||
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2));
|
||||
when(cache.getLastNumber()).thenReturn(Optional.of(ACCOUNT2));
|
||||
|
||||
boolean accelerated = crawler.doPeriodicWork();
|
||||
assertThat(accelerated).isFalse();
|
||||
|
||||
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
|
||||
verify(cache, times(1)).getLastUuid();
|
||||
verify(cache, times(1)).getLastNumber();
|
||||
verify(accounts, times(0)).getAllFrom(eq(CHUNK_SIZE));
|
||||
verify(accounts, times(1)).getAllFrom(eq(ACCOUNT2), eq(CHUNK_SIZE));
|
||||
verify(account1, times(0)).getNumber();
|
||||
verify(account2, times(0)).getNumber();
|
||||
verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2)));
|
||||
verify(cache, times(1)).setLastUuid(eq(Optional.empty()));
|
||||
verify(cache, times(1)).setLastNumber(eq(Optional.empty()));
|
||||
verify(cache, times(1)).clearAccelerate();
|
||||
verify(cache, times(1)).isAccelerated();
|
||||
verify(cache, times(1)).releaseActiveWork(any(String.class));
|
||||
|
|
|
|||
|
|
@ -173,8 +173,6 @@ public class AccountsTest {
|
|||
accounts.create(account);
|
||||
}
|
||||
|
||||
users.sort((account, t1) -> UUIDComparator.staticCompare(account.getUuid(), t1.getUuid()));
|
||||
|
||||
List<Account> retrieved = accounts.getAllFrom(10);
|
||||
assertThat(retrieved.size()).isEqualTo(10);
|
||||
|
||||
|
|
@ -183,7 +181,7 @@ public class AccountsTest {
|
|||
}
|
||||
|
||||
for (int j=0;j<9;j++) {
|
||||
retrieved = accounts.getAllFrom(retrieved.get(9).getUuid(), 10);
|
||||
retrieved = accounts.getAllFrom(retrieved.get(9).getNumber(), 10);
|
||||
assertThat(retrieved.size()).isEqualTo(10);
|
||||
|
||||
for (int i=0;i<retrieved.size();i++) {
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ import org.junit.Test;
|
|||
import redis.clients.jedis.Jedis;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.Optional;
|
||||
|
||||
|
|
@ -47,13 +46,9 @@ import static org.mockito.Mockito.when;
|
|||
|
||||
public class ActiveUserCounterTest {
|
||||
|
||||
private final UUID UUID_IOS = UUID.randomUUID();
|
||||
private final UUID UUID_ANDROID = UUID.randomUUID();
|
||||
private final UUID UUID_NODEVICE = UUID.randomUUID();
|
||||
|
||||
private final String ACCOUNT_NUMBER_IOS = "+15551234567";
|
||||
private final String ACCOUNT_NUMBER_ANDROID = "+5511987654321";
|
||||
private final String ACCOUNT_NUMBER_NODEVICE = "+5215551234567";
|
||||
private final String NUMBER_IOS = "+15551234567";
|
||||
private final String NUMBER_ANDROID = "+5511987654321";
|
||||
private final String NUMBER_NODEVICE = "+5215551234567";
|
||||
|
||||
private final String TALLY_KEY = "active_user_tally";
|
||||
|
||||
|
|
@ -84,17 +79,14 @@ public class ActiveUserCounterTest {
|
|||
when(iosDevice.getGcmId()).thenReturn(null);
|
||||
when(iosDevice.getLastSeen()).thenReturn(halfDayAgo);
|
||||
|
||||
when(iosAccount.getUuid()).thenReturn(UUID_IOS);
|
||||
when(iosAccount.getNumber()).thenReturn(NUMBER_IOS);
|
||||
when(iosAccount.getMasterDevice()).thenReturn(Optional.of(iosDevice));
|
||||
when(iosAccount.getNumber()).thenReturn(ACCOUNT_NUMBER_IOS);
|
||||
|
||||
when(androidAccount.getUuid()).thenReturn(UUID_ANDROID);
|
||||
when(androidAccount.getNumber()).thenReturn(NUMBER_ANDROID);
|
||||
when(androidAccount.getMasterDevice()).thenReturn(Optional.of(androidDevice));
|
||||
when(androidAccount.getNumber()).thenReturn(ACCOUNT_NUMBER_ANDROID);
|
||||
|
||||
when(noDeviceAccount.getUuid()).thenReturn(UUID_NODEVICE);
|
||||
when(noDeviceAccount.getNumber()).thenReturn(NUMBER_NODEVICE);
|
||||
when(noDeviceAccount.getMasterDevice()).thenReturn(Optional.ofNullable(null));
|
||||
when(noDeviceAccount.getNumber()).thenReturn(ACCOUNT_NUMBER_NODEVICE);
|
||||
|
||||
when(jedis.get(any(String.class))).thenReturn("{\"fromNumber\":\"+\",\"platforms\":{},\"countries\":{}}");
|
||||
when(jedisPool.getWriteResource()).thenReturn(jedis);
|
||||
|
|
@ -145,7 +137,7 @@ public class ActiveUserCounterTest {
|
|||
|
||||
@Test
|
||||
public void testCrawlChunkValidAccount() throws AccountDatabaseCrawlerRestartException {
|
||||
activeUserCounter.onCrawlChunk(Optional.of(UUID_IOS), Arrays.asList(iosAccount));
|
||||
activeUserCounter.onCrawlChunk(Optional.of(NUMBER_IOS), Arrays.asList(iosAccount));
|
||||
|
||||
verify(iosAccount, times(1)).getMasterDevice();
|
||||
verify(iosAccount, times(1)).getNumber();
|
||||
|
|
@ -156,7 +148,7 @@ public class ActiveUserCounterTest {
|
|||
|
||||
verify(jedisPool, times(1)).getWriteResource();
|
||||
verify(jedis, times(1)).get(any(String.class));
|
||||
verify(jedis, times(1)).set(any(String.class), eq("{\"fromUuid\":\""+UUID_IOS.toString()+"\",\"platforms\":{\"ios\":[1,1,1,1,1]},\"countries\":{\"1\":[1,1,1,1,1]}}"));
|
||||
verify(jedis, times(1)).set(any(String.class), eq("{\"fromNumber\":\""+NUMBER_IOS+"\",\"platforms\":{\"ios\":[1,1,1,1,1]},\"countries\":{\"1\":[1,1,1,1,1]}}"));
|
||||
verify(jedis, times(1)).close();
|
||||
|
||||
verify(metricsFactory, times(0)).getReporters();
|
||||
|
|
@ -174,13 +166,13 @@ public class ActiveUserCounterTest {
|
|||
|
||||
@Test
|
||||
public void testCrawlChunkNoDeviceAccount() throws AccountDatabaseCrawlerRestartException {
|
||||
activeUserCounter.onCrawlChunk(Optional.of(UUID_NODEVICE), Arrays.asList(noDeviceAccount));
|
||||
activeUserCounter.onCrawlChunk(Optional.of(NUMBER_NODEVICE), Arrays.asList(noDeviceAccount));
|
||||
|
||||
verify(noDeviceAccount, times(1)).getMasterDevice();
|
||||
|
||||
verify(jedisPool, times(1)).getWriteResource();
|
||||
verify(jedis, times(1)).get(eq(TALLY_KEY));
|
||||
verify(jedis, times(1)).set(any(String.class), eq("{\"fromUuid\":\""+UUID_NODEVICE+"\",\"platforms\":{},\"countries\":{}}"));
|
||||
verify(jedis, times(1)).set(any(String.class), eq("{\"fromNumber\":\""+NUMBER_NODEVICE+"\",\"platforms\":{},\"countries\":{}}"));
|
||||
verify(jedis, times(1)).close();
|
||||
|
||||
verify(metricsFactory, times(0)).getReporters();
|
||||
|
|
@ -198,7 +190,7 @@ public class ActiveUserCounterTest {
|
|||
|
||||
@Test
|
||||
public void testCrawlChunkMixedAccount() throws AccountDatabaseCrawlerRestartException {
|
||||
activeUserCounter.onCrawlChunk(Optional.of(UUID_IOS), Arrays.asList(iosAccount, androidAccount, noDeviceAccount));
|
||||
activeUserCounter.onCrawlChunk(Optional.of(NUMBER_IOS), Arrays.asList(iosAccount, androidAccount, noDeviceAccount));
|
||||
|
||||
verify(iosAccount, times(1)).getMasterDevice();
|
||||
verify(iosAccount, times(1)).getNumber();
|
||||
|
|
@ -216,7 +208,7 @@ public class ActiveUserCounterTest {
|
|||
|
||||
verify(jedisPool, times(1)).getWriteResource();
|
||||
verify(jedis, times(1)).get(eq(TALLY_KEY));
|
||||
verify(jedis, times(1)).set(any(String.class), eq("{\"fromUuid\":\""+UUID_IOS+"\",\"platforms\":{\"android\":[0,0,0,1,1],\"ios\":[1,1,1,1,1]},\"countries\":{\"55\":[0,0,0,1,1],\"1\":[1,1,1,1,1]}}"));
|
||||
verify(jedis, times(1)).set(any(String.class), eq("{\"fromNumber\":\""+NUMBER_IOS+"\",\"platforms\":{\"android\":[0,0,0,1,1],\"ios\":[1,1,1,1,1]},\"countries\":{\"55\":[0,0,0,1,1],\"1\":[1,1,1,1,1]}}"));
|
||||
verify(jedis, times(1)).close();
|
||||
|
||||
verify(metricsFactory, times(0)).getReporters();
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ import org.whispersystems.textsecuregcm.util.Util;
|
|||
|
||||
import java.util.Arrays;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
|
|
@ -41,10 +40,8 @@ import static org.mockito.ArgumentMatchers.eq;
|
|||
import static org.mockito.Mockito.*;
|
||||
|
||||
public class DirectoryReconcilerTest {
|
||||
private static final UUID VALID_UUID = UUID.randomUUID();
|
||||
private static final String VALID_NUMBERRR = "+14152222222";
|
||||
private static final UUID INACTIVE_UUID = UUID.randomUUID();
|
||||
private static final String INACTIVE_NUMBERRR = "+14151111111";
|
||||
private static final String VALID_NUMBER = "valid";
|
||||
private static final String INACTIVE_NUMBER = "inactive";
|
||||
|
||||
private final Account activeAccount = mock(Account.class);
|
||||
private final Account inactiveAccount = mock(Account.class);
|
||||
|
|
@ -58,11 +55,9 @@ public class DirectoryReconcilerTest {
|
|||
|
||||
@Before
|
||||
public void setup() {
|
||||
when(activeAccount.getUuid()).thenReturn(VALID_UUID);
|
||||
when(activeAccount.getNumber()).thenReturn(VALID_NUMBER);
|
||||
when(activeAccount.isEnabled()).thenReturn(true);
|
||||
when(activeAccount.getNumber()).thenReturn(VALID_NUMBERRR);
|
||||
when(inactiveAccount.getUuid()).thenReturn(INACTIVE_UUID);
|
||||
when(inactiveAccount.getNumber()).thenReturn(INACTIVE_NUMBERRR);
|
||||
when(inactiveAccount.getNumber()).thenReturn(INACTIVE_NUMBER);
|
||||
when(inactiveAccount.isEnabled()).thenReturn(false);
|
||||
when(directoryManager.startBatchOperation()).thenReturn(batchOperationHandle);
|
||||
}
|
||||
|
|
@ -70,29 +65,27 @@ public class DirectoryReconcilerTest {
|
|||
@Test
|
||||
public void testCrawlChunkValid() throws AccountDatabaseCrawlerRestartException {
|
||||
when(reconciliationClient.sendChunk(any())).thenReturn(successResponse);
|
||||
directoryReconciler.onCrawlChunk(Optional.of(VALID_UUID), Arrays.asList(activeAccount, inactiveAccount));
|
||||
directoryReconciler.onCrawlChunk(Optional.of(VALID_NUMBER), Arrays.asList(activeAccount, inactiveAccount));
|
||||
|
||||
verify(activeAccount, atLeastOnce()).getUuid();
|
||||
verify(activeAccount, atLeastOnce()).getNumber();
|
||||
verify(activeAccount, atLeastOnce()).isEnabled();
|
||||
verify(inactiveAccount, atLeastOnce()).getUuid();
|
||||
verify(inactiveAccount, atLeastOnce()).getNumber();
|
||||
verify(inactiveAccount, atLeastOnce()).isEnabled();
|
||||
|
||||
ArgumentCaptor<DirectoryReconciliationRequest> request = ArgumentCaptor.forClass(DirectoryReconciliationRequest.class);
|
||||
verify(reconciliationClient, times(1)).sendChunk(request.capture());
|
||||
|
||||
assertThat(request.getValue().getFromUuid()).isEqualTo(VALID_UUID);
|
||||
assertThat(request.getValue().getToUuid()).isEqualTo(INACTIVE_UUID);
|
||||
assertThat(request.getValue().getNumbers()).isEqualTo(Arrays.asList(VALID_NUMBERRR));
|
||||
assertThat(request.getValue().getFromNumber()).isEqualTo(VALID_NUMBER);
|
||||
assertThat(request.getValue().getToNumber()).isEqualTo(INACTIVE_NUMBER);
|
||||
assertThat(request.getValue().getNumbers()).isEqualTo(Arrays.asList(VALID_NUMBER));
|
||||
|
||||
ArgumentCaptor<ClientContact> addedContact = ArgumentCaptor.forClass(ClientContact.class);
|
||||
verify(directoryManager, times(1)).startBatchOperation();
|
||||
verify(directoryManager, times(1)).add(eq(batchOperationHandle), addedContact.capture());
|
||||
verify(directoryManager, times(1)).remove(eq(batchOperationHandle), eq(INACTIVE_NUMBERRR));
|
||||
verify(directoryManager, times(1)).remove(eq(batchOperationHandle), eq(INACTIVE_NUMBER));
|
||||
verify(directoryManager, times(1)).stopBatchOperation(eq(batchOperationHandle));
|
||||
|
||||
assertThat(addedContact.getValue().getToken()).isEqualTo(Util.getContactToken(VALID_NUMBERRR));
|
||||
assertThat(addedContact.getValue().getToken()).isEqualTo(Util.getContactToken(VALID_NUMBER));
|
||||
|
||||
verifyNoMoreInteractions(activeAccount);
|
||||
verifyNoMoreInteractions(inactiveAccount);
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import java.util.Collections;
|
|||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.mockito.Mockito.*;
|
||||
|
|
@ -63,7 +62,7 @@ public class PushFeedbackProcessorTest {
|
|||
@Test
|
||||
public void testEmpty() {
|
||||
PushFeedbackProcessor processor = new PushFeedbackProcessor(accountsManager, directoryQueue);
|
||||
processor.onCrawlChunk(Optional.of(UUID.randomUUID()), Collections.emptyList());
|
||||
processor.onCrawlChunk(Optional.of("+14152222222"), Collections.emptyList());
|
||||
|
||||
verifyZeroInteractions(accountsManager);
|
||||
verifyZeroInteractions(directoryQueue);
|
||||
|
|
@ -72,7 +71,7 @@ public class PushFeedbackProcessorTest {
|
|||
@Test
|
||||
public void testUpdate() {
|
||||
PushFeedbackProcessor processor = new PushFeedbackProcessor(accountsManager, directoryQueue);
|
||||
processor.onCrawlChunk(Optional.of(UUID.randomUUID()), List.of(uninstalledAccount, mixedAccount, stillActiveAccount, freshAccount, cleanAccount));
|
||||
processor.onCrawlChunk(Optional.of("+14153333333"), List.of(uninstalledAccount, mixedAccount, stillActiveAccount, freshAccount, cleanAccount));
|
||||
|
||||
verify(uninstalledDevice).setApnId(isNull());
|
||||
verify(uninstalledDevice).setGcmId(isNull());
|
||||
|
|
|
|||
Loading…
Reference in New Issue