MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

account crawler: remove `set*Dynamo` methods

This commit is contained in:
Chris Eager 2023-04-25 17:58:39 -05:00 committed by Chris Eager
parent 08333d5989
commit 42a9f1b3e4
3 changed files with 13 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawler.java
View File

@ -166,11 +166,11 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
} }
private Optional<UUID> getLastUuid() { private Optional<UUID> getLastUuid() {
return cache.getLastUuidDynamo(); return cache.getLastUuid();
} }
private void cacheLastUuid(final Optional<UUID> lastUuid) { private void cacheLastUuid(final Optional<UUID> lastUuid) {
cache.setLastUuidDynamo(lastUuid); cache.setLastUuid(lastUuid);
} }
private synchronized void sleepWhileRunning(long delayMs) { private synchronized void sleepWhileRunning(long delayMs) {

23
service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountDatabaseCrawlerCache.java
View File

@ -20,7 +20,6 @@ public class AccountDatabaseCrawlerCache {
public static final String ACCOUNT_CLEANER_PREFIX = "account-cleaner"; public static final String ACCOUNT_CLEANER_PREFIX = "account-cleaner";
private static final String ACTIVE_WORKER_KEY = "account_database_crawler_cache_active_worker"; private static final String ACTIVE_WORKER_KEY = "account_database_crawler_cache_active_worker";
private static final String LAST_UUID_KEY = "account_database_crawler_cache_last_uuid";
private static final String ACCELERATE_KEY = "account_database_crawler_cache_accelerate"; private static final String ACCELERATE_KEY = "account_database_crawler_cache_accelerate";
private static final String LAST_UUID_DYNAMO_KEY = "account_database_crawler_cache_last_uuid_dynamo"; private static final String LAST_UUID_DYNAMO_KEY = "account_database_crawler_cache_last_uuid_dynamo";
@ -62,26 +61,6 @@ public class AccountDatabaseCrawlerCache {
} }
public Optional<UUID> getLastUuid() { public Optional<UUID> getLastUuid() {
final String lastUuidString = cacheCluster.withCluster(
connection -> connection.sync().get(getPrefixedKey(LAST_UUID_KEY)));
if (lastUuidString == null) {
return Optional.empty();
} else {
return Optional.of(UUID.fromString(lastUuidString));
}
}
public void setLastUuid(Optional<UUID> lastUuid) {
if (lastUuid.isPresent()) {
cacheCluster.useCluster(connection -> connection.sync()
.psetex(getPrefixedKey(LAST_UUID_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
} else {
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_KEY)));
}
}
public Optional<UUID> getLastUuidDynamo() {
final String lastUuidString = cacheCluster.withCluster( final String lastUuidString = cacheCluster.withCluster(
connection -> connection.sync().get(getPrefixedKey(LAST_UUID_DYNAMO_KEY))); connection -> connection.sync().get(getPrefixedKey(LAST_UUID_DYNAMO_KEY)));
@ -92,7 +71,7 @@ public class AccountDatabaseCrawlerCache {
} }
} }
public void setLastUuidDynamo(Optional<UUID> lastUuid) { public void setLastUuid(Optional<UUID> lastUuid) {
if (lastUuid.isPresent()) { if (lastUuid.isPresent()) {
cacheCluster.useCluster( cacheCluster.useCluster(
connection -> connection.sync() connection -> connection.sync()

35
service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AccountDatabaseCrawlerTest.java
View File

@ -70,21 +70,18 @@ class AccountDatabaseCrawlerTest {
@Test @Test
void testCrawlStart() throws AccountDatabaseCrawlerRestartException { void testCrawlStart() throws AccountDatabaseCrawlerRestartException {
when(cache.getLastUuid()).thenReturn(Optional.empty()); when(cache.getLastUuid()).thenReturn(Optional.empty());
when(cache.getLastUuidDynamo()).thenReturn(Optional.empty());
boolean accelerated = crawler.doPeriodicWork(); boolean accelerated = crawler.doPeriodicWork();
assertThat(accelerated).isFalse(); assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(0)).getLastUuid(); verify(cache, times(1)).getLastUuid();
verify(cache, times(1)).getLastUuidDynamo();
verify(listener, times(1)).onCrawlStart(); verify(listener, times(1)).onCrawlStart();
verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE)); verify(accounts, times(0)).getAllFromDynamo(any(UUID.class), eq(CHUNK_SIZE));
verify(account1, times(0)).getUuid(); verify(account1, times(0)).getUuid();
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.empty()), eq(List.of(account1, account2))); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.empty()), eq(List.of(account1, account2)));
verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
@ -98,19 +95,16 @@ class AccountDatabaseCrawlerTest {
@Test @Test
void testCrawlChunk() throws AccountDatabaseCrawlerRestartException { void testCrawlChunk() throws AccountDatabaseCrawlerRestartException {
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
boolean accelerated = crawler.doPeriodicWork(); boolean accelerated = crawler.doPeriodicWork();
assertThat(accelerated).isFalse(); assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(0)).getLastUuid(); verify(cache, times(1)).getLastUuid();
verify(cache, times(1)).getLastUuidDynamo();
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2))); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2)));
verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
@ -126,19 +120,16 @@ class AccountDatabaseCrawlerTest {
void testCrawlChunkAccelerated() throws AccountDatabaseCrawlerRestartException { void testCrawlChunkAccelerated() throws AccountDatabaseCrawlerRestartException {
when(cache.isAccelerated()).thenReturn(true); when(cache.isAccelerated()).thenReturn(true);
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
boolean accelerated = crawler.doPeriodicWork(); boolean accelerated = crawler.doPeriodicWork();
assertThat(accelerated).isTrue(); assertThat(accelerated).isTrue();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(0)).getLastUuid(); verify(cache, times(1)).getLastUuid();
verify(cache, times(1)).getLastUuidDynamo();
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2))); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2)));
verify(cache, times(0)).setLastUuid(eq(Optional.of(ACCOUNT2))); verify(cache, times(1)).setLastUuid(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.of(ACCOUNT2)));
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
@ -153,7 +144,6 @@ class AccountDatabaseCrawlerTest {
@Test @Test
void testCrawlChunkRestart() throws AccountDatabaseCrawlerRestartException { void testCrawlChunkRestart() throws AccountDatabaseCrawlerRestartException {
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1)); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT1));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT1));
doThrow(AccountDatabaseCrawlerRestartException.class).when(listener) doThrow(AccountDatabaseCrawlerRestartException.class).when(listener)
.timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2))); .timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2)));
@ -161,14 +151,12 @@ class AccountDatabaseCrawlerTest {
assertThat(accelerated).isFalse(); assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(0)).getLastUuid(); verify(cache, times(1)).getLastUuid();
verify(cache, times(1)).getLastUuidDynamo();
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT1), eq(CHUNK_SIZE));
verify(account2, times(0)).getNumber(); verify(account2, times(0)).getNumber();
verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2))); verify(listener, times(1)).timeAndProcessCrawlChunk(eq(Optional.of(ACCOUNT1)), eq(List.of(account2)));
verify(cache, times(0)).setLastUuid(eq(Optional.empty())); verify(cache, times(1)).setLastUuid(eq(Optional.empty()));
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.empty()));
verify(cache, times(1)).setAccelerated(false); verify(cache, times(1)).setAccelerated(false);
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));
@ -184,21 +172,18 @@ class AccountDatabaseCrawlerTest {
@Test @Test
void testCrawlEnd() { void testCrawlEnd() {
when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2)); when(cache.getLastUuid()).thenReturn(Optional.of(ACCOUNT2));
when(cache.getLastUuidDynamo()).thenReturn(Optional.of(ACCOUNT2));
boolean accelerated = crawler.doPeriodicWork(); boolean accelerated = crawler.doPeriodicWork();
assertThat(accelerated).isFalse(); assertThat(accelerated).isFalse();
verify(cache, times(1)).claimActiveWork(any(String.class), anyLong()); verify(cache, times(1)).claimActiveWork(any(String.class), anyLong());
verify(cache, times(0)).getLastUuid(); verify(cache, times(1)).getLastUuid();
verify(cache, times(1)).getLastUuidDynamo();
verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE)); verify(accounts, times(0)).getAllFromDynamo(eq(CHUNK_SIZE));
verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE)); verify(accounts, times(1)).getAllFromDynamo(eq(ACCOUNT2), eq(CHUNK_SIZE));
verify(account1, times(0)).getNumber(); verify(account1, times(0)).getNumber();
verify(account2, times(0)).getNumber(); verify(account2, times(0)).getNumber();
verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2))); verify(listener, times(1)).onCrawlEnd(eq(Optional.of(ACCOUNT2)));
verify(cache, times(0)).setLastUuid(eq(Optional.empty())); verify(cache, times(1)).setLastUuid(eq(Optional.empty()));
verify(cache, times(1)).setLastUuidDynamo(eq(Optional.empty()));
verify(cache, times(1)).setAccelerated(false); verify(cache, times(1)).setAccelerated(false);
verify(cache, times(1)).isAccelerated(); verify(cache, times(1)).isAccelerated();
verify(cache, times(1)).releaseActiveWork(any(String.class)); verify(cache, times(1)).releaseActiveWork(any(String.class));