From e320626c6ef6e197cad79151770e5b9672e06a46 Mon Sep 17 00:00:00 2001 From: Chris Eager <79161849+eager-signal@users.noreply.github.com> Date: Thu, 13 May 2021 17:19:34 -0500 Subject: [PATCH] Add report message API --- .../WhisperServerConfiguration.java | 10 +++- .../textsecuregcm/WhisperServerService.java | 15 ++++- .../controllers/MessageController.java | 17 ++++++ .../storage/MessagesManager.java | 17 ++++-- .../storage/ReportMessageDynamoDb.java | 48 +++++++++++++++ .../storage/ReportMessageManager.java | 60 +++++++++++++++++++ .../workers/DeleteUserCommand.java | 19 +++++- .../MessageControllerMetricsTest.java | 2 + .../MessagePersisterIntegrationTest.java | 2 +- .../storage/MessagesManagerTest.java | 46 ++++++++++++++ .../storage/ReportMessageDynamoDbTest.java | 59 ++++++++++++++++++ .../storage/ReportMessageManagerTest.java | 59 ++++++++++++++++++ .../controllers/MessageControllerTest.java | 24 +++++++- .../WebSocketConnectionIntegrationTest.java | 5 +- 14 files changed, 370 insertions(+), 13 deletions(-) create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/ReportMessageDynamoDb.java create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/ReportMessageManager.java create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesManagerTest.java create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/ReportMessageDynamoDbTest.java create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/ReportMessageManagerTest.java diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index 10e3ac913..26aa42b8d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -12,7 +12,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import javax.validation.Valid; -import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import org.whispersystems.textsecuregcm.configuration.AccountDatabaseCrawlerConfiguration; import org.whispersystems.textsecuregcm.configuration.AccountsDatabaseConfiguration; @@ -157,6 +156,11 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private DynamoDbConfiguration pushChallengeDynamoDb; + @Valid + @NotNull + @JsonProperty + private DynamoDbConfiguration reportMessageDynamoDb; + @Valid @NotNull @JsonProperty @@ -442,6 +446,10 @@ public class WhisperServerConfiguration extends Configuration { return pushChallengeDynamoDb; } + public DynamoDbConfiguration getReportMessageDynamoDbConfiguration() { + return reportMessageDynamoDb; + } + public TorExitNodeConfiguration getTorExitNodeConfiguration() { return tor; } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index f6cbee28f..e1e11aa5d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -181,6 +181,8 @@ import org.whispersystems.textsecuregcm.storage.PushFeedbackProcessor; import org.whispersystems.textsecuregcm.storage.RegistrationLockVersionCounter; import org.whispersystems.textsecuregcm.storage.RemoteConfigs; import org.whispersystems.textsecuregcm.storage.RemoteConfigsManager; +import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb; +import org.whispersystems.textsecuregcm.storage.ReportMessageManager; import org.whispersystems.textsecuregcm.storage.ReservedUsernames; import org.whispersystems.textsecuregcm.storage.Usernames; import org.whispersystems.textsecuregcm.storage.UsernamesManager; @@ -329,6 +331,13 @@ public class WhisperServerService extends Application source, Account destinationAccount, Device destinationDevice, diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesManager.java index f1280b5a2..5ff1a226d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesManager.java @@ -34,18 +34,27 @@ public class MessagesManager { private final MessagesDynamoDb messagesDynamoDb; private final MessagesCache messagesCache; private final PushLatencyManager pushLatencyManager; + private final ReportMessageManager reportMessageManager; public MessagesManager( - MessagesDynamoDb messagesDynamoDb, - MessagesCache messagesCache, - PushLatencyManager pushLatencyManager) { + final MessagesDynamoDb messagesDynamoDb, + final MessagesCache messagesCache, + final PushLatencyManager pushLatencyManager, + final ReportMessageManager reportMessageManager) { this.messagesDynamoDb = messagesDynamoDb; this.messagesCache = messagesCache; this.pushLatencyManager = pushLatencyManager; + this.reportMessageManager = reportMessageManager; } public void insert(UUID destinationUuid, long destinationDevice, Envelope message) { - messagesCache.insert(UUID.randomUUID(), destinationUuid, destinationDevice, message); + final UUID messageGuid = UUID.randomUUID(); + + messagesCache.insert(messageGuid, destinationUuid, destinationDevice, message); + + if (message.hasSource() && !destinationUuid.toString().equals(message.getSourceUuid())) { + reportMessageManager.store(message.getSource(), messageGuid); + } } public void insertEphemeral(final UUID destinationUuid, final long destinationDevice, final Envelope message) { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/ReportMessageDynamoDb.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/ReportMessageDynamoDb.java new file mode 100644 index 000000000..97b05947f --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/ReportMessageDynamoDb.java @@ -0,0 +1,48 @@ +package org.whispersystems.textsecuregcm.storage; + +import com.amazonaws.services.dynamodbv2.document.DeleteItemOutcome; +import com.amazonaws.services.dynamodbv2.document.DynamoDB; +import com.amazonaws.services.dynamodbv2.document.Item; +import com.amazonaws.services.dynamodbv2.document.Table; +import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec; +import com.amazonaws.services.dynamodbv2.model.ReturnValue; +import java.time.Duration; +import java.time.Instant; + +public class ReportMessageDynamoDb { + + static final String KEY_HASH = "H"; + static final String ATTR_TTL = "E"; + + static final Duration TIME_TO_LIVE = Duration.ofDays(7); + + private final Table table; + + public ReportMessageDynamoDb(final DynamoDB dynamoDB, final String tableName) { + + this.table = dynamoDB.getTable(tableName); + } + + public void store(byte[] hash) { + + table.putItem(buildItemForHash(hash)); + } + + private Item buildItemForHash(byte[] hash) { + return new Item() + .withBinary(KEY_HASH, hash) + .withLong(ATTR_TTL, Instant.now().plus(TIME_TO_LIVE).getEpochSecond()); + } + + public boolean remove(byte[] hash) { + + final DeleteItemSpec deleteItemSpec = new DeleteItemSpec() + .withPrimaryKey(KEY_HASH, hash) + .withReturnValues(ReturnValue.ALL_OLD); + + final DeleteItemOutcome outcome = table.deleteItem(deleteItemSpec); + + return outcome.getItem() != null; + } + +} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/ReportMessageManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/ReportMessageManager.java new file mode 100644 index 000000000..2c928b393 --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/ReportMessageManager.java @@ -0,0 +1,60 @@ +package org.whispersystems.textsecuregcm.storage; + +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Objects; +import java.util.UUID; +import org.whispersystems.textsecuregcm.util.UUIDUtil; +import org.whispersystems.textsecuregcm.util.Util; + +public class ReportMessageManager { + + @VisibleForTesting + static final String REPORT_COUNTER_NAME = "reported"; + + private final ReportMessageDynamoDb reportMessageDynamoDb; + private final MeterRegistry meterRegistry; + + public ReportMessageManager(ReportMessageDynamoDb reportMessageDynamoDb, final MeterRegistry meterRegistry) { + + this.reportMessageDynamoDb = reportMessageDynamoDb; + this.meterRegistry = meterRegistry; + } + + public void store(String sourceNumber, UUID messageGuid) { + + Objects.requireNonNull(sourceNumber); + + reportMessageDynamoDb.store(hash(messageGuid, sourceNumber)); + } + + public void report(String sourceNumber, UUID messageGuid) { + + final boolean found = reportMessageDynamoDb.remove(hash(messageGuid, sourceNumber)); + + if (found) { + Counter.builder(REPORT_COUNTER_NAME) + .tag("countryCode", Util.getCountryCode(sourceNumber)) + .register(meterRegistry) + .increment(); + } + } + + private byte[] hash(UUID messageGuid, String otherId) { + final MessageDigest sha256; + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + + sha256.update(UUIDUtil.toBytes(messageGuid)); + sha256.update(otherId.getBytes(StandardCharsets.UTF_8)); + + return sha256.digest(); + } +} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java index 7bb1fa159..de73ed21b 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/DeleteUserCommand.java @@ -20,6 +20,7 @@ import io.dropwizard.cli.EnvironmentCommand; import io.dropwizard.jdbi3.JdbiFactory; import io.dropwizard.setup.Environment; import io.lettuce.core.resource.ClientResources; +import io.micrometer.core.instrument.Metrics; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingDeque; @@ -39,20 +40,22 @@ import org.whispersystems.textsecuregcm.securebackup.SecureBackupClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.sqs.DirectoryQueue; import org.whispersystems.textsecuregcm.storage.Account; -import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts; import org.whispersystems.textsecuregcm.storage.Accounts; import org.whispersystems.textsecuregcm.storage.AccountsDynamoDb; import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.AccountsManager.DeletionReason; -import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; import org.whispersystems.textsecuregcm.storage.KeysDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesCache; import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesManager; +import org.whispersystems.textsecuregcm.storage.MigrationDeletedAccounts; +import org.whispersystems.textsecuregcm.storage.MigrationRetryAccounts; import org.whispersystems.textsecuregcm.storage.Profiles; import org.whispersystems.textsecuregcm.storage.ProfilesManager; +import org.whispersystems.textsecuregcm.storage.ReportMessageDynamoDb; +import org.whispersystems.textsecuregcm.storage.ReportMessageManager; import org.whispersystems.textsecuregcm.storage.ReservedUsernames; import org.whispersystems.textsecuregcm.storage.Usernames; import org.whispersystems.textsecuregcm.storage.UsernamesManager; @@ -142,8 +145,16 @@ public class DeleteUserCommand extends EnvironmentCommand assertFalse(reportMessageDynamoDb.remove(hash1)), + () -> assertFalse(reportMessageDynamoDb.remove(hash2)) + ); + + reportMessageDynamoDb.store(hash1); + reportMessageDynamoDb.store(hash2); + + assertAll("both hashes should be found", + () -> assertTrue(reportMessageDynamoDb.remove(hash1)), + () -> assertTrue(reportMessageDynamoDb.remove(hash2)) + ); + + assertAll( "database should be empty", + () -> assertFalse(reportMessageDynamoDb.remove(hash1)), + () -> assertFalse(reportMessageDynamoDb.remove(hash2)) + ); + } + +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/ReportMessageManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/ReportMessageManagerTest.java new file mode 100644 index 000000000..a4863abb3 --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/ReportMessageManagerTest.java @@ -0,0 +1,59 @@ +package org.whispersystems.textsecuregcm.storage; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +class ReportMessageManagerTest { + + private final ReportMessageDynamoDb reportMessageDynamoDb = mock(ReportMessageDynamoDb.class); + private final MeterRegistry meterRegistry = new SimpleMeterRegistry(); + + private final ReportMessageManager reportMessageManager = new ReportMessageManager(reportMessageDynamoDb, meterRegistry); + + @Test + void testStore() { + + final UUID messageGuid = UUID.randomUUID(); + final String number = "+15105551111"; + + assertThrows(NullPointerException.class, () -> reportMessageManager.store(null, messageGuid)); + + reportMessageManager.store(number, messageGuid); + + verify(reportMessageDynamoDb).store(any()); + } + + @Test + void testReport() { + final String sourceNumber = "+15105551111"; + final UUID messageGuid = UUID.randomUUID(); + + when(reportMessageDynamoDb.remove(any())).thenReturn(false); + reportMessageManager.report(sourceNumber, messageGuid); + + assertEquals(0, getCounterTotal(ReportMessageManager.REPORT_COUNTER_NAME)); + + when(reportMessageDynamoDb.remove(any())).thenReturn(true); + reportMessageManager.report(sourceNumber, messageGuid); + + assertEquals(1, getCounterTotal(ReportMessageManager.REPORT_COUNTER_NAME)); + } + + private double getCounterTotal(final String counterName) { + return meterRegistry.find(counterName).counters().stream() + .map(Counter::count) + .reduce(Double::sum) + .orElse(0.0); + } + +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/MessageControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/MessageControllerTest.java index f5726c81b..5dc1af0d5 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/MessageControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/MessageControllerTest.java @@ -95,6 +95,7 @@ import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.storage.Device; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.storage.MessagesManager; +import org.whispersystems.textsecuregcm.storage.ReportMessageManager; import org.whispersystems.textsecuregcm.tests.util.AuthHelper; import org.whispersystems.textsecuregcm.tests.util.RedisClusterHelper; import org.whispersystems.textsecuregcm.util.ua.ClientPlatform; @@ -127,6 +128,7 @@ class MessageControllerTest { private static final ApnFallbackManager apnFallbackManager = mock(ApnFallbackManager.class); private static final DynamicConfigurationManager dynamicConfigurationManager = mock(DynamicConfigurationManager.class); private static final RateLimitChallengeManager rateLimitChallengeManager = mock(RateLimitChallengeManager.class); + private static final ReportMessageManager reportMessageManager = mock(ReportMessageManager.class); private static final FaultTolerantRedisCluster metricsCluster = RedisClusterHelper.buildMockRedisCluster(redisCommands); private static final ScheduledExecutorService receiptExecutor = mock(ScheduledExecutorService.class); @@ -139,7 +141,8 @@ class MessageControllerTest { .addProvider(new RateLimitChallengeExceptionMapper(rateLimitChallengeManager)) .setTestContainerFactory(new GrizzlyWebTestContainerFactory()) .addResource(new MessageController(rateLimiters, messageSender, receiptSender, accountsManager, - messagesManager, unsealedSenderRateLimiter, apnFallbackManager, dynamicConfigurationManager, rateLimitChallengeManager, metricsCluster, receiptExecutor)) + messagesManager, unsealedSenderRateLimiter, apnFallbackManager, dynamicConfigurationManager, + rateLimitChallengeManager, reportMessageManager, metricsCluster, receiptExecutor)) .build(); @BeforeEach @@ -198,6 +201,7 @@ class MessageControllerTest { apnFallbackManager, dynamicConfigurationManager, rateLimitChallengeManager, + reportMessageManager, metricsCluster, receiptExecutor ); @@ -602,4 +606,22 @@ class MessageControllerTest { Arguments.of("fixtures/online_message_false_nested_property.json", false) ); } + + @Test + void testReportMessage() { + + final String senderNumber = "+12125550001"; + final UUID messageGuid = UUID.randomUUID(); + + final Response response = + resources.getJerseyTest() + .target(String.format("/v1/messages/report/%s/%s", senderNumber, messageGuid)) + .request() + .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD)) + .post(null); + + assertThat(response.getStatus(), is(equalTo(202))); + + verify(reportMessageManager).report(senderNumber, messageGuid); + } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnectionIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnectionIntegrationTest.java index 310613f35..925efd585 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnectionIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnectionIntegrationTest.java @@ -47,6 +47,7 @@ import org.whispersystems.textsecuregcm.storage.Device; import org.whispersystems.textsecuregcm.storage.MessagesCache; import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb; import org.whispersystems.textsecuregcm.storage.MessagesManager; +import org.whispersystems.textsecuregcm.storage.ReportMessageManager; import org.whispersystems.textsecuregcm.tests.util.MessagesDynamoDbRule; import org.whispersystems.websocket.WebSocketClient; import org.whispersystems.websocket.messages.WebSocketResponseMessage; @@ -59,6 +60,7 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest private ExecutorService executorService; private MessagesDynamoDb messagesDynamoDb; private MessagesCache messagesCache; + private ReportMessageManager reportMessageManager; private Account account; private Device device; private WebSocketClient webSocketClient; @@ -75,6 +77,7 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest executorService = Executors.newSingleThreadExecutor(); messagesCache = new MessagesCache(getRedisCluster(), getRedisCluster(), executorService); messagesDynamoDb = new MessagesDynamoDb(messagesDynamoDbRule.getDynamoDB(), MessagesDynamoDbRule.TABLE_NAME, Duration.ofDays(7)); + reportMessageManager = mock(ReportMessageManager.class); account = mock(Account.class); device = mock(Device.class); webSocketClient = mock(WebSocketClient.class); @@ -86,7 +89,7 @@ public class WebSocketConnectionIntegrationTest extends AbstractRedisClusterTest webSocketConnection = new WebSocketConnection( mock(ReceiptSender.class), - new MessagesManager(messagesDynamoDb, messagesCache, mock(PushLatencyManager.class)), + new MessagesManager(messagesDynamoDb, messagesCache, mock(PushLatencyManager.class), reportMessageManager), account, device, webSocketClient,