From 87c30d00e8ac593204c3ae983ad51a5cfc686acb Mon Sep 17 00:00:00 2001 From: Jon Chambers Date: Wed, 25 Jun 2025 11:08:14 -0400 Subject: [PATCH] Store compressed envelopes at rest --- .../textsecuregcm/storage/MessagesCacheInsertScript.java | 2 +- .../textsecuregcm/storage/MessagesDynamoDb.java | 2 +- .../storage/MessagesCacheGetItemsScriptTest.java | 4 ++-- .../storage/MessagesCacheInsertScriptTest.java | 8 +++++--- .../storage/MessagesCacheRemoveByGuidScriptTest.java | 6 +++--- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesCacheInsertScript.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesCacheInsertScript.java index 7e302900d..a9ce68015 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesCacheInsertScript.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesCacheInsertScript.java @@ -57,7 +57,7 @@ class MessagesCacheInsertScript { ); final List args = new ArrayList<>(Arrays.asList( - envelope.toByteArray(), // message + EnvelopeUtil.compress(envelope).toByteArray(), // message String.valueOf(envelope.getServerTimestamp()).getBytes(StandardCharsets.UTF_8), // currentTime envelope.getServerGuid().getBytes(StandardCharsets.UTF_8), // guid NEW_MESSAGE_EVENT_BYTES // eventPayload diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesDynamoDb.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesDynamoDb.java index 5a4c02f64..46a7f61f9 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesDynamoDb.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/MessagesDynamoDb.java @@ -105,7 +105,7 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore { .put(KEY_SORT, convertSortKey(message.getServerTimestamp(), messageUuid)) .put(LOCAL_INDEX_MESSAGE_UUID_KEY_SORT, convertLocalIndexMessageUuidSortKey(messageUuid)) .put(KEY_TTL, AttributeValues.fromLong(getTtlForMessage(message))) - .put(KEY_ENVELOPE_BYTES, AttributeValue.builder().b(SdkBytes.fromByteArray(message.toByteArray())).build()); + .put(KEY_ENVELOPE_BYTES, AttributeValue.builder().b(SdkBytes.fromByteArray(EnvelopeUtil.compress(message).toByteArray())).build()); writeItems.add(WriteRequest.builder().putRequest(PutRequest.builder() .item(item.build()) diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheGetItemsScriptTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheGetItemsScriptTest.java index f13b0644f..3520aed2b 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheGetItemsScriptTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheGetItemsScriptTest.java @@ -50,8 +50,8 @@ class MessagesCacheGetItemsScriptTest { assertNotNull(messageAndScores); assertEquals(2, messageAndScores.size()); - final MessageProtos.Envelope resultEnvelope = MessageProtos.Envelope.parseFrom( - messageAndScores.getFirst()); + final MessageProtos.Envelope resultEnvelope = + EnvelopeUtil.expand(MessageProtos.Envelope.parseFrom(messageAndScores.getFirst())); assertEquals(serverGuid, resultEnvelope.getServerGuid()); } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheInsertScriptTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheInsertScriptTest.java index c6bfe2d1f..d21710f05 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheInsertScriptTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheInsertScriptTest.java @@ -43,7 +43,7 @@ class MessagesCacheInsertScriptTest { insertScript.executeAsync(destinationUuid, deviceId, envelope1); - assertEquals(List.of(envelope1), getStoredMessages(destinationUuid, deviceId)); + assertEquals(List.of(EnvelopeUtil.compress(envelope1)), getStoredMessages(destinationUuid, deviceId)); final MessageProtos.Envelope envelope2 = MessageProtos.Envelope.newBuilder() .setServerTimestamp(Instant.now().getEpochSecond()) @@ -52,11 +52,13 @@ class MessagesCacheInsertScriptTest { insertScript.executeAsync(destinationUuid, deviceId, envelope2); - assertEquals(List.of(envelope1, envelope2), getStoredMessages(destinationUuid, deviceId)); + assertEquals(List.of(EnvelopeUtil.compress(envelope1), EnvelopeUtil.compress(envelope2)), + getStoredMessages(destinationUuid, deviceId)); insertScript.executeAsync(destinationUuid, deviceId, envelope1); - assertEquals(List.of(envelope1, envelope2), getStoredMessages(destinationUuid, deviceId), + assertEquals(List.of(EnvelopeUtil.compress(envelope1), EnvelopeUtil.compress(envelope2)), + getStoredMessages(destinationUuid, deviceId), "Messages with same GUID should be deduplicated"); } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheRemoveByGuidScriptTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheRemoveByGuidScriptTest.java index db9c188c3..b3571af9e 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheRemoveByGuidScriptTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/MessagesCacheRemoveByGuidScriptTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.whispersystems.textsecuregcm.entities.MessageProtos; import org.whispersystems.textsecuregcm.redis.RedisClusterExtension; +import org.whispersystems.textsecuregcm.util.UUIDUtil; class MessagesCacheRemoveByGuidScriptTest { @@ -44,9 +45,8 @@ class MessagesCacheRemoveByGuidScriptTest { assertEquals(1, removedMessages.size()); - final MessageProtos.Envelope resultMessage = MessageProtos.Envelope.parseFrom( - removedMessages.getFirst()); + final MessageProtos.Envelope resultMessage = MessageProtos.Envelope.parseFrom(removedMessages.getFirst()); - assertEquals(serverGuid, UUID.fromString(resultMessage.getServerGuid())); + assertEquals(serverGuid, UUIDUtil.fromByteString(resultMessage.getServerGuidBinary())); } }