From b6ecfc713171bdd80c874951d82472157ce4fa91 Mon Sep 17 00:00:00 2001 From: ravi-signal <99042880+ravi-signal@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:54:57 -0600 Subject: [PATCH] Add archive listing --- service/config/sample.yml | 5 + .../WhisperServerConfiguration.java | 12 +- .../textsecuregcm/WhisperServerService.java | 10 +- .../textsecuregcm/backup/BackupManager.java | 127 +++++++++-- .../backup/BackupMediaEncrypter.java | 1 - .../textsecuregcm/backup/BackupsDb.java | 215 ++++++------------ .../backup/Cdn3RemoteStorageManager.java | 180 ++++++++++++++- .../backup/MediaEncryptionParameters.java | 7 + .../backup/RemoteStorageManager.java | 39 ++++ .../textsecuregcm/backup/UsageInfo.java | 7 + .../Cdn3StorageManagerConfiguration.java | 6 + .../configuration/DynamoDbTables.java | 8 - .../controllers/ArchiveController.java | 78 ++++++- .../textsecuregcm/util/HttpUtils.java | 20 ++ .../backup/BackupManagerTest.java | 121 +++++++--- .../backup/BackupMediaEncrypterTest.java | 20 ++ .../textsecuregcm/backup/BackupsDbTest.java | 46 ++-- .../backup/Cdn3RemoteStorageManagerTest.java | 68 +++++- .../controllers/ArchiveControllerTest.java | 40 ++++ .../storage/DynamoDbExtensionSchema.java | 12 - .../textsecuregcm/util/HttpUtilsTest.java | 34 +++ 21 files changed, 798 insertions(+), 258 deletions(-) create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/backup/UsageInfo.java create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/Cdn3StorageManagerConfiguration.java create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypterTest.java create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/util/HttpUtilsTest.java diff --git a/service/config/sample.yml b/service/config/sample.yml index b51fa3e99..f8f12a6e3 100644 --- a/service/config/sample.yml +++ b/service/config/sample.yml @@ -273,6 +273,11 @@ clientCdn: AAAAAAAAAAAAAAAAAAAA -----END CERTIFICATE----- +cdn3StorageManager: + baseUri: https://storage-manager.example.com + clientId: example + clientSecret: secret://cdn3StorageManager.clientSecret + dogstatsd: environment: dev diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index e42753a64..6c7bca05d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -22,6 +22,7 @@ import org.whispersystems.textsecuregcm.configuration.ArtServiceConfiguration; import org.whispersystems.textsecuregcm.configuration.AwsAttachmentsConfiguration; import org.whispersystems.textsecuregcm.configuration.BadgesConfiguration; import org.whispersystems.textsecuregcm.configuration.BraintreeConfiguration; +import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration; import org.whispersystems.textsecuregcm.configuration.CdnConfiguration; import org.whispersystems.textsecuregcm.configuration.ClientCdnConfiguration; import org.whispersystems.textsecuregcm.configuration.ClientReleaseConfiguration; @@ -108,6 +109,11 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private ClientCdnConfiguration clientCdn; + @NotNull + @Valid + @JsonProperty + private Cdn3StorageManagerConfiguration cdn3StorageManager; + @NotNull @Valid @JsonProperty @@ -421,10 +427,14 @@ public class WhisperServerConfiguration extends Configuration { return cdn; } - public ClientCdnConfiguration getClientCdn() { + public ClientCdnConfiguration getClientCdnConfiguration() { return clientCdn; } + public Cdn3StorageManagerConfiguration getCdn3StorageManagerConfiguration() { + return cdn3StorageManager; + } + public DogstatsdConfiguration getDatadogConfiguration() { return dogstatsd; } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index b7cc75a9c..1ae99f76f 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -665,7 +665,6 @@ public class WhisperServerService extends Application info.mediaUsedSpace() - .filter(usedSpace -> MAX_TOTAL_BACKUP_MEDIA_BYTES - usedSpace >= mediaLength) - .isPresent()); + return backupsDb.getMediaUsage(backupUser) + .thenComposeAsync(info -> { + final boolean canStore = MAX_TOTAL_BACKUP_MEDIA_BYTES - info.usageInfo().bytesUsed() >= mediaLength; + if (canStore || info.lastRecalculationTime().isAfter(clock.instant().minus(MAX_QUOTA_STALENESS))) { + return CompletableFuture.completedFuture(canStore); + } + + // The user is out of quota, and we have not recently recalculated the user's usage. Double check by doing a + // hard recalculation before actually forbidding the user from storing additional media. + final String mediaPrefix = "%s/%s/".formatted(encodeBackupIdForCdn(backupUser), MEDIA_DIRECTORY_NAME); + return this.remoteStorageManager.calculateBytesUsed(mediaPrefix) + .thenCompose(usage -> backupsDb + .setMediaUsage(backupUser, usage) + .thenApply(ignored -> usage.bytesUsed())) + .whenComplete((newUsage, throwable) -> { + boolean usageChanged = throwable == null && !newUsage.equals(info.usageInfo()); + Metrics.counter(USAGE_RECALCULATION_COUNTER_NAME, "usageChanged", String.valueOf(usageChanged)) + .increment(); + }) + .thenApply(usedSpace -> MAX_TOTAL_BACKUP_MEDIA_BYTES - usedSpace >= mediaLength); + }); } public record StorageDescriptor(int cdn, byte[] key) {} + public record StorageDescriptorWithLength(int cdn, byte[] key, long length) {} + /** * Copy an encrypted object to the backup cdn, adding a layer of encryption *

* Implementation notes:

This method guarantees that any object that gets successfully copied to the backup cdn - * will also have an entry for the user in the database.

+ * will also be deducted from the user's quota.

*

- * However, the converse isn't true; there may be entries in the database that have not made it to the cdn. On list, - * these entries are checked against the cdn and removed. + * However, the converse isn't true. It's possible we may charge the user for media they failed to copy. As a result, + * the quota may be over reported and it should be recalculated before taking quota enforcement actions. * * @return A stage that completes successfully with location of the twice-encrypted object on the backup cdn. The * returned CompletionStage can be completed exceptionally with the following exceptions. @@ -221,21 +250,27 @@ public class BackupManager { final MessageBackupUploadDescriptor dst = cdn3BackupCredentialGenerator.generateUpload( encodeBackupIdForCdn(backupUser), - encodeForCdn(destinationMediaId)); + "%s/%s".formatted(MEDIA_DIRECTORY_NAME, encodeForCdn(destinationMediaId))); + final int destinationLength = encryptionParameters.outputSize(sourceLength); + + final URI sourceUri = attachmentReadUri(sourceCdn, sourceKey); return this.backupsDb // Write the ddb updates before actually updating backing storage - .trackMedia(backupUser, destinationMediaId, sourceLength) - - // copy the objects. On a failure, make a best-effort attempt to reverse the ddb transaction. If cleanup fails - // the client may be left with some cleanup to do if they don't eventually upload the media id. - .thenCompose(ignored -> remoteStorageManager - // actually perform the copy - .copy(attachmentReadUri(sourceCdn, sourceKey), sourceLength, encryptionParameters, dst) - // best effort: on failure, untrack the copied media - .exceptionallyCompose(copyError -> backupsDb.untrackMedia(backupUser, destinationMediaId, sourceLength) - .thenCompose(ignoredSuccess -> CompletableFuture.failedFuture(copyError)))) + .trackMedia(backupUser, destinationLength) + // Actually copy the objects. If the copy fails, our estimated quota usage may not be exact + .thenComposeAsync(ignored -> remoteStorageManager.copy(sourceUri, sourceLength, encryptionParameters, dst)) + .exceptionallyCompose(throwable -> { + final Throwable unwrapped = ExceptionUtils.unwrap(throwable); + if (!(unwrapped instanceof SourceObjectNotFoundException) && !(unwrapped instanceof InvalidLengthException)) { + throw ExceptionUtils.wrap(unwrapped); + } + // In cases where we know the copy fails without writing anything, we can try to restore the user's quota + return this.backupsDb.trackMedia(backupUser, -destinationLength).whenComplete((ignored, ignoredEx) -> { + throw ExceptionUtils.wrap(unwrapped); + }); + }) // indicates where the backup was stored .thenApply(ignore -> new StorageDescriptor(dst.cdn(), destinationMediaId)); @@ -268,12 +303,55 @@ public class BackupManager { throw Status.PERMISSION_DENIED .withDescription("credential does not support read auth operation") .asRuntimeException(); - } final String encodedBackupId = encodeBackupIdForCdn(backupUser); return cdn3BackupCredentialGenerator.readHeaders(encodedBackupId); } + + /** + * List of media stored for a particular backup id + * + * @param media A page of media entries + * @param cursor If set, can be passed back to a subsequent list request to resume listing from the previous point + */ + public record ListMediaResult(List media, Optional cursor) {} + + /** + * List the media stored by the backupUser + * + * @param backupUser An already ZK authenticated backup user + * @param cursor A cursor returned by a previous call that can be used to resume listing + * @param limit The maximum number of list results to return + * @return A {@link ListMediaResult} + */ + public CompletionStage list( + final AuthenticatedBackupUser backupUser, + final Optional cursor, + final int limit) { + if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) { + Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); + throw Status.PERMISSION_DENIED + .withDescription("credential does not support list operation") + .asRuntimeException(); + } + final String mediaPrefix = "%s/%s/".formatted(MEDIA_DIRECTORY_NAME, encodeBackupIdForCdn(backupUser)); + return remoteStorageManager.list(mediaPrefix, cursor, limit) + .thenApply(result -> + new ListMediaResult( + result + .objects() + .stream() + .map(entry -> new StorageDescriptorWithLength( + remoteStorageManager.cdnNumber(), + decodeFromCdn(entry.key()), + entry.length() + )) + .toList(), + result.cursor() + )); + } + /** * Authenticate the ZK anonymous backup credential's presentation *

@@ -369,7 +447,8 @@ public class BackupManager { }); } - private static String encodeBackupIdForCdn(final AuthenticatedBackupUser backupUser) { + @VisibleForTesting + static String encodeBackupIdForCdn(final AuthenticatedBackupUser backupUser) { return encodeForCdn(BackupsDb.hashedBackupId(backupUser.backupId())); } @@ -377,4 +456,8 @@ public class BackupManager { return Base64.getUrlEncoder().encodeToString(bytes); } + private static byte[] decodeFromCdn(final String base64) { + return Base64.getUrlDecoder().decode(base64); + } + } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypter.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypter.java index 4f68209c9..38e46d671 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypter.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypter.java @@ -1,6 +1,5 @@ package org.whispersystems.textsecuregcm.backup; -import java.net.http.HttpRequest; import java.nio.ByteBuffer; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java index 287061225..318bed985 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java @@ -4,6 +4,7 @@ import io.grpc.Status; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Clock; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -11,7 +12,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import org.signal.libsignal.protocol.ecc.ECPublicKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,31 +22,24 @@ import org.whispersystems.textsecuregcm.util.Util; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.CancellationReason; import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException; -import software.amazon.awssdk.services.dynamodb.model.Delete; import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; -import software.amazon.awssdk.services.dynamodb.model.Put; -import software.amazon.awssdk.services.dynamodb.model.ReturnValuesOnConditionCheckFailure; -import software.amazon.awssdk.services.dynamodb.model.TransactWriteItem; -import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest; -import software.amazon.awssdk.services.dynamodb.model.TransactionCanceledException; import software.amazon.awssdk.services.dynamodb.model.Update; import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest; /** * Tracks backup metadata in a persistent store. - * + *

* It's assumed that the caller has already validated that the backupUser being operated on has valid credentials and * possesses the appropriate {@link BackupTier} to perform the current operation. */ public class BackupsDb { + private static final Logger logger = LoggerFactory.getLogger(BackupsDb.class); static final int BACKUP_CDN = 3; private final DynamoDbAsyncClient dynamoClient; private final String backupTableName; - private final String backupMediaTableName; private final Clock clock; // The backups table @@ -68,31 +61,25 @@ public class BackupsDb { public static final String ATTR_MEDIA_COUNT = "MC"; // N: The cdn number where the message backup is stored public static final String ATTR_CDN = "CDN"; - - // The stored media table (hashedBackupId, mediaId, cdn, objectLength) - - // B: 15-byte mediaId - public static final String KEY_MEDIA_ID = "M"; - // N: The length of the encrypted media object - public static final String ATTR_LENGTH = "L"; + // N: Time in seconds since epoch of last backup media usage recalculation. This timestamp is updated whenever we + // recalculate the up-to-date bytes used by querying the cdn(s) directly. + public static final String ATTR_MEDIA_USAGE_LAST_RECALCULATION = "MBTS"; public BackupsDb( final DynamoDbAsyncClient dynamoClient, final String backupTableName, - final String backupMediaTableName, final Clock clock) { this.dynamoClient = dynamoClient; this.backupTableName = backupTableName; - this.backupMediaTableName = backupMediaTableName; this.clock = clock; } /** * Set the public key associated with a backupId. * - * @param authenticatedBackupId The backup-id bytes that should be associated with the provided public key + * @param authenticatedBackupId The backup-id bytes that should be associated with the provided public key * @param authenticatedBackupTier The backup tier - * @param publicKey The public key to associate with the backup id + * @param publicKey The public key to associate with the backup id * @return A stage that completes when the public key has been set. If the backup-id already has a set public key that * does not match, the stage will be completed exceptionally with a {@link PublicKeyConflictException} */ @@ -136,103 +123,27 @@ public class BackupsDb { /** - * Add media to the backup media table and update the quota in the backup table - * - * @param backupUser The - * @param mediaId The mediaId to add - * @param mediaLength The length of the media before encryption (the length of the source media) - * @return A stage that completes successfully once the tables are updated. If the media with the provided id has - * previously been tracked with a different length, the stage will complete exceptionally with an - * {@link InvalidLengthException} - */ - CompletableFuture trackMedia( - final AuthenticatedBackupUser backupUser, - final byte[] mediaId, - final int mediaLength) { - final byte[] hashedBackupId = hashedBackupId(backupUser); - return dynamoClient - .transactWriteItems(TransactWriteItemsRequest.builder().transactItems( - - // Add the media to the media table - TransactWriteItem.builder().put(Put.builder() - .tableName(backupMediaTableName) - .returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD) - .item(Map.of( - KEY_BACKUP_ID_HASH, AttributeValues.b(hashedBackupId), - KEY_MEDIA_ID, AttributeValues.b(mediaId), - ATTR_CDN, AttributeValues.n(BACKUP_CDN), - ATTR_LENGTH, AttributeValues.n(mediaLength))) - .conditionExpression("attribute_not_exists(#mediaId)") - .expressionAttributeNames(Map.of("#mediaId", KEY_MEDIA_ID)) - .build()).build(), - - // Update the media quota and TTL - TransactWriteItem.builder().update( - UpdateBuilder.forUser(backupTableName, backupUser) - .setRefreshTimes(clock) - .incrementMediaBytes(mediaLength) - .incrementMediaCount(1) - .transactItemBuilder() - .build()).build()).build()) - .exceptionally(throwable -> { - if (ExceptionUtils.unwrap(throwable) instanceof TransactionCanceledException txCancelled) { - final long oldItemLength = conditionCheckFailed(txCancelled, 0) - .flatMap(item -> Optional.ofNullable(item.get(ATTR_LENGTH))) - .map(attr -> Long.parseLong(attr.n())) - .orElseThrow(() -> ExceptionUtils.wrap(throwable)); - if (oldItemLength != mediaLength) { - throw new CompletionException( - new InvalidLengthException("Previously tried to copy media with a different length. " - + "Provided " + mediaLength + " was " + oldItemLength)); - } - // The client already "paid" for this media, can let them through - return null; - } else { - // rethrow original exception - throw ExceptionUtils.wrap(throwable); - } - }) - .thenRun(Util.NOOP); - } - - /** - * Remove media from backup media table and update the quota in the backup table + * Update the quota in the backup table * * @param backupUser The backup user - * @param mediaId The mediaId to add - * @param mediaLength The length of the media before encryption (the length of the source media) - * @return A stage that completes successfully once the tables are updated + * @param mediaLength The length of the media after encryption. A negative length implies the media is being removed + * @return A stage that completes successfully once the table are updated. */ - CompletableFuture untrackMedia( - final AuthenticatedBackupUser backupUser, - final byte[] mediaId, - final int mediaLength) { - final byte[] hashedBackupId = hashedBackupId(backupUser); - return dynamoClient.transactWriteItems(TransactWriteItemsRequest.builder().transactItems( - TransactWriteItem.builder().delete(Delete.builder() - .tableName(backupMediaTableName) - .returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD) - .key(Map.of( - KEY_BACKUP_ID_HASH, AttributeValues.b(hashedBackupId), - KEY_MEDIA_ID, AttributeValues.b(mediaId) - )) - .conditionExpression("#length = :length") - .expressionAttributeNames(Map.of("#length", ATTR_LENGTH)) - .expressionAttributeValues(Map.of(":length", AttributeValues.n(mediaLength))) - .build()).build(), - - // Don't update TTLs, since we're just cleaning up media - TransactWriteItem.builder().update(UpdateBuilder.forUser(backupTableName, backupUser) - .incrementMediaBytes(-mediaLength) - .incrementMediaCount(-1) - .transactItemBuilder().build()).build()).build()) - .exceptionally(error -> { - logger.warn("failed cleanup after failed copy operation", error); - return null; - }) + CompletableFuture trackMedia(final AuthenticatedBackupUser backupUser, final int mediaLength) { + final Instant now = clock.instant(); + return dynamoClient + .updateItem( + // Update the media quota and TTL + UpdateBuilder.forUser(backupTableName, backupUser) + .setRefreshTimes(now) + .incrementMediaBytes(mediaLength) + .incrementMediaCount(Integer.signum(mediaLength)) + .updateItemBuilder() + .build()) .thenRun(Util.NOOP); } + /** * Update the last update timestamps for the backupId in the presentation * @@ -249,6 +160,7 @@ public class BackupsDb { /** * Track that a backup will be stored for the user + * * @param backupUser an already authorized backup user */ CompletableFuture addMessageBackup(final AuthenticatedBackupUser backupUser) { @@ -276,8 +188,8 @@ public class BackupsDb { return dynamoClient.getItem(GetItemRequest.builder() .tableName(backupTableName) .key(Map.of(KEY_BACKUP_ID_HASH, AttributeValues.b(hashedBackupId(backupUser)))) - .projectionExpression("#cdn,#bytesUsed") - .expressionAttributeNames(Map.of("#cdn", ATTR_CDN, "#bytesUsed", ATTR_MEDIA_BYTES_USED)) + .projectionExpression("#cdn,#mediaBytesUsed") + .expressionAttributeNames(Map.of("#cdn", ATTR_CDN, "#mediaBytesUsed", ATTR_MEDIA_BYTES_USED)) .consistentRead(true) .build()) .thenApply(response -> { @@ -297,6 +209,46 @@ public class BackupsDb { }); } + public record TimestampedUsageInfo(UsageInfo usageInfo, Instant lastRecalculationTime) {} + + CompletableFuture getMediaUsage(final AuthenticatedBackupUser backupUser) { + return dynamoClient.getItem(GetItemRequest.builder() + .tableName(backupTableName) + .key(Map.of(KEY_BACKUP_ID_HASH, AttributeValues.b(hashedBackupId(backupUser)))) + .projectionExpression("#mediaBytesUsed,#mediaCount,#usageRecalc") + .expressionAttributeNames(Map.of( + "#mediaBytesUsed", ATTR_MEDIA_BYTES_USED, + "#mediaCount", ATTR_MEDIA_COUNT, + "#usageRecalc", ATTR_MEDIA_USAGE_LAST_RECALCULATION)) + .consistentRead(true) + .build()) + .thenApply(response -> { + final long mediaUsed = AttributeValues.getLong(response.item(), ATTR_MEDIA_BYTES_USED, 0L); + final long mediaCount = AttributeValues.getLong(response.item(), ATTR_MEDIA_COUNT, 0L); + final long recalcSeconds = AttributeValues.getLong(response.item(), ATTR_MEDIA_USAGE_LAST_RECALCULATION, 0L); + return new TimestampedUsageInfo(new UsageInfo(mediaUsed, mediaCount), Instant.ofEpochSecond(recalcSeconds)); + }); + + + } + + CompletableFuture setMediaUsage(final AuthenticatedBackupUser backupUser, UsageInfo usageInfo) { + return dynamoClient.updateItem( + UpdateBuilder.forUser(backupTableName, backupUser) + .addSetExpression("#mediaBytesUsed = :mediaBytesUsed", + Map.entry("#mediaBytesUsed", ATTR_MEDIA_BYTES_USED), + Map.entry(":mediaBytesUsed", AttributeValues.n(usageInfo.bytesUsed()))) + .addSetExpression("#mediaCount = :mediaCount", + Map.entry("#mediaCount", ATTR_MEDIA_COUNT), + Map.entry(":mediaCount", AttributeValues.n(usageInfo.numObjects()))) + .addSetExpression("#mediaRecalc = :mediaRecalc", + Map.entry("#mediaRecalc", ATTR_MEDIA_USAGE_LAST_RECALCULATION), + Map.entry(":mediaRecalc", AttributeValues.n(clock.instant().getEpochSecond()))) + .updateItemBuilder() + .build()) + .thenRun(Util.NOOP); + } + /** * Build ddb update statements for the backups table @@ -396,19 +348,22 @@ public class BackupsDb { * Set the lastRefresh time as part of the update *

* This always updates lastRefreshTime, and updates lastMediaRefreshTime if the backup user has the appropriate - * tier + * tier. */ UpdateBuilder setRefreshTimes(final Clock clock) { - final long refreshTimeSecs = clock.instant().getEpochSecond(); + return this.setRefreshTimes(clock.instant()); + } + + UpdateBuilder setRefreshTimes(final Instant refreshTime) { addSetExpression("#lastRefreshTime = :lastRefreshTime", Map.entry("#lastRefreshTime", ATTR_LAST_REFRESH), - Map.entry(":lastRefreshTime", AttributeValues.n(refreshTimeSecs))); + Map.entry(":lastRefreshTime", AttributeValues.n(refreshTime.getEpochSecond()))); if (backupTier.compareTo(BackupTier.MEDIA) >= 0) { // update the media time if we have the appropriate tier addSetExpression("#lastMediaRefreshTime = :lastMediaRefreshTime", Map.entry("#lastMediaRefreshTime", ATTR_LAST_MEDIA_REFRESH), - Map.entry(":lastMediaRefreshTime", AttributeValues.n(refreshTimeSecs))); + Map.entry(":lastMediaRefreshTime", AttributeValues.n(refreshTime.getEpochSecond()))); } return this; } @@ -462,28 +417,4 @@ public class BackupsDb { throw new AssertionError(e); } } - - /** - * Check if a DynamoDb error indicates a condition check failed error, and return the value of the item failed to - * update. - * - * @param e The error returned by {@link DynamoDbAsyncClient#transactWriteItems} attempt - * @param itemIndex The index of the item in the transaction that had a condition expression - * @return The remote value of the item that failed to update, or empty if the error was not a condition check failure - */ - private static Optional> conditionCheckFailed(TransactionCanceledException e, - int itemIndex) { - if (!e.hasCancellationReasons()) { - return Optional.empty(); - } - if (e.cancellationReasons().size() < itemIndex + 1) { - return Optional.empty(); - } - final CancellationReason reason = e.cancellationReasons().get(itemIndex); - if (!"ConditionalCheckFailed".equals(reason.code()) || !reason.hasItem()) { - return Optional.empty(); - } - return Optional.of(reason.item()); - } - } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java index fe366d8ec..e274257be 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java @@ -1,41 +1,94 @@ package org.whispersystems.textsecuregcm.backup; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.security.cert.CertificateException; import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Stream; +import javax.annotation.Nullable; +import javax.validation.constraints.NotNull; import javax.ws.rs.core.Response; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Timer; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration; import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; import org.whispersystems.textsecuregcm.configuration.RetryConfiguration; import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient; +import org.whispersystems.textsecuregcm.metrics.MetricsUtil; +import org.whispersystems.textsecuregcm.util.ExceptionUtils; +import org.whispersystems.textsecuregcm.util.HttpUtils; +import org.whispersystems.textsecuregcm.util.SystemMapper; public class Cdn3RemoteStorageManager implements RemoteStorageManager { - private final FaultTolerantHttpClient httpClient; + private static final Logger logger = LoggerFactory.getLogger(Cdn3RemoteStorageManager.class); + + private final FaultTolerantHttpClient cdnHttpClient; + private final FaultTolerantHttpClient storageManagerHttpClient; + private final String storageManagerBaseUrl; + private final String clientId; + private final String clientSecret; + static final String CLIENT_ID_HEADER = "CF-Access-Client-Id"; + static final String CLIENT_SECRET_HEADER = "CF-Access-Client-Secret"; + + private static final String STORAGE_MANAGER_STATUS_COUNTER_NAME = MetricsUtil.name(Cdn3RemoteStorageManager.class, + "storageManagerStatus"); + + private static final String STORAGE_MANAGER_TIMER_NAME = MetricsUtil.name(Cdn3RemoteStorageManager.class, + "storageManager"); + private static final String OPERATION_TAG_NAME = "op"; + private static final String STATUS_TAG_NAME = "status"; public Cdn3RemoteStorageManager( final ScheduledExecutorService retryExecutor, final CircuitBreakerConfiguration circuitBreakerConfiguration, final RetryConfiguration retryConfiguration, - final List caCertificates) throws CertificateException { - this.httpClient = FaultTolerantHttpClient.newBuilder() - .withName("cdn3-remote-storage") + final List cdnCaCertificates, + final Cdn3StorageManagerConfiguration configuration) throws CertificateException { + + // strip trailing "/" for easier URI construction + this.storageManagerBaseUrl = StringUtils.removeEnd(configuration.baseUri(), "/"); + this.clientId = configuration.clientId(); + this.clientSecret = configuration.clientSecret(); + + // Client used to read/write to cdn + this.cdnHttpClient = FaultTolerantHttpClient.newBuilder() + .withName("cdn-client") + .withCircuitBreaker(circuitBreakerConfiguration) + .withExecutor(Executors.newCachedThreadPool()) + .withRetryExecutor(retryExecutor) + .withRetry(retryConfiguration) + .withConnectTimeout(Duration.ofSeconds(10)) + .withVersion(HttpClient.Version.HTTP_2) + .withTrustedServerCertificates(cdnCaCertificates.toArray(new String[0])) + .build(); + + // Client used for calls to storage-manager + // storage-manager has an external CA so uses a different client + this.storageManagerHttpClient = FaultTolerantHttpClient.newBuilder() + .withName("cdn3-storage-manager") .withCircuitBreaker(circuitBreakerConfiguration) .withExecutor(Executors.newCachedThreadPool()) .withRetryExecutor(retryExecutor) .withRetry(retryConfiguration) .withConnectTimeout(Duration.ofSeconds(10)) .withVersion(HttpClient.Version.HTTP_2) - .withTrustedServerCertificates(caCertificates.toArray(new String[0])) .build(); } @@ -55,10 +108,10 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager { throw new IllegalArgumentException("Cdn3RemoteStorageManager can only copy to cdn3"); } + final Timer.Sample sample = Timer.start(); final BackupMediaEncrypter encrypter = new BackupMediaEncrypter(encryptionParameters); - final HttpRequest request = HttpRequest.newBuilder().GET().uri(sourceUri).build(); - return httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofPublisher()).thenCompose(response -> { + return cdnHttpClient.sendAsync(request, HttpResponse.BodyHandlers.ofPublisher()).thenCompose(response -> { if (response.statusCode() == Response.Status.NOT_FOUND.getStatusCode()) { throw new CompletionException(new SourceObjectNotFoundException()); } else if (response.statusCode() != Response.Status.OK.getStatusCode()) { @@ -90,13 +143,122 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager { .POST(encryptedBody) .build(); - return httpClient.sendAsync(put, HttpResponse.BodyHandlers.discarding()); + return cdnHttpClient.sendAsync(put, HttpResponse.BodyHandlers.discarding()); }) .thenAccept(response -> { if (response.statusCode() != Response.Status.CREATED.getStatusCode() && response.statusCode() != Response.Status.OK.getStatusCode()) { throw new CompletionException(new IOException("Failed to copy object: " + response.statusCode())); } - }); + }) + .whenComplete((ignored, ignoredException) -> + sample.stop(Metrics.timer(STORAGE_MANAGER_TIMER_NAME, OPERATION_TAG_NAME, "copy"))); } + + @Override + public CompletionStage list( + final String prefix, + final Optional cursor, + final long limit) { + final Timer.Sample sample = Timer.start(); + + final Map queryParams = new HashMap<>(); + queryParams.put("prefix", prefix); + queryParams.put("limit", Long.toString(limit)); + cursor.ifPresent(s -> queryParams.put("cursor", cursor.get())); + + final HttpRequest request = HttpRequest.newBuilder().GET() + .uri(URI.create("%s/%s/%s".formatted( + storageManagerBaseUrl, + Cdn3BackupCredentialGenerator.CDN_PATH, + HttpUtils.queryParamString(queryParams.entrySet())))) + .header(CLIENT_ID_HEADER, clientId) + .header(CLIENT_SECRET_HEADER, clientSecret) + .build(); + + return this.storageManagerHttpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()) + .thenApply(response -> { + Metrics.counter(STORAGE_MANAGER_STATUS_COUNTER_NAME, + OPERATION_TAG_NAME, "list", + STATUS_TAG_NAME, Integer.toString(response.statusCode())) + .increment(); + try { + return parseListResponse(response, prefix); + } catch (IOException e) { + throw ExceptionUtils.wrap(e); + } + }) + .whenComplete((ignored, ignoredException) -> + sample.stop(Metrics.timer(STORAGE_MANAGER_TIMER_NAME, OPERATION_TAG_NAME, "list"))); + } + + /** + * Serialized list response from storage manager + */ + record Cdn3ListResponse(@NotNull List objects, @Nullable String cursor) { + + record Entry(@NotNull String key, @NotNull long size) {} + } + + private static ListResult parseListResponse(final HttpResponse httpListResponse, final String prefix) + throws IOException { + if (!HttpUtils.isSuccessfulResponse(httpListResponse.statusCode())) { + throw new IOException("Failed to list objects: " + httpListResponse.statusCode()); + } + final Cdn3ListResponse result = SystemMapper.jsonMapper() + .readValue(httpListResponse.body(), Cdn3ListResponse.class); + + final List objects = new ArrayList<>(result.objects.size()); + for (Cdn3ListResponse.Entry entry : result.objects) { + if (!entry.key().startsWith(prefix)) { + logger.error("unexpected listing result from cdn3 - entry {} does not contain requested prefix {}", + entry.key(), prefix); + throw new IOException("prefix listing returned unexpected result"); + } + objects.add(new ListResult.Entry(entry.key().substring(prefix.length()), entry.size())); + } + return new ListResult(objects, Optional.ofNullable(result.cursor)); + } + + + /** + * Serialized usage response from storage manager + */ + record UsageResponse(@NotNull long numObjects, @NotNull long bytesUsed) {} + + @Override + public CompletionStage calculateBytesUsed(final String prefix) { + final Timer.Sample sample = Timer.start(); + final HttpRequest request = HttpRequest.newBuilder().GET() + .uri(URI.create("%s/usage%s".formatted( + storageManagerBaseUrl, + HttpUtils.queryParamString(Map.of("prefix", prefix).entrySet())))) + .header(CLIENT_ID_HEADER, clientId) + .header(CLIENT_SECRET_HEADER, clientSecret) + .build(); + return this.storageManagerHttpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()) + .thenApply(response -> { + Metrics.counter(STORAGE_MANAGER_STATUS_COUNTER_NAME, + OPERATION_TAG_NAME, "usage", + STATUS_TAG_NAME, Integer.toString(response.statusCode())) + .increment(); + try { + return parseUsageResponse(response); + } catch (IOException e) { + throw ExceptionUtils.wrap(e); + } + }) + .whenComplete((ignored, ignoredException) -> + sample.stop(Metrics.timer(STORAGE_MANAGER_TIMER_NAME, OPERATION_TAG_NAME, "usage"))); + } + + private static UsageInfo parseUsageResponse(final HttpResponse httpUsageResponse) throws IOException { + if (!HttpUtils.isSuccessfulResponse(httpUsageResponse.statusCode())) { + throw new IOException("Failed to retrieve usage: " + httpUsageResponse.statusCode()); + } + final UsageResponse response = SystemMapper.jsonMapper().readValue(httpUsageResponse.body(), UsageResponse.class); + return new UsageInfo(response.bytesUsed(), response.numObjects); + } + + } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/MediaEncryptionParameters.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/MediaEncryptionParameters.java index f18efc1fc..6d726bf6a 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/MediaEncryptionParameters.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/MediaEncryptionParameters.java @@ -14,4 +14,11 @@ public record MediaEncryptionParameters( new SecretKeySpec(macKey, "HmacSHA256"), new IvParameterSpec(iv)); } + + public int outputSize(final int inputSize) { + // AES-256 has 16-byte block size, and always adds a block if the plaintext is a multiple of the block size + final int numBlocks = (inputSize + 16) / 16; + // IV + AES-256 encrypted data + HmacSHA256 + return this.iv().getIV().length + (numBlocks * 16) + 32; + } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java index 627ac65d5..c770546b6 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java @@ -1,6 +1,8 @@ package org.whispersystems.textsecuregcm.backup; import java.net.URI; +import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletionStage; /** @@ -35,4 +37,41 @@ public interface RemoteStorageManager { int expectedSourceLength, MediaEncryptionParameters encryptionParameters, MessageBackupUploadDescriptor uploadDescriptor); + + /** + * Result of a {@link #list} operation + * + * @param objects An {@link Entry} for each object returned by the list request + * @param cursor An opaque string that can be used to resume listing from where a previous request left off, empty if + * the request reached the end of the list of matching objects. + */ + record ListResult(List objects, Optional cursor) { + + /** + * An entry representing a remote stored object under a prefix + * + * @param key The name of the object with the prefix removed + * @param length The length of the object in bytes + */ + record Entry(String key, long length) {} + } + + /** + * List objects on the remote cdn. + * + * @param prefix The prefix of the objects to list + * @param cursor The cursor returned by a previous call to list, or empty if starting from the first object with the + * provided prefix + * @param limit The maximum number of items to return in the list + * @return A {@link ListResult} of objects that match the prefix. + */ + CompletionStage list(final String prefix, final Optional cursor, final long limit); + + /** + * Calculate the total number of bytes stored by objects with the provided prefix + * + * @param prefix The prefix of the objects to sum + * @return The number of bytes used + */ + CompletionStage calculateBytesUsed(final String prefix); } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/UsageInfo.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/UsageInfo.java new file mode 100644 index 000000000..4b7406af6 --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/UsageInfo.java @@ -0,0 +1,7 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ +package org.whispersystems.textsecuregcm.backup; + +public record UsageInfo(long bytesUsed, long numObjects) {} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/Cdn3StorageManagerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/Cdn3StorageManagerConfiguration.java new file mode 100644 index 000000000..d57bce812 --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/Cdn3StorageManagerConfiguration.java @@ -0,0 +1,6 @@ +package org.whispersystems.textsecuregcm.configuration; + +public record Cdn3StorageManagerConfiguration( + String baseUri, + String clientId, + String clientSecret) {} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/DynamoDbTables.java b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/DynamoDbTables.java index cb38960c4..5e083be06 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/DynamoDbTables.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/DynamoDbTables.java @@ -49,7 +49,6 @@ public class DynamoDbTables { private final AccountsTableConfiguration accounts; private final Table backups; - private final Table backupMedia; private final Table clientReleases; private final Table deletedAccounts; private final Table deletedAccountsLock; @@ -96,7 +95,6 @@ public class DynamoDbTables { this.accounts = accounts; this.backups = backups; - this.backupMedia = backupMedia; this.clientReleases = clientReleases; this.deletedAccounts = deletedAccounts; this.deletedAccountsLock = deletedAccountsLock; @@ -130,12 +128,6 @@ public class DynamoDbTables { return backups; } - @NotNull - @Valid - public Table getBackupMedia() { - return backupMedia; - } - @NotNull @Valid public Table getClientReleases() { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java index 6ef7a9bc3..33dc0120b 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java @@ -27,6 +27,8 @@ import java.util.Optional; import java.util.concurrent.CompletionStage; import javax.annotation.Nullable; import javax.validation.Valid; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; import javax.validation.constraints.Size; @@ -231,7 +233,10 @@ public class ArchiveController { @Schema(description = "If present, the CDN type where the message backup is stored") int cdn, - @Schema(description = "If present, the directory of your backup data on the cdn.") + @Schema(description = """ + If present, the directory of your backup data on the cdn. The message backup can be found at /backupDir/backupName + and stored media can be found at /backupDir/media/mediaId. + """) String backupDir, @Schema(description = "If present, the name of the most recent message backup on the cdn. The backup is at /backupDir/backupName") @@ -409,9 +414,10 @@ public class ArchiveController { description = """ Copy and re-encrypt media from the attachments cdn into the backup cdn. - The original, already encrypted, attachment will be encrypted with the provided key material before being copied + The original, already encrypted, attachment will be encrypted with the provided key material before being copied. - If the destination media already exists, the copy will be skipped and a 200 will be returned. + A particular destination media id should not be reused with a different source media id or different encryption + parameters. """) @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = CopyMediaResponse.class))) @ApiResponse(responseCode = "400", description = "The provided object length was incorrect") @@ -610,4 +616,70 @@ public class ArchiveController { .thenCompose(backupManager::ttlRefresh) .thenApply(Util.ASYNC_EMPTY_RESPONSE); } + + record StoredMediaObject( + + @Schema(description = "The backup cdn where this media object is stored") + @NotNull + Integer cdn, + + @Schema(description = "The mediaId of the object in URL-safe base64", implementation = String.class) + @JsonSerialize(using = ByteArrayBase64UrlAdapter.Serializing.class) + @JsonDeserialize(using = ByteArrayBase64UrlAdapter.Deserializing.class) + @NotNull + @ExactlySize(15) + byte[] mediaId, + + @Schema(description = "The length of the object in bytes") + @NotNull + Long objectLength) {} + + public record ListResponse( + @Schema(description = "A page of media objects stored for this backup ID") + List storedMediaObjects, + + @Schema(description = "If set, the cursor value to pass to the next list request to continue listing. If absent, all objects have been listed") + String cursor) {} + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Path("/media") + @Operation(summary = "List media objects", + description = """ + Retrieve a list of media objects stored for this backup-id. A client may have previously stored media objects + that are no longer referenced in their current backup. To reclaim storage space used by these orphaned + objects, perform a list operation and remove any unreferenced media objects via DELETE /v1/backups/. + """) + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = ListResponse.class))) + @ApiResponse(responseCode = "400", description = "Invalid cursor or limit") + @ApiResponse(responseCode = "429", description = "Rate limited.") + @ApiResponseZkAuth + public CompletionStage listMedia( + @Auth final Optional account, + + @Parameter(description = BackupAuthCredentialPresentationHeader.DESCRIPTION, schema = @Schema(implementation = String.class)) + @NotNull + @HeaderParam(X_SIGNAL_ZK_AUTH) final BackupAuthCredentialPresentationHeader presentation, + + @Parameter(description = BackupAuthCredentialPresentationSignature.DESCRIPTION, schema = @Schema(implementation = String.class)) + @NotNull + @HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature, + + @Parameter(description = "A cursor returned by a previous call") + @QueryParam("cursor") final Optional cursor, + + @Parameter(description = "The number of entries to return per call") + @QueryParam("limit") final Optional<@Min(1) @Max(10_000) Integer> limit) { + if (account.isPresent()) { + throw new BadRequestException("must not use authenticated connection for anonymous operations"); + } + return backupManager + .authenticateBackupUser(presentation.presentation, signature.signature) + .thenCompose(backupUser -> backupManager.list(backupUser, cursor, limit.orElse(1000))) + .thenApply(result -> new ListResponse( + result.media() + .stream().map(entry -> new StoredMediaObject(entry.cdn(), entry.key(), entry.length())) + .toList(), + result.cursor().orElse(null))); + } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/util/HttpUtils.java b/service/src/main/java/org/whispersystems/textsecuregcm/util/HttpUtils.java index 4f5169fd5..5fcab3a6f 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/util/HttpUtils.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/util/HttpUtils.java @@ -5,6 +5,12 @@ package org.whispersystems.textsecuregcm.util; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; + public final class HttpUtils { private HttpUtils() { @@ -14,4 +20,18 @@ public final class HttpUtils { public static boolean isSuccessfulResponse(final int statusCode) { return statusCode >= 200 && statusCode < 300; } + + public static String queryParamString(final Collection> params) { + final StringBuilder sb = new StringBuilder(); + if (params.isEmpty()) { + return sb.toString(); + } + sb.append("?"); + sb.append(params.stream() + .map(e -> "%s=%s".formatted( + URLEncoder.encode(e.getKey(), StandardCharsets.UTF_8), + URLEncoder.encode(e.getValue(), StandardCharsets.UTF_8))) + .collect(Collectors.joining("&"))); + return sb.toString(); + } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java index 7d2e32135..b419e3897 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java @@ -9,11 +9,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatNoException; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import io.grpc.Status; @@ -36,6 +38,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.EnumSource; import org.signal.libsignal.protocol.ecc.Curve; import org.signal.libsignal.protocol.ecc.ECKeyPair; @@ -55,8 +58,7 @@ public class BackupManagerTest { @RegisterExtension public static final DynamoDbExtension DYNAMO_DB_EXTENSION = new DynamoDbExtension( - DynamoDbExtensionSchema.Tables.BACKUPS, - DynamoDbExtensionSchema.Tables.BACKUP_MEDIA); + DynamoDbExtensionSchema.Tables.BACKUPS); private final TestClock testClock = TestClock.now(); private final BackupAuthTestUtil backupAuthTestUtil = new BackupAuthTestUtil(testClock); @@ -66,15 +68,18 @@ public class BackupManagerTest { private final UUID aci = UUID.randomUUID(); private BackupManager backupManager; + private BackupsDb backupsDb; @BeforeEach public void setup() { reset(tusCredentialGenerator); testClock.unpin(); + this.backupsDb = new BackupsDb( + DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(), + DynamoDbExtensionSchema.Tables.BACKUPS.tableName(), + testClock); this.backupManager = new BackupManager( - new BackupsDb(DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(), - DynamoDbExtensionSchema.Tables.BACKUPS.tableName(), DynamoDbExtensionSchema.Tables.BACKUP_MEDIA.tableName(), - testClock), + backupsDb, backupAuthTestUtil.params, tusCredentialGenerator, remoteStorageManager, @@ -256,25 +261,23 @@ public class BackupManagerTest { .thenReturn(new MessageBackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any())) .thenReturn(CompletableFuture.completedFuture(null)); + final MediaEncryptionParameters encryptionParams = new MediaEncryptionParameters( + TestRandomUtil.nextBytes(32), + TestRandomUtil.nextBytes(32), + TestRandomUtil.nextBytes(16)); final BackupManager.StorageDescriptor copied = backupManager.copyToBackup( - backupUser, 3, "abc", 100, mock(MediaEncryptionParameters.class), - "def".getBytes(StandardCharsets.UTF_8)).join(); + backupUser, 3, "abc", 100, encryptionParams, "def".getBytes(StandardCharsets.UTF_8)).join(); assertThat(copied.cdn()).isEqualTo(3); assertThat(copied.key()).isEqualTo("def".getBytes(StandardCharsets.UTF_8)); final Map backup = getBackupItem(backupUser); final long bytesUsed = AttributeValues.getLong(backup, BackupsDb.ATTR_MEDIA_BYTES_USED, 0L); - assertThat(bytesUsed).isEqualTo(100); + assertThat(bytesUsed).isEqualTo(encryptionParams.outputSize(100)); final long mediaCount = AttributeValues.getLong(backup, BackupsDb.ATTR_MEDIA_COUNT, 0L); assertThat(mediaCount).isEqualTo(1); - - final Map mediaItem = getBackupMediaItem(backupUser, - "def".getBytes(StandardCharsets.UTF_8)); - final long mediaLength = AttributeValues.getLong(mediaItem, BackupsDb.ATTR_LENGTH, 0L); - assertThat(mediaLength).isEqualTo(100L); } @Test @@ -292,12 +295,89 @@ public class BackupManagerTest { mock(MediaEncryptionParameters.class), "def".getBytes(StandardCharsets.UTF_8))); + // usage should be rolled back after a known copy failure final Map backup = getBackupItem(backupUser); assertThat(AttributeValues.getLong(backup, BackupsDb.ATTR_MEDIA_BYTES_USED, -1L)).isEqualTo(0L); assertThat(AttributeValues.getLong(backup, BackupsDb.ATTR_MEDIA_COUNT, -1L)).isEqualTo(0L); + } - final Map media = getBackupMediaItem(backupUser, "def".getBytes(StandardCharsets.UTF_8)); - assertThat(media).isEmpty(); + @Test + public void quotaEnforcementNoRecalculation() { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + verifyNoInteractions(remoteStorageManager); + + // set the backupsDb to be out of quota at t=0 + testClock.pin(Instant.ofEpochSecond(1)); + backupsDb.setMediaUsage(backupUser, new UsageInfo(BackupManager.MAX_TOTAL_BACKUP_MEDIA_BYTES, 1000)).join(); + // check still within staleness bound (t=0 + 1 day - 1 sec) + testClock.pin(Instant.ofEpochSecond(0) + .plus(BackupManager.MAX_QUOTA_STALENESS) + .minus(Duration.ofSeconds(1))); + assertThat(backupManager.canStoreMedia(backupUser, 10).join()).isFalse(); + } + + @Test + public void quotaEnforcementRecalculation() { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + final String backupMediaPrefix = "%s/%s/".formatted( + BackupManager.encodeBackupIdForCdn(backupUser), + BackupManager.MEDIA_DIRECTORY_NAME); + + // on recalculation, say there's actually 10 bytes left + when(remoteStorageManager.calculateBytesUsed(eq(backupMediaPrefix))) + .thenReturn( + CompletableFuture.completedFuture(new UsageInfo(BackupManager.MAX_TOTAL_BACKUP_MEDIA_BYTES - 10, 1000))); + + // set the backupsDb to be out of quota at t=0 + testClock.pin(Instant.ofEpochSecond(0)); + backupsDb.setMediaUsage(backupUser, new UsageInfo(BackupManager.MAX_TOTAL_BACKUP_MEDIA_BYTES, 1000)).join(); + testClock.pin(Instant.ofEpochSecond(0).plus(BackupManager.MAX_QUOTA_STALENESS)); + assertThat(backupManager.canStoreMedia(backupUser, 10).join()).isTrue(); + + // backupsDb should have the new value + final BackupsDb.TimestampedUsageInfo info = backupsDb.getMediaUsage(backupUser).join(); + assertThat(info.lastRecalculationTime()).isEqualTo( + Instant.ofEpochSecond(0).plus(BackupManager.MAX_QUOTA_STALENESS)); + assertThat(info.usageInfo().bytesUsed()).isEqualTo(BackupManager.MAX_TOTAL_BACKUP_MEDIA_BYTES - 10); + } + + @ParameterizedTest + @CsvSource({ + "true, 10, 10, true", + "true, 10, 11, false", + "true, 0, 1, false", + "true, 0, 0, true", + "false, 10, 10, true", + "false, 10, 11, false", + "false, 0, 1, false", + "false, 0, 0, true", + }) + public void quotaEnforcement( + boolean recalculation, + final long spaceLeft, + final long mediaToAddSize, + boolean shouldAccept) { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + final String backupMediaPrefix = "%s/%s/".formatted( + BackupManager.encodeBackupIdForCdn(backupUser), + BackupManager.MEDIA_DIRECTORY_NAME); + + // set the backupsDb to be out of quota at t=0 + testClock.pin(Instant.ofEpochSecond(0)); + backupsDb.setMediaUsage(backupUser, new UsageInfo(BackupManager.MAX_TOTAL_BACKUP_MEDIA_BYTES - spaceLeft, 1000)) + .join(); + + if (recalculation) { + testClock.pin(Instant.ofEpochSecond(0).plus(BackupManager.MAX_QUOTA_STALENESS).plus(Duration.ofSeconds(1))); + when(remoteStorageManager.calculateBytesUsed(eq(backupMediaPrefix))) + .thenReturn(CompletableFuture.completedFuture( + new UsageInfo(BackupManager.MAX_TOTAL_BACKUP_MEDIA_BYTES - spaceLeft, 1000))); + } + assertThat(backupManager.canStoreMedia(backupUser, mediaToAddSize).join()).isEqualTo(shouldAccept); + if (recalculation && !shouldAccept) { + // should have recalculated if we exceeded quota + verify(remoteStorageManager, times(1)).calculateBytesUsed(anyString()); + } } private Map getBackupItem(final AuthenticatedBackupUser backupUser) { @@ -308,17 +388,6 @@ public class BackupManagerTest { .item(); } - private Map getBackupMediaItem(final AuthenticatedBackupUser backupUser, - final byte[] mediaId) { - return DYNAMO_DB_EXTENSION.getDynamoDbClient().getItem(GetItemRequest.builder() - .tableName(DynamoDbExtensionSchema.Tables.BACKUP_MEDIA.tableName()) - .key(Map.of( - BackupsDb.KEY_BACKUP_ID_HASH, AttributeValues.b(hashedBackupId(backupUser.backupId())), - BackupsDb.KEY_MEDIA_ID, AttributeValues.b(mediaId))) - .build()) - .item(); - } - private void checkExpectedExpirations( final Instant expectedExpiration, final @Nullable Instant expectedMediaExpiration, diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypterTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypterTest.java new file mode 100644 index 000000000..19ce440c7 --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypterTest.java @@ -0,0 +1,20 @@ +package org.whispersystems.textsecuregcm.backup; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.whispersystems.textsecuregcm.util.TestRandomUtil; + +public class BackupMediaEncrypterTest { + + @ParameterizedTest + @ValueSource(ints = {0, 1, 2, 15, 16, 17, 63, 64, 65, 1023, 1024, 1025}) + public void sizeCalc() { + final MediaEncryptionParameters params = new MediaEncryptionParameters( + TestRandomUtil.nextBytes(32), + TestRandomUtil.nextBytes(32), TestRandomUtil.nextBytes(16)); + final BackupMediaEncrypter encrypter = new BackupMediaEncrypter(params); + assertThat(params.outputSize(1)).isEqualTo(encrypter.outputSize(1)); + } +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java index a72bfb4bb..0b7c82d9a 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java @@ -12,10 +12,13 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.Instant; import java.util.Arrays; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser; import org.whispersystems.textsecuregcm.storage.DynamoDbExtension; import org.whispersystems.textsecuregcm.storage.DynamoDbExtensionSchema; @@ -27,8 +30,7 @@ public class BackupsDbTest { @RegisterExtension public static final DynamoDbExtension DYNAMO_DB_EXTENSION = new DynamoDbExtension( - DynamoDbExtensionSchema.Tables.BACKUPS, - DynamoDbExtensionSchema.Tables.BACKUP_MEDIA); + DynamoDbExtensionSchema.Tables.BACKUPS); private final TestClock testClock = TestClock.now(); private BackupsDb backupsDb; @@ -37,26 +39,10 @@ public class BackupsDbTest { public void setup() { testClock.unpin(); backupsDb = new BackupsDb(DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(), - DynamoDbExtensionSchema.Tables.BACKUPS.tableName(), DynamoDbExtensionSchema.Tables.BACKUP_MEDIA.tableName(), + DynamoDbExtensionSchema.Tables.BACKUPS.tableName(), testClock); } - @Test - public void trackMediaIdempotent() { - final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); - this.backupsDb.trackMedia(backupUser, "abc".getBytes(StandardCharsets.UTF_8), 100).join(); - assertDoesNotThrow(() -> - this.backupsDb.trackMedia(backupUser, "abc".getBytes(StandardCharsets.UTF_8), 100).join()); - } - - @Test - public void trackMediaLengthChange() { - final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); - this.backupsDb.trackMedia(backupUser, "abc".getBytes(StandardCharsets.UTF_8), 100).join(); - CompletableFutureTestUtil.assertFailsWithCause(InvalidLengthException.class, - this.backupsDb.trackMedia(backupUser, "abc".getBytes(StandardCharsets.UTF_8), 99)); - } - @Test public void trackMediaStats() { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); @@ -64,27 +50,33 @@ public class BackupsDbTest { backupsDb.addMessageBackup(backupUser).join(); int total = 0; for (int i = 0; i < 5; i++) { - this.backupsDb.trackMedia(backupUser, Integer.toString(i).getBytes(StandardCharsets.UTF_8), i).join(); + this.backupsDb.trackMedia(backupUser, i).join(); total += i; final BackupsDb.BackupDescription description = this.backupsDb.describeBackup(backupUser).join(); assertThat(description.mediaUsedSpace().get()).isEqualTo(total); } for (int i = 0; i < 5; i++) { - this.backupsDb.untrackMedia(backupUser, Integer.toString(i).getBytes(StandardCharsets.UTF_8), i).join(); + this.backupsDb.trackMedia(backupUser, -i).join(); total -= i; final BackupsDb.BackupDescription description = this.backupsDb.describeBackup(backupUser).join(); assertThat(description.mediaUsedSpace().get()).isEqualTo(total); } } - - private static byte[] hashedBackupId(final byte[] backupId) { - try { - return Arrays.copyOf(MessageDigest.getInstance("SHA-256").digest(backupId), 16); - } catch (NoSuchAlgorithmException e) { - throw new AssertionError(e); + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void setUsage(boolean mediaAlreadyExists) { + testClock.pin(Instant.ofEpochSecond(5)); + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + if (mediaAlreadyExists) { + this.backupsDb.trackMedia(backupUser, 10).join(); } + backupsDb.setMediaUsage(backupUser, new UsageInfo( 113, 17)).join(); + final BackupsDb.TimestampedUsageInfo info = backupsDb.getMediaUsage(backupUser).join(); + assertThat(info.lastRecalculationTime()).isEqualTo(Instant.ofEpochSecond(5)); + assertThat(info.usageInfo().bytesUsed()).isEqualTo(113L); + assertThat(info.usageInfo().numObjects()).isEqualTo(17L); } private AuthenticatedBackupUser backupUser(final byte[] backupId, final BackupTier backupTier) { diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java index c8776f824..396e7b6fc 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java @@ -1,16 +1,20 @@ package org.whispersystems.textsecuregcm.backup; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import com.fasterxml.jackson.core.JsonProcessingException; import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.InvalidAlgorithmParameterException; @@ -19,6 +23,8 @@ import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.concurrent.Executors; import javax.crypto.BadPaddingException; import javax.crypto.Cipher; @@ -33,9 +39,11 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration; import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; import org.whispersystems.textsecuregcm.configuration.RetryConfiguration; import org.whispersystems.textsecuregcm.util.CompletableFutureTestUtil; +import org.whispersystems.textsecuregcm.util.SystemMapper; import org.whispersystems.textsecuregcm.util.TestRandomUtil; @ExtendWith(DropwizardExtensionsSupport.class) @@ -62,7 +70,8 @@ public class Cdn3RemoteStorageManagerTest { Executors.newSingleThreadScheduledExecutor(), new CircuitBreakerConfiguration(), new RetryConfiguration(), - Collections.emptyList()); + Collections.emptyList(), + new Cdn3StorageManagerConfiguration(wireMock.url("storage-manager/"), "clientId", "clientSecret")); wireMock.stubFor(get(urlEqualTo("/cdn2/source/small")) .willReturn(aResponse() @@ -125,7 +134,9 @@ public class Cdn3RemoteStorageManagerTest { .toCompletableFuture().join(); final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).get(0).getBody(); - assertThat(destBody.length).isEqualTo(new BackupMediaEncrypter(params).outputSize(LARGE.length())); + assertThat(destBody.length) + .isEqualTo(new BackupMediaEncrypter(params).outputSize(LARGE.length())) + .isEqualTo(params.outputSize(LARGE.length())); assertThat(new String(decrypt(destBody), StandardCharsets.UTF_8)).isEqualTo(LARGE); } @@ -176,4 +187,57 @@ public class Cdn3RemoteStorageManagerTest { cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(AES_KEY, "AES"), new IvParameterSpec(IV)); return cipher.doFinal(encrypted, IV.length, encrypted.length - IV.length - mac.getMacLength()); } + + @Test + public void list() throws JsonProcessingException { + wireMock.stubFor(get(urlPathEqualTo("/storage-manager/backups/")) + .withQueryParam("prefix", equalTo("abc/")) + .withQueryParam("limit", equalTo("3")) + .withHeader(Cdn3RemoteStorageManager.CLIENT_ID_HEADER, equalTo("clientId")) + .withHeader(Cdn3RemoteStorageManager.CLIENT_SECRET_HEADER, equalTo("clientSecret")) + .willReturn(aResponse() + .withBody(SystemMapper.jsonMapper().writeValueAsString(new Cdn3RemoteStorageManager.Cdn3ListResponse( + List.of( + new Cdn3RemoteStorageManager.Cdn3ListResponse.Entry("abc/x/y", 3), + new Cdn3RemoteStorageManager.Cdn3ListResponse.Entry("abc/y", 4), + new Cdn3RemoteStorageManager.Cdn3ListResponse.Entry("abc/z", 5) + ), "cursor"))))); + final RemoteStorageManager.ListResult result = remoteStorageManager + .list("abc/", Optional.empty(), 3) + .toCompletableFuture().join(); + assertThat(result.cursor()).get().isEqualTo("cursor"); + assertThat(result.objects()).hasSize(3); + + // should strip the common prefix + assertThat(result.objects()).isEqualTo(List.of( + new RemoteStorageManager.ListResult.Entry("x/y", 3), + new RemoteStorageManager.ListResult.Entry("y", 4), + new RemoteStorageManager.ListResult.Entry("z", 5))); + } + + @Test + public void prefixMissing() throws JsonProcessingException { + wireMock.stubFor(get(urlPathEqualTo("/storage-manager/backups/")) + .willReturn(aResponse() + .withBody(SystemMapper.jsonMapper().writeValueAsString(new Cdn3RemoteStorageManager.Cdn3ListResponse( + List.of(new Cdn3RemoteStorageManager.Cdn3ListResponse.Entry("x", 3)), + "cursor"))))); + CompletableFutureTestUtil.assertFailsWithCause(IOException.class, + remoteStorageManager.list("abc/", Optional.empty(), 3).toCompletableFuture()); + } + + @Test + public void usage() throws JsonProcessingException { + wireMock.stubFor(get(urlPathEqualTo("/storage-manager/usage")) + .withQueryParam("prefix", equalTo("abc/")) + .willReturn(aResponse() + .withBody(SystemMapper.jsonMapper().writeValueAsString(new Cdn3RemoteStorageManager.UsageResponse( + 17, + 113))))); + final UsageInfo result = remoteStorageManager.calculateBytesUsed("abc/") + .toCompletableFuture() + .join(); + assertThat(result.numObjects()).isEqualTo(17); + assertThat(result.bytesUsed()).isEqualTo(113); + } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java index 78942375a..3b5f9b545 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java @@ -46,6 +46,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.junitpioneer.jupiter.cartesian.CartesianTest; import org.signal.libsignal.protocol.ecc.Curve; import org.signal.libsignal.zkgroup.VerificationFailedException; import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialPresentation; @@ -438,4 +439,43 @@ public class ArchiveControllerTest { ).toList()))); assertThat(response.getStatus()).isEqualTo(413); } + + @CartesianTest + public void list( + @CartesianTest.Values(booleans = {true, false}) final boolean cursorProvided, + @CartesianTest.Values(booleans = {true, false}) final boolean cursorReturned) + throws VerificationFailedException { + final BackupAuthCredentialPresentation presentation = backupAuthTestUtil.getPresentation( + BackupTier.MEDIA, backupKey, aci); + when(backupManager.authenticateBackupUser(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + new AuthenticatedBackupUser(presentation.getBackupId(), BackupTier.MEDIA))); + + final byte[] mediaId = TestRandomUtil.nextBytes(15); + final Optional expectedCursor = cursorProvided ? Optional.of("myCursor") : Optional.empty(); + final Optional returnedCursor = cursorReturned ? Optional.of("newCursor") : Optional.empty(); + + when(backupManager.list(any(), eq(expectedCursor), eq(17))) + .thenReturn(CompletableFuture.completedFuture(new BackupManager.ListMediaResult( + List.of(new BackupManager.StorageDescriptorWithLength(1, mediaId, 100)), + returnedCursor + ))); + + WebTarget target = resources.getJerseyTest() + .target("v1/archives/media/") + .queryParam("limit", 17); + if (cursorProvided) { + target = target.queryParam("cursor", "myCursor"); + } + final ArchiveController.ListResponse response = target + .request() + .header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize())) + .header("X-Signal-ZK-Auth-Signature", "aaa") + .get(ArchiveController.ListResponse.class); + + assertThat(response.storedMediaObjects()).hasSize(1); + assertThat(response.storedMediaObjects().get(0).objectLength()).isEqualTo(100); + assertThat(response.storedMediaObjects().get(0).mediaId()).isEqualTo(mediaId); + assertThat(response.cursor()).isEqualTo(returnedCursor.orElse(null)); + } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamoDbExtensionSchema.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamoDbExtensionSchema.java index 8381da844..3d1c879ba 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamoDbExtensionSchema.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamoDbExtensionSchema.java @@ -57,18 +57,6 @@ public final class DynamoDbExtensionSchema { .attributeType(ScalarAttributeType.B).build()), Collections.emptyList(), Collections.emptyList()), - BACKUP_MEDIA("backups_media_test", - BackupsDb.KEY_BACKUP_ID_HASH, - BackupsDb.KEY_MEDIA_ID, - List.of( - AttributeDefinition.builder() - .attributeName(BackupsDb.KEY_BACKUP_ID_HASH) - .attributeType(ScalarAttributeType.B).build(), - AttributeDefinition.builder() - .attributeName(BackupsDb.KEY_MEDIA_ID) - .attributeType(ScalarAttributeType.B).build()), - Collections.emptyList(), Collections.emptyList()), - CLIENT_RELEASES("client_releases_test", ClientReleases.ATTR_PLATFORM, ClientReleases.ATTR_VERSION, diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/util/HttpUtilsTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/util/HttpUtilsTest.java new file mode 100644 index 000000000..24a019cd6 --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/util/HttpUtilsTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.textsecuregcm.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Map; + +public class HttpUtilsTest { + + @Test + public void queryParameterStringPreservesOrder() { + final String result = HttpUtils.queryParamString(List.of( + Map.entry("a", "aval"), + Map.entry("b", "bval1"), + Map.entry("b", "bval2") + )); + // https://url.spec.whatwg.org/#example-constructing-urlsearchparams allows multiple parameters with the same key + // https://url.spec.whatwg.org/#example-searchparams-sort implies that the relative order of values for parameters + // with the same key must be preserved + assertThat(result).isEqualTo("?a=aval&b=bval1&b=bval2"); + } + + @Test + public void queryParameterStringEncodesUnsafeChars() { + final String result = HttpUtils.queryParamString(List.of(Map.entry("&k?e=y/!", "=v/a?l&u;e"))); + assertThat(result).isEqualTo("?%26k%3Fe%3Dy%2F%21=%3Dv%2Fa%3Fl%26u%3Be"); + } +}