From d36df3eaa9282e8b3b5f1be4dbad62b184eed0f8 Mon Sep 17 00:00:00 2001 From: ravi-signal <99042880+ravi-signal@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:47:46 -0500 Subject: [PATCH] Add new upload-for-copy backup endpoint --- .../textsecuregcm/WhisperServerService.java | 5 +- .../textsecuregcm/backup/BackupManager.java | 104 ++++++++++-------- ...iptor.java => BackupUploadDescriptor.java} | 2 +- .../backup/Cdn3BackupCredentialGenerator.java | 4 +- .../backup/Cdn3RemoteStorageManager.java | 4 +- .../backup/RemoteStorageManager.java | 2 +- .../controllers/ArchiveController.java | 50 ++++++++- .../textsecuregcm/limits/RateLimiters.java | 1 + .../workers/CommandDependencies.java | 6 + .../backup/BackupManagerTest.java | 41 ++++++- .../Cdn3BackupCredentialGeneratorTest.java | 2 +- .../backup/Cdn3RemoteStorageManagerTest.java | 8 +- .../controllers/ArchiveControllerTest.java | 36 +++++- 13 files changed, 202 insertions(+), 63 deletions(-) rename service/src/main/java/org/whispersystems/textsecuregcm/backup/{MessageBackupUploadDescriptor.java => BackupUploadDescriptor.java} (85%) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 5f4a06d86..c6b3554ad 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -709,6 +709,7 @@ public class WhisperServerService extends Application attachmentCdnBaseUris; + private final SecureRandom secureRandom = new SecureRandom(); private final Clock clock; public BackupManager( final BackupsDb backupsDb, final GenericServerSecretParams serverSecretParams, + final RateLimiters rateLimiters, + final TusAttachmentGenerator tusAttachmentGenerator, final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator, final RemoteStorageManager remoteStorageManager, final Map attachmentCdnBaseUris, final Clock clock) { this.backupsDb = backupsDb; this.serverSecretParams = serverSecretParams; + this.rateLimiters = rateLimiters; + this.tusAttachmentGenerator = tusAttachmentGenerator; this.cdn3BackupCredentialGenerator = cdn3BackupCredentialGenerator; this.remoteStorageManager = remoteStorageManager; this.clock = clock; @@ -131,26 +144,38 @@ public class BackupManager { * @param backupUser an already ZK authenticated backup user * @return the upload form */ - public CompletableFuture createMessageBackupUploadDescriptor( + public CompletableFuture createMessageBackupUploadDescriptor( final AuthenticatedBackupUser backupUser) { + checkBackupTier(backupUser, BackupTier.MESSAGES); + // this could race with concurrent updates, but the only effect would be last-writer-wins on the timestamp return backupsDb .addMessageBackup(backupUser) .thenApply(result -> cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser))); } + public BackupUploadDescriptor createTemporaryAttachmentUploadDescriptor(final AuthenticatedBackupUser backupUser) + throws RateLimitExceededException { + checkBackupTier(backupUser, BackupTier.MEDIA); + + RateLimiter.adaptLegacyException(() -> rateLimiters + .forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT) + .validate(rateLimitKey(backupUser))); + + final byte[] bytes = new byte[15]; + secureRandom.nextBytes(bytes); + final String attachmentKey = Base64.getUrlEncoder().encodeToString(bytes); + final AttachmentGenerator.Descriptor descriptor = tusAttachmentGenerator.generateAttachment(attachmentKey); + return new BackupUploadDescriptor(3, attachmentKey, descriptor.headers(), descriptor.signedUploadLocation()); + } + /** * Update the last update timestamps for the backupId in the presentation * * @param backupUser an already ZK authenticated backup user */ public CompletableFuture ttlRefresh(final AuthenticatedBackupUser backupUser) { - if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) { - Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); - throw Status.PERMISSION_DENIED - .withDescription("credential does not support ttl operation") - .asRuntimeException(); - } + checkBackupTier(backupUser, BackupTier.MESSAGES); // update message backup TTL return backupsDb.ttlRefresh(backupUser); } @@ -165,11 +190,7 @@ public class BackupManager { * @return Information about the existing backup */ public CompletableFuture backupInfo(final AuthenticatedBackupUser backupUser) { - if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) { - Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); - throw Status.PERMISSION_DENIED.withDescription("credential does not support info operation") - .asRuntimeException(); - } + checkBackupTier(backupUser, BackupTier.MESSAGES); return backupsDb.describeBackup(backupUser) .thenApply(backupDescription -> new BackupInfo( backupDescription.cdn(), @@ -187,12 +208,7 @@ public class BackupManager { * @return true if mediaLength bytes can be stored */ public CompletableFuture canStoreMedia(final AuthenticatedBackupUser backupUser, final long mediaLength) { - if (backupUser.backupTier().compareTo(BackupTier.MEDIA) < 0) { - Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); - throw Status.PERMISSION_DENIED - .withDescription("credential does not support storing media") - .asRuntimeException(); - } + checkBackupTier(backupUser, BackupTier.MEDIA); return backupsDb.getMediaUsage(backupUser) .thenComposeAsync(info -> { final boolean canStore = MAX_TOTAL_BACKUP_MEDIA_BYTES - info.usageInfo().bytesUsed() >= mediaLength; @@ -243,12 +259,7 @@ public class BackupManager { final int sourceLength, final MediaEncryptionParameters encryptionParameters, final byte[] destinationMediaId) { - if (backupUser.backupTier().compareTo(BackupTier.MEDIA) < 0) { - Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); - throw Status.PERMISSION_DENIED - .withDescription("credential does not support storing media") - .asRuntimeException(); - } + checkBackupTier(backupUser, BackupTier.MEDIA); if (sourceLength > MAX_MEDIA_OBJECT_SIZE) { throw Status.INVALID_ARGUMENT .withDescription("Invalid sourceObject size") @@ -262,7 +273,7 @@ public class BackupManager { return CompletableFuture.failedFuture(e); } - final MessageBackupUploadDescriptor dst = cdn3BackupCredentialGenerator.generateUpload( + final BackupUploadDescriptor dst = cdn3BackupCredentialGenerator.generateUpload( cdnMediaPath(backupUser, destinationMediaId)); final int destinationLength = encryptionParameters.outputSize(sourceLength); @@ -309,12 +320,7 @@ public class BackupManager { * @return A map of headers to include with CDN requests */ public Map generateReadAuth(final AuthenticatedBackupUser backupUser) { - if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) { - Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); - throw Status.PERMISSION_DENIED - .withDescription("credential does not support read auth operation") - .asRuntimeException(); - } + checkBackupTier(backupUser, BackupTier.MESSAGES); return cdn3BackupCredentialGenerator.readHeaders(backupUser.backupDir()); } @@ -339,12 +345,7 @@ public class BackupManager { final AuthenticatedBackupUser backupUser, final Optional cursor, final int limit) { - if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) { - Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); - throw Status.PERMISSION_DENIED - .withDescription("credential does not support list operation") - .asRuntimeException(); - } + checkBackupTier(backupUser, BackupTier.MESSAGES); return remoteStorageManager.list(cdnMediaDirectory(backupUser), cursor, limit) .thenApply(result -> new ListMediaResult( @@ -370,12 +371,7 @@ public class BackupManager { public CompletableFuture delete(final AuthenticatedBackupUser backupUser, final List storageDescriptors) { - if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) { - Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); - throw Status.PERMISSION_DENIED - .withDescription("credential does not support list operation") - .asRuntimeException(); - } + checkBackupTier(backupUser, BackupTier.MESSAGES); if (storageDescriptors.stream().anyMatch(sd -> sd.cdn() != remoteStorageManager.cdnNumber())) { throw Status.INVALID_ARGUMENT @@ -430,6 +426,7 @@ public class BackupManager { } private static final ECPublicKey INVALID_PUBLIC_KEY = Curve.generateKeyPair().getPublicKey(); + /** * Authenticate the ZK anonymous backup credential's presentation *

@@ -532,6 +529,7 @@ public class BackupManager { } interface PresentationSignatureVerifier { + BackupTier verifySignature(byte[] signature, ECPublicKey publicKey); } @@ -576,6 +574,22 @@ public class BackupManager { }; } + /** + * Check that the authenticated backup user is authorized to use the provided backupTier + * + * @param backupUser The backup user to check + * @param backupTier The authorization level to verify the backupUser has access to + * @throws {@link Status#PERMISSION_DENIED} error if the backup user is not authorized to access {@code backupTier} + */ + private static void checkBackupTier(final AuthenticatedBackupUser backupUser, final BackupTier backupTier) { + if (backupUser.backupTier().compareTo(backupTier) < 0) { + Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); + throw Status.PERMISSION_DENIED + .withDescription("credential does not support the requested operation") + .asRuntimeException(); + } + } + @VisibleForTesting static String encodeMediaIdForCdn(final byte[] bytes) { return Base64.getUrlEncoder().encodeToString(bytes); @@ -596,4 +610,8 @@ public class BackupManager { private static String cdnMediaPath(final AuthenticatedBackupUser backupUser, final byte[] mediaId) { return "%s%s".formatted(cdnMediaDirectory(backupUser), encodeMediaIdForCdn(mediaId)); } + + static String rateLimitKey(final AuthenticatedBackupUser backupUser) { + return Base64.getEncoder().encodeToString(BackupsDb.hashedBackupId(backupUser.backupId())); + } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/MessageBackupUploadDescriptor.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupUploadDescriptor.java similarity index 85% rename from service/src/main/java/org/whispersystems/textsecuregcm/backup/MessageBackupUploadDescriptor.java rename to service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupUploadDescriptor.java index dff64dd20..6485201ea 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/MessageBackupUploadDescriptor.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupUploadDescriptor.java @@ -7,7 +7,7 @@ package org.whispersystems.textsecuregcm.backup; import java.util.Map; -public record MessageBackupUploadDescriptor( +public record BackupUploadDescriptor( int cdn, String key, Map headers, diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java index 536ee0179..075b1708f 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java @@ -49,7 +49,7 @@ public class Cdn3BackupCredentialGenerator { .build(); } - public MessageBackupUploadDescriptor generateUpload(final String key) { + public BackupUploadDescriptor generateUpload(final String key) { if (key.isBlank()) { throw new IllegalArgumentException("Upload descriptors must have non-empty keys"); } @@ -60,7 +60,7 @@ public class Cdn3BackupCredentialGenerator { HttpHeaders.AUTHORIZATION, HeaderUtils.basicAuthHeader(credentials), "Upload-Metadata", String.format("filename %s", b64Key)); - return new MessageBackupUploadDescriptor( + return new BackupUploadDescriptor( BACKUP_CDN, key, headers, diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java index 2508a3b32..8d5757504 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java @@ -113,7 +113,7 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager { final URI sourceUri, final int expectedSourceLength, final MediaEncryptionParameters encryptionParameters, - final MessageBackupUploadDescriptor uploadDescriptor) { + final BackupUploadDescriptor uploadDescriptor) { if (uploadDescriptor.cdn() != cdnNumber()) { throw new IllegalArgumentException("Cdn3RemoteStorageManager can only copy to cdn3"); @@ -152,7 +152,7 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager { private HttpRequest createCopyRequest( final int expectedSourceLength, - final MessageBackupUploadDescriptor uploadDescriptor, + final BackupUploadDescriptor uploadDescriptor, BackupMediaEncrypter encrypter, HttpResponse>> response) throws IOException { if (response.statusCode() == Response.Status.NOT_FOUND.getStatusCode()) { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java index 9212d2c7b..672ac1a02 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java @@ -36,7 +36,7 @@ public interface RemoteStorageManager { URI sourceUri, int expectedSourceLength, MediaEncryptionParameters encryptionParameters, - MessageBackupUploadDescriptor uploadDescriptor); + BackupUploadDescriptor uploadDescriptor); /** * Result of a {@link #list} operation diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java index 00b4e7c31..49186070a 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java @@ -384,7 +384,7 @@ public class ArchiveController { } - public record MessageBackupResponse( + public record UploadDescriptorResponse( @Schema(description = "Indicates the CDN type. 3 indicates resumable uploads using TUS") int cdn, @Schema(description = "The location within the specified cdn where the finished upload can be found.") @@ -400,10 +400,10 @@ public class ArchiveController { @Operation( summary = "Fetch message backup upload form", description = "Retrieve an upload form that can be used to perform a resumable upload of a message backup.") - @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = MessageBackupResponse.class))) + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = UploadDescriptorResponse.class))) @ApiResponse(responseCode = "429", description = "Rate limited.") @ApiResponseZkAuth - public CompletionStage backup( + public CompletionStage backup( @ReadOnly @Auth final Optional account, @Parameter(description = BackupAuthCredentialPresentationHeader.DESCRIPTION, schema = @Schema(implementation = String.class)) @@ -418,7 +418,49 @@ public class ArchiveController { } return backupManager.authenticateBackupUser(presentation.presentation, signature.signature) .thenCompose(backupManager::createMessageBackupUploadDescriptor) - .thenApply(result -> new MessageBackupResponse( + .thenApply(result -> new UploadDescriptorResponse( + result.cdn(), + result.key(), + result.headers(), + result.signedUploadLocation())); + } + + @GET + @Path("/media/upload/form") + @Produces(MediaType.APPLICATION_JSON) + @Operation( + summary = "Fetch media attachment upload form", + description = """ + Retrieve an upload form that can be used to perform a resumable upload of an attachment. After uploading, the + attachment can be copied into the backup at PUT /archives/media/. + + Like the account authenticated version at /attachments, the uploaded object is only temporary. + """) + @ApiResponse(responseCode = "200", content = @Content(schema = @Schema(implementation = UploadDescriptorResponse.class))) + @ApiResponse(responseCode = "429", description = "Rate limited.") + @ApiResponseZkAuth + public CompletionStage uploadTemporaryAttachment( + @ReadOnly @Auth final Optional account, + + @Parameter(description = BackupAuthCredentialPresentationHeader.DESCRIPTION, schema = @Schema(implementation = String.class)) + @NotNull + @HeaderParam(X_SIGNAL_ZK_AUTH) final ArchiveController.BackupAuthCredentialPresentationHeader presentation, + + @Parameter(description = BackupAuthCredentialPresentationSignature.DESCRIPTION, schema = @Schema(implementation = String.class)) + @NotNull + @HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature) { + if (account.isPresent()) { + throw new BadRequestException("must not use authenticated connection for anonymous operations"); + } + return backupManager.authenticateBackupUser(presentation.presentation, signature.signature) + .thenApply(backupUser -> { + try { + return backupManager.createTemporaryAttachmentUploadDescriptor(backupUser); + } catch (RateLimitExceededException e) { + throw ExceptionUtils.wrap(e); + } + }) + .thenApply(result -> new UploadDescriptorResponse( result.cdn(), result.key(), result.headers(), diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/limits/RateLimiters.java b/service/src/main/java/org/whispersystems/textsecuregcm/limits/RateLimiters.java index 0d2f2ae48..4b510e0fc 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/limits/RateLimiters.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/limits/RateLimiters.java @@ -26,6 +26,7 @@ public class RateLimiters extends BaseRateLimiters { VERIFY("verify", false, new RateLimiterConfig(6, Duration.ofSeconds(30))), PIN("pin", false, new RateLimiterConfig(10, Duration.ofDays(1))), ATTACHMENT("attachmentCreate", false, new RateLimiterConfig(50, Duration.ofMillis(1200))), + BACKUP_ATTACHMENT("backupAttachmentCreate", true, new RateLimiterConfig(10_000, Duration.ofSeconds(1))), PRE_KEYS("prekeys", false, new RateLimiterConfig(6, Duration.ofMinutes(10))), MESSAGES("messages", false, new RateLimiterConfig(60, Duration.ofSeconds(1))), STORIES("stories", false, new RateLimiterConfig(5_000, Duration.ofSeconds(8))), diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java index f28e6c3a7..082c26cc8 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java @@ -19,6 +19,7 @@ import org.signal.libsignal.zkgroup.GenericServerSecretParams; import org.signal.libsignal.zkgroup.InvalidInputException; import org.whispersystems.textsecuregcm.WhisperServerConfiguration; import org.whispersystems.textsecuregcm.WhisperServerService; +import org.whispersystems.textsecuregcm.attachments.TusAttachmentGenerator; import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator; import org.whispersystems.textsecuregcm.backup.BackupManager; import org.whispersystems.textsecuregcm.backup.BackupsDb; @@ -27,6 +28,7 @@ import org.whispersystems.textsecuregcm.backup.Cdn3RemoteStorageManager; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.controllers.SecureStorageController; import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery2Controller; +import org.whispersystems.textsecuregcm.limits.RateLimiters; import org.whispersystems.textsecuregcm.metrics.MetricsUtil; import org.whispersystems.textsecuregcm.push.ClientPresenceManager; import org.whispersystems.textsecuregcm.redis.ClusterFaultTolerantRedisCluster; @@ -198,6 +200,8 @@ record CommandDependencies( secureStorageClient, secureValueRecovery2Client, clientPresenceManager, registrationRecoveryPasswordsManager, accountLockExecutor, clientPresenceExecutor, clock); + RateLimiters rateLimiters = RateLimiters.createAndValidate(configuration.getLimitsConfiguration(), + dynamicConfigurationManager, rateLimitersCluster); final BackupsDb backupsDb = new BackupsDb(dynamoDbAsyncClient, configuration.getDynamoDbTables().getBackups().getTableName(), clock); final GenericServerSecretParams backupsGenericZkSecretParams; @@ -210,6 +214,8 @@ record CommandDependencies( final BackupManager backupManager = new BackupManager( backupsDb, backupsGenericZkSecretParams, + rateLimiters, + new TusAttachmentGenerator(configuration.getTus()), new Cdn3BackupCredentialGenerator(configuration.getTus()), new Cdn3RemoteStorageManager( remoteStorageExecutor, diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java index 7478c8772..c75f67e8e 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java @@ -13,6 +13,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; @@ -60,7 +61,11 @@ import org.signal.libsignal.protocol.ecc.ECKeyPair; import org.signal.libsignal.zkgroup.GenericServerSecretParams; import org.signal.libsignal.zkgroup.VerificationFailedException; import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialPresentation; +import org.whispersystems.textsecuregcm.attachments.TusAttachmentGenerator; import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser; +import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; +import org.whispersystems.textsecuregcm.limits.RateLimiter; +import org.whispersystems.textsecuregcm.limits.RateLimiters; import org.whispersystems.textsecuregcm.storage.DynamoDbExtension; import org.whispersystems.textsecuregcm.storage.DynamoDbExtensionSchema; import org.whispersystems.textsecuregcm.util.AttributeValues; @@ -79,6 +84,8 @@ public class BackupManagerTest { private final TestClock testClock = TestClock.now(); private final BackupAuthTestUtil backupAuthTestUtil = new BackupAuthTestUtil(testClock); + private final RateLimiter mediaUploadLimiter = mock(RateLimiter.class); + private final TusAttachmentGenerator tusAttachmentGenerator = mock(TusAttachmentGenerator.class); private final Cdn3BackupCredentialGenerator tusCredentialGenerator = mock(Cdn3BackupCredentialGenerator.class); private final RemoteStorageManager remoteStorageManager = mock(RemoteStorageManager.class); private final byte[] backupKey = TestRandomUtil.nextBytes(32); @@ -90,8 +97,12 @@ public class BackupManagerTest { @BeforeEach public void setup() { - reset(tusCredentialGenerator); + reset(tusCredentialGenerator, mediaUploadLimiter); testClock.unpin(); + + final RateLimiters rateLimiters = mock(RateLimiters.class); + when(rateLimiters.forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT)).thenReturn(mediaUploadLimiter); + this.backupsDb = new BackupsDb( DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(), DynamoDbExtensionSchema.Tables.BACKUPS.tableName(), @@ -99,6 +110,8 @@ public class BackupManagerTest { this.backupManager = new BackupManager( backupsDb, backupAuthTestUtil.params, + rateLimiters, + tusAttachmentGenerator, tusCredentialGenerator, remoteStorageManager, Map.of(3, "cdn3.example.org/attachments"), @@ -127,6 +140,28 @@ public class BackupManagerTest { checkExpectedExpirations(now, backupTier == BackupTier.MEDIA ? now : null, backupUser); } + @Test + public void createTemporaryMediaAttachmentRateLimited() throws RateLimitExceededException { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + doThrow(new RateLimitExceededException(null, true)) + .when(mediaUploadLimiter) + .validate(eq(BackupManager.rateLimitKey(backupUser))); + + assertThatExceptionOfType(RateLimitExceededException.class) + .isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser)) + .satisfies(e -> assertThat(e.isLegacy()).isFalse()); + } + + @Test + public void createTemporaryMediaAttachmentWrongTier() throws RateLimitExceededException { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MESSAGES); + assertThatExceptionOfType(StatusRuntimeException.class) + .isThrownBy(() -> backupManager.createTemporaryAttachmentUploadDescriptor(backupUser)) + .extracting(StatusRuntimeException::getStatus) + .extracting(Status::getCode) + .isEqualTo(Status.Code.PERMISSION_DENIED); + } + @ParameterizedTest @EnumSource(mode = EnumSource.Mode.EXCLUDE, names = {"NONE"}) public void ttlRefresh(final BackupTier backupTier) { @@ -317,7 +352,7 @@ public class BackupManagerTest { public void copySuccess() { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); when(tusCredentialGenerator.generateUpload(any())) - .thenReturn(new MessageBackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); + .thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any())) .thenReturn(CompletableFuture.completedFuture(null)); final MediaEncryptionParameters encryptionParams = new MediaEncryptionParameters( @@ -343,7 +378,7 @@ public class BackupManagerTest { public void copyFailure() { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); when(tusCredentialGenerator.generateUpload(any())) - .thenReturn(new MessageBackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); + .thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any())) .thenReturn(CompletableFuture.failedFuture(new SourceObjectNotFoundException())); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java index 3bd01a210..8f38a9ecb 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java @@ -22,7 +22,7 @@ public class Cdn3BackupCredentialGeneratorTest { new SecretBytes(TestRandomUtil.nextBytes(32)), "https://example.org/upload")); - final MessageBackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir/key"); + final BackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir/key"); assertThat(messageBackupUploadDescriptor.signedUploadLocation()).isEqualTo("https://example.org/upload/backups"); assertThat(messageBackupUploadDescriptor.key()).isEqualTo("subdir/key"); assertThat(messageBackupUploadDescriptor.headers()).containsKey("Authorization"); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java index 40b0bd9e3..9920016e7 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java @@ -124,7 +124,7 @@ public class Cdn3RemoteStorageManagerTest { URI.create(wireMock.url("/cdn" + sourceCdn + "/source/small")), expectedSource.length(), encryptionParameters, - new MessageBackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) + new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) .toCompletableFuture().join(); final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).get(0).getBody(); @@ -148,7 +148,7 @@ public class Cdn3RemoteStorageManagerTest { URI.create(wireMock.url("/cdn3/source/large")), LARGE.length(), params, - new MessageBackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) + new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) .toCompletableFuture().join(); final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).get(0).getBody(); @@ -165,7 +165,7 @@ public class Cdn3RemoteStorageManagerTest { URI.create(wireMock.url("/cdn3/source/small")), SMALL_CDN3.length() - 1, new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV), - new MessageBackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) + new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) .toCompletableFuture()); } @@ -176,7 +176,7 @@ public class Cdn3RemoteStorageManagerTest { URI.create(wireMock.url("/cdn3/source/missing")), 1, new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV), - new MessageBackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) + new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) .toCompletableFuture()); } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java index 76b121e3c..977497c30 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java @@ -26,6 +26,7 @@ import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -36,7 +37,6 @@ import javax.ws.rs.client.Invocation; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import org.checkerframework.checker.units.qual.A; import org.glassfish.jersey.server.ServerProperties; import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory; import org.junit.jupiter.api.BeforeEach; @@ -68,6 +68,7 @@ import org.whispersystems.textsecuregcm.backup.BackupManager; import org.whispersystems.textsecuregcm.backup.BackupTier; import org.whispersystems.textsecuregcm.backup.InvalidLengthException; import org.whispersystems.textsecuregcm.backup.SourceObjectNotFoundException; +import org.whispersystems.textsecuregcm.backup.BackupUploadDescriptor; import org.whispersystems.textsecuregcm.mappers.CompletionExceptionMapper; import org.whispersystems.textsecuregcm.mappers.GrpcStatusRuntimeExceptionMapper; import org.whispersystems.textsecuregcm.mappers.RateLimitExceededExceptionMapper; @@ -108,6 +109,7 @@ public class ArchiveControllerTest { GET, v1/archives/auth/read, GET, v1/archives/, GET, v1/archives/upload/form, + GET, v1/archives/media/upload/form, POST, v1/archives/, PUT, v1/archives/keys, '{"backupIdPublicKey": "aaaaa"}' PUT, v1/archives/media, '{ @@ -531,6 +533,38 @@ public class ArchiveControllerTest { assertThat(response.getStatus()).isEqualTo(204); } + @Test + public void mediaUploadForm() throws RateLimitExceededException, VerificationFailedException { + final BackupAuthCredentialPresentation presentation = + backupAuthTestUtil.getPresentation(BackupTier.MEDIA, backupKey, aci); + when(backupManager.authenticateBackupUser(any(), any())) + .thenReturn(CompletableFuture.completedFuture(backupUser(presentation.getBackupId(), BackupTier.MEDIA))); + when(backupManager.createTemporaryAttachmentUploadDescriptor(any())) + .thenReturn(new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org")); + final ArchiveController.UploadDescriptorResponse desc = resources.getJerseyTest() + .target("v1/archives/media/upload/form") + .request() + .header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize())) + .header("X-Signal-ZK-Auth-Signature", "aaa") + .get(ArchiveController.UploadDescriptorResponse.class); + assertThat(desc.cdn()).isEqualTo(3); + assertThat(desc.key()).isEqualTo("abc"); + assertThat(desc.headers()).containsExactlyEntriesOf(Map.of("k", "v")); + assertThat(desc.signedUploadLocation()).isEqualTo("example.org"); + + // rate limit + when(backupManager.createTemporaryAttachmentUploadDescriptor(any())) + .thenThrow(new RateLimitExceededException(null, false)); + final Response response = resources.getJerseyTest() + .target("v1/archives/media/upload/form") + .request() + .header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize())) + .header("X-Signal-ZK-Auth-Signature", "aaa") + .get(); + assertThat(response.getStatus()).isEqualTo(429); + } + + private static AuthenticatedBackupUser backupUser(byte[] backupId, BackupTier backupTier) { return new AuthenticatedBackupUser(backupId, backupTier, "myBackupDir", "myMediaDir"); }