From cc6cf8194f21f54849a23a17ed9cf759e6a30f70 Mon Sep 17 00:00:00 2001 From: Ravi Khadiwala Date: Tue, 9 Jan 2024 13:52:15 -0600 Subject: [PATCH] Add media deletion endpoint --- .../textsecuregcm/backup/BackupManager.java | 103 ++++++++++++++++-- .../textsecuregcm/backup/BackupsDb.java | 9 +- .../backup/Cdn3BackupCredentialGenerator.java | 5 +- .../backup/Cdn3RemoteStorageManager.java | 58 ++++++++-- .../backup/RemoteStorageManager.java | 8 ++ .../controllers/ArchiveController.java | 49 +++++++++ .../backup/BackupManagerTest.java | 95 +++++++++++++++- .../textsecuregcm/backup/BackupsDbTest.java | 6 +- .../Cdn3BackupCredentialGeneratorTest.java | 2 +- .../backup/Cdn3RemoteStorageManagerTest.java | 14 +++ .../controllers/ArchiveControllerTest.java | 25 +++++ 11 files changed, 340 insertions(+), 34 deletions(-) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java index 2e73f1427..f8539db45 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java @@ -11,6 +11,7 @@ import io.micrometer.core.instrument.Metrics; import java.net.URI; import java.time.Clock; import java.time.Duration; +import java.util.ArrayList; import java.util.Base64; import java.util.HexFormat; import java.util.List; @@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory; import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser; import org.whispersystems.textsecuregcm.metrics.MetricsUtil; import org.whispersystems.textsecuregcm.util.ExceptionUtils; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; public class BackupManager { @@ -125,12 +128,10 @@ public class BackupManager { */ public CompletableFuture createMessageBackupUploadDescriptor( final AuthenticatedBackupUser backupUser) { - final String encodedBackupId = encodeBackupIdForCdn(backupUser); - // this could race with concurrent updates, but the only effect would be last-writer-wins on the timestamp return backupsDb .addMessageBackup(backupUser) - .thenApply(result -> cdn3BackupCredentialGenerator.generateUpload(encodedBackupId, MESSAGE_BACKUP_NAME)); + .thenApply(result -> cdn3BackupCredentialGenerator.generateUpload(cdnMessageBackupName(backupUser))); } /** @@ -194,8 +195,7 @@ public class BackupManager { // The user is out of quota, and we have not recently recalculated the user's usage. Double check by doing a // hard recalculation before actually forbidding the user from storing additional media. - final String mediaPrefix = "%s/%s/".formatted(encodeBackupIdForCdn(backupUser), MEDIA_DIRECTORY_NAME); - return this.remoteStorageManager.calculateBytesUsed(mediaPrefix) + return this.remoteStorageManager.calculateBytesUsed(cdnMediaDirectory(backupUser)) .thenCompose(usage -> backupsDb .setMediaUsage(backupUser, usage) .thenApply(ignored -> usage)) @@ -249,15 +249,14 @@ public class BackupManager { } final MessageBackupUploadDescriptor dst = cdn3BackupCredentialGenerator.generateUpload( - encodeBackupIdForCdn(backupUser), - "%s/%s".formatted(MEDIA_DIRECTORY_NAME, encodeForCdn(destinationMediaId))); + cdnMediaPath(backupUser, destinationMediaId)); final int destinationLength = encryptionParameters.outputSize(sourceLength); final URI sourceUri = attachmentReadUri(sourceCdn, sourceKey); return this.backupsDb // Write the ddb updates before actually updating backing storage - .trackMedia(backupUser, destinationLength) + .trackMedia(backupUser, 1, destinationLength) // Actually copy the objects. If the copy fails, our estimated quota usage may not be exact .thenComposeAsync(ignored -> remoteStorageManager.copy(sourceUri, sourceLength, encryptionParameters, dst)) @@ -267,7 +266,7 @@ public class BackupManager { throw ExceptionUtils.wrap(unwrapped); } // In cases where we know the copy fails without writing anything, we can try to restore the user's quota - return this.backupsDb.trackMedia(backupUser, -destinationLength).whenComplete((ignored, ignoredEx) -> { + return this.backupsDb.trackMedia(backupUser, -1, -destinationLength).whenComplete((ignored, ignoredEx) -> { throw ExceptionUtils.wrap(unwrapped); }); }) @@ -335,8 +334,7 @@ public class BackupManager { .withDescription("credential does not support list operation") .asRuntimeException(); } - final String mediaPrefix = "%s/%s/".formatted(encodeBackupIdForCdn(backupUser), MEDIA_DIRECTORY_NAME); - return remoteStorageManager.list(mediaPrefix, cursor, limit) + return remoteStorageManager.list(cdnMediaDirectory(backupUser), cursor, limit) .thenApply(result -> new ListMediaResult( result @@ -352,6 +350,74 @@ public class BackupManager { )); } + + private sealed interface Either permits DeleteSuccess, DeleteFailure {} + + private record DeleteSuccess(long usage) implements Either {} + + private record DeleteFailure(Throwable e) implements Either {} + + public CompletableFuture delete(final AuthenticatedBackupUser backupUser, + final List storageDescriptors) { + if (backupUser.backupTier().compareTo(BackupTier.MESSAGES) < 0) { + Metrics.counter(ZK_AUTHZ_FAILURE_COUNTER_NAME).increment(); + throw Status.PERMISSION_DENIED + .withDescription("credential does not support list operation") + .asRuntimeException(); + } + + if (storageDescriptors.stream().anyMatch(sd -> sd.cdn() != remoteStorageManager.cdnNumber())) { + throw Status.INVALID_ARGUMENT + .withDescription("unsupported media cdn provided") + .asRuntimeException(); + } + + return Flux + .fromIterable(storageDescriptors) + + // Issue deletes for all storage descriptors (proceeds with default flux concurrency) + .flatMap(descriptor -> Mono.fromCompletionStage( + remoteStorageManager + .delete(cdnMediaPath(backupUser, descriptor.key)) + // Squash errors/success into a single type + .handle((bytesDeleted, throwable) -> throwable != null + ? new DeleteFailure(throwable) + : new DeleteSuccess(bytesDeleted)) + )) + + // Update backupsDb with the change in usage + .collectList() + .flatMap(eithers -> { + // count up usage changes + long totalBytesDeleted = 0; + long totalCountDeleted = 0; + final List toThrow = new ArrayList<>(); + for (Either either : eithers) { + switch (either) { + case DeleteFailure f: + toThrow.add(f.e()); + break; + case DeleteSuccess s when s.usage() > 0: + totalBytesDeleted += s.usage(); + totalCountDeleted++; + break; + default: + break; + } + } + final Mono result = toThrow.isEmpty() + ? Mono.empty() + : Mono.error(toThrow.stream().reduce((t1, t2) -> { + t1.addSuppressed(t2); + return t1; + }).get()); + return Mono + .fromCompletionStage(this.backupsDb.trackMedia(backupUser, -totalCountDeleted, -totalBytesDeleted)) + .then(result); + }) + .toFuture(); + } + /** * Authenticate the ZK anonymous backup credential's presentation *

@@ -452,7 +518,8 @@ public class BackupManager { return encodeForCdn(BackupsDb.hashedBackupId(backupUser.backupId())); } - private static String encodeForCdn(final byte[] bytes) { + @VisibleForTesting + static String encodeForCdn(final byte[] bytes) { return Base64.getUrlEncoder().encodeToString(bytes); } @@ -460,4 +527,16 @@ public class BackupManager { return Base64.getUrlDecoder().decode(base64); } + private static String cdnMessageBackupName(final AuthenticatedBackupUser backupUser) { + return "%s/%s".formatted(encodeBackupIdForCdn(backupUser), MESSAGE_BACKUP_NAME); + } + + private static String cdnMediaDirectory(final AuthenticatedBackupUser backupUser) { + return "%s/%s/".formatted(encodeBackupIdForCdn(backupUser), MEDIA_DIRECTORY_NAME); + } + + private static String cdnMediaPath(final AuthenticatedBackupUser backupUser, final byte[] mediaId) { + return "%s%s".formatted(cdnMediaDirectory(backupUser), encodeForCdn(mediaId)); + } + } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java index 318bed985..9476f76e7 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupsDb.java @@ -126,18 +126,19 @@ public class BackupsDb { * Update the quota in the backup table * * @param backupUser The backup user - * @param mediaLength The length of the media after encryption. A negative length implies the media is being removed + * @param mediaBytesDelta The length of the media after encryption. A negative length implies media being removed + * @param mediaCountDelta The number of media objects being added, or if negative, removed * @return A stage that completes successfully once the table are updated. */ - CompletableFuture trackMedia(final AuthenticatedBackupUser backupUser, final int mediaLength) { + CompletableFuture trackMedia(final AuthenticatedBackupUser backupUser, final long mediaCountDelta, final long mediaBytesDelta) { final Instant now = clock.instant(); return dynamoClient .updateItem( // Update the media quota and TTL UpdateBuilder.forUser(backupTableName, backupUser) .setRefreshTimes(now) - .incrementMediaBytes(mediaLength) - .incrementMediaCount(Integer.signum(mediaLength)) + .incrementMediaBytes(mediaBytesDelta) + .incrementMediaCount(mediaCountDelta) .updateItemBuilder() .build()) .thenRun(Util.NOOP); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java index 3b056d883..536ee0179 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGenerator.java @@ -49,11 +49,10 @@ public class Cdn3BackupCredentialGenerator { .build(); } - public MessageBackupUploadDescriptor generateUpload(final String hashedBackupId, final String objectName) { - if (hashedBackupId.isBlank() || objectName.isBlank()) { + public MessageBackupUploadDescriptor generateUpload(final String key) { + if (key.isBlank()) { throw new IllegalArgumentException("Upload descriptors must have non-empty keys"); } - final String key = "%s/%s".formatted(hashedBackupId, objectName); final String entity = WRITE_ENTITY_PREFIX + key; final ExternalServiceCredentials credentials = credentialsGenerator.generateFor(entity); final String b64Key = Base64.getEncoder().encodeToString(key.getBytes(StandardCharsets.UTF_8)); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java index 99076b721..4c278ef44 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java @@ -1,5 +1,7 @@ package org.whispersystems.textsecuregcm.backup; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Timer; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -21,8 +23,6 @@ import java.util.stream.Stream; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; import javax.ws.rs.core.Response; -import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.Timer; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -168,10 +168,7 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager { cursor.ifPresent(s -> queryParams.put("cursor", cursor.get())); final HttpRequest request = HttpRequest.newBuilder().GET() - .uri(URI.create("%s/%s/%s".formatted( - storageManagerBaseUrl, - Cdn3BackupCredentialGenerator.CDN_PATH, - HttpUtils.queryParamString(queryParams.entrySet())))) + .uri(URI.create("%s%s".formatted(listUrl(), HttpUtils.queryParamString(queryParams.entrySet())))) .header(CLIENT_ID_HEADER, clientId) .header(CLIENT_SECRET_HEADER, clientSecret) .build(); @@ -226,12 +223,13 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager { */ record UsageResponse(@NotNull long numObjects, @NotNull long bytesUsed) {} + @Override public CompletionStage calculateBytesUsed(final String prefix) { final Timer.Sample sample = Timer.start(); final HttpRequest request = HttpRequest.newBuilder().GET() - .uri(URI.create("%s/usage%s".formatted( - storageManagerBaseUrl, + .uri(URI.create("%s%s".formatted( + usageUrl(), HttpUtils.queryParamString(Map.of("prefix", prefix).entrySet())))) .header(CLIENT_ID_HEADER, clientId) .header(CLIENT_SECRET_HEADER, clientSecret) @@ -260,5 +258,49 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager { return new UsageInfo(response.bytesUsed(), response.numObjects); } + /** + * Serialized delete response from storage manager + */ + record DeleteResponse(@NotNull long bytesDeleted) {} + + public CompletionStage delete(final String key) { + final HttpRequest request = HttpRequest.newBuilder().DELETE() + .uri(URI.create(deleteUrl(key))) + .header(CLIENT_ID_HEADER, clientId) + .header(CLIENT_SECRET_HEADER, clientSecret) + .build(); + return this.storageManagerHttpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()) + .thenApply(response -> { + Metrics.counter(STORAGE_MANAGER_STATUS_COUNTER_NAME, + OPERATION_TAG_NAME, "delete", + STATUS_TAG_NAME, Integer.toString(response.statusCode())) + .increment(); + try { + return parseDeleteResponse(response); + } catch (IOException e) { + throw ExceptionUtils.wrap(e); + } + }); + } + + private long parseDeleteResponse(final HttpResponse httpDeleteResponse) throws IOException { + if (!HttpUtils.isSuccessfulResponse(httpDeleteResponse.statusCode())) { + throw new IOException("Failed to retrieve usage: " + httpDeleteResponse.statusCode()); + } + return SystemMapper.jsonMapper().readValue(httpDeleteResponse.body(), DeleteResponse.class).bytesDeleted(); + } + + private String deleteUrl(final String key) { + return "%s/%s/%s".formatted(storageManagerBaseUrl, Cdn3BackupCredentialGenerator.CDN_PATH, key); + } + + private String usageUrl() { + return "%s/usage".formatted(storageManagerBaseUrl); + } + + private String listUrl() { + return "%s/%s/".formatted(storageManagerBaseUrl, Cdn3BackupCredentialGenerator.CDN_PATH); + } + } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java index c770546b6..9212d2c7b 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java @@ -74,4 +74,12 @@ public interface RemoteStorageManager { * @return The number of bytes used */ CompletionStage calculateBytesUsed(final String prefix); + + /** + * Delete the specified object. + * + * @param key the key of the stored object to delete. + * @return the number of bytes freed by the deletion operation + */ + CompletionStage delete(final String key); } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java index b5522c6b1..bfb1ef052 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/ArchiveController.java @@ -683,4 +683,53 @@ public class ArchiveController { .toList(), result.cursor().orElse(null))); } + + public record DeleteMedia(@Size(min = 1, max = 1000) List<@Valid MediaToDelete> mediaToDelete) { + + public record MediaToDelete( + @Schema(description = "The backup cdn where this media object is stored") + @NotNull + Integer cdn, + + @Schema(description = "The mediaId of the object in URL-safe base64", implementation = String.class) + @JsonSerialize(using = ByteArrayBase64UrlAdapter.Serializing.class) + @JsonDeserialize(using = ByteArrayBase64UrlAdapter.Deserializing.class) + @NotNull + @ExactlySize(15) + byte[] mediaId + ) {} + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + @Path("/media/delete") + @Operation(summary = "Delete media objects", + description = "Delete media objects stored with this backup-id") + @ApiResponse(responseCode = "204") + @ApiResponse(responseCode = "429", description = "Rate limited.") + @ApiResponseZkAuth + public CompletionStage deleteMedia( + @Auth final Optional account, + + @Parameter(description = BackupAuthCredentialPresentationHeader.DESCRIPTION, schema = @Schema(implementation = String.class)) + @NotNull + @HeaderParam(X_SIGNAL_ZK_AUTH) final BackupAuthCredentialPresentationHeader presentation, + + @Parameter(description = BackupAuthCredentialPresentationSignature.DESCRIPTION, schema = @Schema(implementation = String.class)) + @NotNull + @HeaderParam(X_SIGNAL_ZK_AUTH_SIGNATURE) final BackupAuthCredentialPresentationSignature signature, + + @Valid @NotNull DeleteMedia deleteMedia) { + if (account.isPresent()) { + throw new BadRequestException("must not use authenticated connection for anonymous operations"); + } + + return backupManager + .authenticateBackupUser(presentation.presentation, signature.signature) + .thenCompose(authenticatedBackupUser -> backupManager.delete(authenticatedBackupUser, + deleteMedia.mediaToDelete().stream() + .map(media -> new BackupManager.StorageDescriptor(media.cdn(), media.mediaId)) + .toList())) + .thenApply(Util.ASYNC_EMPTY_RESPONSE); + } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java index 7f6335289..4a5540925 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java @@ -8,6 +8,7 @@ package org.whispersystems.textsecuregcm.backup; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -20,12 +21,14 @@ import static org.mockito.Mockito.when; import io.grpc.Status; import io.grpc.StatusRuntimeException; +import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collections; @@ -36,6 +39,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -102,7 +106,7 @@ public class BackupManagerTest { backupManager.createMessageBackupUploadDescriptor(backupUser).join(); verify(tusCredentialGenerator, times(1)) - .generateUpload(encodedBackupId, BackupManager.MESSAGE_BACKUP_NAME); + .generateUpload("%s/%s".formatted(encodedBackupId, BackupManager.MESSAGE_BACKUP_NAME)); final BackupManager.BackupInfo info = backupManager.backupInfo(backupUser).join(); assertThat(info.backupSubdir()).isEqualTo(encodedBackupId); @@ -260,7 +264,7 @@ public class BackupManagerTest { @Test public void copySuccess() { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); - when(tusCredentialGenerator.generateUpload(any(), any())) + when(tusCredentialGenerator.generateUpload(any())) .thenReturn(new MessageBackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any())) .thenReturn(CompletableFuture.completedFuture(null)); @@ -286,7 +290,7 @@ public class BackupManagerTest { @Test public void copyFailure() { final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); - when(tusCredentialGenerator.generateUpload(any(), any())) + when(tusCredentialGenerator.generateUpload(any())) .thenReturn(new MessageBackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any())) .thenReturn(CompletableFuture.failedFuture(new SourceObjectNotFoundException())); @@ -409,6 +413,91 @@ public class BackupManagerTest { } + @Test + public void delete() { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + final byte[] mediaId = TestRandomUtil.nextBytes(16); + final String backupMediaKey = "%s/%s/%s".formatted( + BackupManager.encodeBackupIdForCdn(backupUser), + BackupManager.MEDIA_DIRECTORY_NAME, + BackupManager.encodeForCdn(mediaId)); + + backupsDb.setMediaUsage(backupUser, new UsageInfo(100, 1000)).join(); + + when(remoteStorageManager.delete(backupMediaKey)) + .thenReturn(CompletableFuture.completedFuture(7L)); + when(remoteStorageManager.cdnNumber()).thenReturn(5); + backupManager.delete(backupUser, List.of(new BackupManager.StorageDescriptor(5, mediaId))).toCompletableFuture() + .join(); + + assertThat(backupsDb.getMediaUsage(backupUser).join().usageInfo()) + .isEqualTo(new UsageInfo(93, 999)); + } + + @Test + public void deleteUnknownCdn() { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + when(remoteStorageManager.cdnNumber()).thenReturn(5); + assertThatThrownBy(() -> + backupManager.delete( backupUser, List.of(new BackupManager.StorageDescriptor(4, TestRandomUtil.nextBytes(15))))) + .isInstanceOf(StatusRuntimeException.class) + .matches(e -> ((StatusRuntimeException) e).getStatus().getCode() == Status.INVALID_ARGUMENT.getCode()); + } + + @Test + public void deletePartialFailure() { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + + final List descriptors = new ArrayList<>(); + long initialBytes = 0; + for (int i = 1; i <= 10; i++) { + final BackupManager.StorageDescriptor descriptor = new BackupManager.StorageDescriptor(5, + TestRandomUtil.nextBytes(15)); + descriptors.add(descriptor); + final String backupMediaKey = "%s/%s/%s".formatted( + BackupManager.encodeBackupIdForCdn(backupUser), + BackupManager.MEDIA_DIRECTORY_NAME, + BackupManager.encodeForCdn(descriptor.key())); + + initialBytes += i; + // fail 2 deletions, otherwise return the corresponding object's size as i + final CompletableFuture deleteResult = + i == 3 || i == 6 + ? CompletableFuture.failedFuture(new IOException("oh no")) + : CompletableFuture.completedFuture(Long.valueOf(i)); + + when(remoteStorageManager.delete(backupMediaKey)).thenReturn(deleteResult); + } + when(remoteStorageManager.cdnNumber()).thenReturn(5); + backupsDb.setMediaUsage(backupUser, new UsageInfo(initialBytes, 10)).join(); + CompletableFutureTestUtil.assertFailsWithCause(IOException.class, backupManager.delete(backupUser, descriptors)); + // 2 objects should have failed to be deleted + assertThat(backupsDb.getMediaUsage(backupUser).join().usageInfo()) + .isEqualTo(new UsageInfo(9, 2)); + + } + + @Test + public void alreadyDeleted() { + final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); + final byte[] mediaId = TestRandomUtil.nextBytes(16); + final String backupMediaKey = "%s/%s/%s".formatted( + BackupManager.encodeBackupIdForCdn(backupUser), + BackupManager.MEDIA_DIRECTORY_NAME, + BackupManager.encodeForCdn(mediaId)); + + backupsDb.setMediaUsage(backupUser, new UsageInfo(100, 5)).join(); + + // Deletion doesn't remove anything + when(remoteStorageManager.delete(backupMediaKey)).thenReturn(CompletableFuture.completedFuture(0L)); + when(remoteStorageManager.cdnNumber()).thenReturn(5); + backupManager.delete(backupUser, List.of(new BackupManager.StorageDescriptor(5, mediaId))).toCompletableFuture() + .join(); + + assertThat(backupsDb.getMediaUsage(backupUser).join().usageInfo()) + .isEqualTo(new UsageInfo(100, 5)); + } + private Map getBackupItem(final AuthenticatedBackupUser backupUser) { return DYNAMO_DB_EXTENSION.getDynamoDbClient().getItem(GetItemRequest.builder() .tableName(DynamoDbExtensionSchema.Tables.BACKUPS.tableName()) diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java index 0b7c82d9a..02d226da8 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupsDbTest.java @@ -50,14 +50,14 @@ public class BackupsDbTest { backupsDb.addMessageBackup(backupUser).join(); int total = 0; for (int i = 0; i < 5; i++) { - this.backupsDb.trackMedia(backupUser, i).join(); + this.backupsDb.trackMedia(backupUser, 1, i).join(); total += i; final BackupsDb.BackupDescription description = this.backupsDb.describeBackup(backupUser).join(); assertThat(description.mediaUsedSpace().get()).isEqualTo(total); } for (int i = 0; i < 5; i++) { - this.backupsDb.trackMedia(backupUser, -i).join(); + this.backupsDb.trackMedia(backupUser, -1, -i).join(); total -= i; final BackupsDb.BackupDescription description = this.backupsDb.describeBackup(backupUser).join(); assertThat(description.mediaUsedSpace().get()).isEqualTo(total); @@ -70,7 +70,7 @@ public class BackupsDbTest { testClock.pin(Instant.ofEpochSecond(5)); final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupTier.MEDIA); if (mediaAlreadyExists) { - this.backupsDb.trackMedia(backupUser, 10).join(); + this.backupsDb.trackMedia(backupUser, 1, 10).join(); } backupsDb.setMediaUsage(backupUser, new UsageInfo( 113, 17)).join(); final BackupsDb.TimestampedUsageInfo info = backupsDb.getMediaUsage(backupUser).join(); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java index 7863021fc..3bd01a210 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3BackupCredentialGeneratorTest.java @@ -22,7 +22,7 @@ public class Cdn3BackupCredentialGeneratorTest { new SecretBytes(TestRandomUtil.nextBytes(32)), "https://example.org/upload")); - final MessageBackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir", "key"); + final MessageBackupUploadDescriptor messageBackupUploadDescriptor = generator.generateUpload("subdir/key"); assertThat(messageBackupUploadDescriptor.signedUploadLocation()).isEqualTo("https://example.org/upload/backups"); assertThat(messageBackupUploadDescriptor.key()).isEqualTo("subdir/key"); assertThat(messageBackupUploadDescriptor.headers()).containsKey("Authorization"); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java index a39f72fbf..d07a4a22c 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java @@ -12,6 +12,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import com.fasterxml.jackson.core.JsonProcessingException; +import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import java.io.IOException; @@ -234,6 +235,8 @@ public class Cdn3RemoteStorageManagerTest { public void usage() throws JsonProcessingException { wireMock.stubFor(get(urlPathEqualTo("/storage-manager/usage")) .withQueryParam("prefix", equalTo("abc/")) + .withHeader(Cdn3RemoteStorageManager.CLIENT_ID_HEADER, equalTo("clientId")) + .withHeader(Cdn3RemoteStorageManager.CLIENT_SECRET_HEADER, equalTo("clientSecret")) .willReturn(aResponse() .withBody(SystemMapper.jsonMapper().writeValueAsString(new Cdn3RemoteStorageManager.UsageResponse( 17, @@ -244,4 +247,15 @@ public class Cdn3RemoteStorageManagerTest { assertThat(result.numObjects()).isEqualTo(17); assertThat(result.bytesUsed()).isEqualTo(113); } + + @Test + public void delete() throws JsonProcessingException { + wireMock.stubFor(WireMock.delete(urlEqualTo("/storage-manager/backups/abc/def")) + .withHeader(Cdn3RemoteStorageManager.CLIENT_ID_HEADER, equalTo("clientId")) + .withHeader(Cdn3RemoteStorageManager.CLIENT_SECRET_HEADER, equalTo("clientSecret")) + .willReturn(aResponse() + .withBody(SystemMapper.jsonMapper().writeValueAsString(new Cdn3RemoteStorageManager.DeleteResponse(9L))))); + final long deleted = remoteStorageManager.delete("abc/def").toCompletableFuture().join(); + assertThat(deleted).isEqualTo(9L); + } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java index 3b5f9b545..663bfe472 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java @@ -478,4 +478,29 @@ public class ArchiveControllerTest { assertThat(response.storedMediaObjects().get(0).mediaId()).isEqualTo(mediaId); assertThat(response.cursor()).isEqualTo(returnedCursor.orElse(null)); } + + @Test + public void delete() throws VerificationFailedException { + final BackupAuthCredentialPresentation presentation = backupAuthTestUtil.getPresentation(BackupTier.MEDIA, + backupKey, aci); + when(backupManager.authenticateBackupUser(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + new AuthenticatedBackupUser(presentation.getBackupId(), BackupTier.MEDIA))); + + final ArchiveController.DeleteMedia deleteRequest = new ArchiveController.DeleteMedia( + IntStream + .range(0, 100) + .mapToObj(i -> new ArchiveController.DeleteMedia.MediaToDelete(3, TestRandomUtil.nextBytes(15))) + .toList()); + + when(backupManager.delete(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + + final Response response = resources.getJerseyTest() + .target("v1/archives/media/delete") + .request() + .header("X-Signal-ZK-Auth", Base64.getEncoder().encodeToString(presentation.serialize())) + .header("X-Signal-ZK-Auth-Signature", "aaa") + .post(Entity.json(deleteRequest)); + assertThat(response.getStatus()).isEqualTo(204); + } }