MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use storage-manager's copy implementation

This commit is contained in:
Ravi Khadiwala 2024-05-02 17:10:29 -05:00 committed by ravi-signal
parent 843151859d
commit fc097db2a0
14 changed files with 167 additions and 536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
service/config/sample.yml
View File

@ -254,37 +254,13 @@ cdn:
secretAccessKey: secret://cdn.accessSecret secretAccessKey: secret://cdn.accessSecret
region: us-west-2 # AWS region region: us-west-2 # AWS region
clientCdn:
attachmentUrls:
2: https://cdn2.example.com/attachments/
caCertificates:
- |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
cdn3StorageManager: cdn3StorageManager:
baseUri: https://storage-manager.example.com baseUri: https://storage-manager.example.com
clientId: example clientId: example
clientSecret: secret://cdn3StorageManager.clientSecret clientSecret: secret://cdn3StorageManager.clientSecret
sourceSchemes:
2: gcs
3: r2
dogstatsd: dogstatsd:
environment: dev environment: dev

9
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java
View File

@ -112,11 +112,6 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty @JsonProperty
private CdnConfiguration cdn; private CdnConfiguration cdn;
@NotNull
@Valid
@JsonProperty
private ClientCdnConfiguration clientCdn;
@NotNull @NotNull
@Valid @Valid
@JsonProperty @JsonProperty
@ -450,10 +445,6 @@ public class WhisperServerConfiguration extends Configuration {
return cdn; return cdn;
} }
public ClientCdnConfiguration getClientCdnConfiguration() {
return clientCdn;
}
public Cdn3StorageManagerConfiguration getCdn3StorageManagerConfiguration() { public Cdn3StorageManagerConfiguration getCdn3StorageManagerConfiguration() {
return cdn3StorageManager; return cdn3StorageManager;
} }

8
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
View File

@ -703,13 +703,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
rateLimiters, rateLimiters,
tusAttachmentGenerator, tusAttachmentGenerator,
cdn3BackupCredentialGenerator, cdn3BackupCredentialGenerator,
new Cdn3RemoteStorageManager( new Cdn3RemoteStorageManager(remoteStorageExecutor, config.getCdn3StorageManagerConfiguration()),
remoteStorageExecutor,
config.getClientCdnConfiguration().getCircuitBreaker(),
config.getClientCdnConfiguration().getRetry(),
config.getClientCdnConfiguration().getCaCertificates(),
config.getCdn3StorageManagerConfiguration()),
config.getClientCdnConfiguration().getAttachmentUrls(),
clock); clock);
final DynamicConfigTurnRouter configTurnRouter = new DynamicConfigTurnRouter(dynamicConfigurationManager); final DynamicConfigTurnRouter configTurnRouter = new DynamicConfigTurnRouter(dynamicConfigurationManager);

46
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java
View File

@ -9,8 +9,7 @@ import com.google.common.annotations.VisibleForTesting;
import io.grpc.Status; import io.grpc.Status;
import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Metrics;
import java.io.IOException; import io.micrometer.core.instrument.Timer;
import java.net.URI;
import java.security.SecureRandom; import java.security.SecureRandom;
import java.time.Clock; import java.time.Clock;
import java.time.Duration; import java.time.Duration;
@ -22,9 +21,6 @@ import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage; import java.util.concurrent.CompletionStage;
import java.util.stream.Collectors;
import io.micrometer.core.instrument.Timer;
import org.apache.commons.lang3.StringUtils;
import org.signal.libsignal.protocol.ecc.Curve; import org.signal.libsignal.protocol.ecc.Curve;
import org.signal.libsignal.protocol.ecc.ECPublicKey; import org.signal.libsignal.protocol.ecc.ECPublicKey;
import org.signal.libsignal.zkgroup.GenericServerSecretParams; import org.signal.libsignal.zkgroup.GenericServerSecretParams;
@ -80,7 +76,6 @@ public class BackupManager {
private final TusAttachmentGenerator tusAttachmentGenerator; private final TusAttachmentGenerator tusAttachmentGenerator;
private final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator; private final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator;
private final RemoteStorageManager remoteStorageManager; private final RemoteStorageManager remoteStorageManager;
private final Map<Integer, String> attachmentCdnBaseUris;
private final SecureRandom secureRandom = new SecureRandom(); private final SecureRandom secureRandom = new SecureRandom();
private final Clock clock; private final Clock clock;
@ -92,7 +87,6 @@ public class BackupManager {
final TusAttachmentGenerator tusAttachmentGenerator, final TusAttachmentGenerator tusAttachmentGenerator,
final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator, final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator,
final RemoteStorageManager remoteStorageManager, final RemoteStorageManager remoteStorageManager,
final Map<Integer, String> attachmentCdnBaseUris,
final Clock clock) { final Clock clock) {
this.backupsDb = backupsDb; this.backupsDb = backupsDb;
this.serverSecretParams = serverSecretParams; this.serverSecretParams = serverSecretParams;
@ -101,11 +95,6 @@ public class BackupManager {
this.cdn3BackupCredentialGenerator = cdn3BackupCredentialGenerator; this.cdn3BackupCredentialGenerator = cdn3BackupCredentialGenerator;
this.remoteStorageManager = remoteStorageManager; this.remoteStorageManager = remoteStorageManager;
this.clock = clock; this.clock = clock;
// strip trailing "/" for easier URI construction
this.attachmentCdnBaseUris = attachmentCdnBaseUris.entrySet().stream().collect(Collectors.toMap(
Map.Entry::getKey,
entry -> StringUtils.removeEnd(entry.getValue(), "/")
));
} }
@ -271,23 +260,15 @@ public class BackupManager {
.asRuntimeException(); .asRuntimeException();
} }
final URI sourceUri; final String destination = cdnMediaPath(backupUser, destinationMediaId);
try {
sourceUri = attachmentReadUri(sourceCdn, sourceKey);
} catch (IOException e) {
return CompletableFuture.failedFuture(e);
}
final BackupUploadDescriptor dst = cdn3BackupCredentialGenerator.generateUpload(
cdnMediaPath(backupUser, destinationMediaId));
final int destinationLength = encryptionParameters.outputSize(sourceLength); final int destinationLength = encryptionParameters.outputSize(sourceLength);
return this.backupsDb return this.backupsDb
// Write the ddb updates before actually updating backing storage // Write the ddb updates before actually updating backing storage
.trackMedia(backupUser, 1, destinationLength) .trackMedia(backupUser, 1, destinationLength)
// Actually copy the objects. If the copy fails, our estimated quota usage may not be exact // Actually copy the objects. If the copy fails, our estimated quota usage may not be exact
.thenComposeAsync(ignored -> remoteStorageManager.copy(sourceUri, sourceLength, encryptionParameters, dst)) .thenComposeAsync(ignored ->
remoteStorageManager.copy(sourceCdn, sourceKey, sourceLength, encryptionParameters, destination))
.exceptionallyCompose(throwable -> { .exceptionallyCompose(throwable -> {
final Throwable unwrapped = ExceptionUtils.unwrap(throwable); final Throwable unwrapped = ExceptionUtils.unwrap(throwable);
if (!(unwrapped instanceof SourceObjectNotFoundException) && !(unwrapped instanceof InvalidLengthException)) { if (!(unwrapped instanceof SourceObjectNotFoundException) && !(unwrapped instanceof InvalidLengthException)) {
@ -299,25 +280,10 @@ public class BackupManager {
}); });
}) })
// indicates where the backup was stored // indicates where the backup was stored
.thenApply(ignore -> new StorageDescriptor(dst.cdn(), destinationMediaId)); .thenApply(ignore -> new StorageDescriptor(remoteStorageManager.cdnNumber(), destinationMediaId));
} }
/**
* Construct the URI for an attachment with the specified key
*
* @param cdn where the attachment is located
* @param key the attachment key
* @return A {@link URI} where the attachment can be retrieved
*/
private URI attachmentReadUri(final int cdn, final String key) throws IOException {
final String baseUri = attachmentCdnBaseUris.get(cdn);
if (baseUri == null) {
throw new SourceObjectNotFoundException("Unknown attachment cdn " + cdn);
}
return URI.create("%s/%s".formatted(baseUri, key));
}
/** /**
* Generate credentials that can be used to read from the backup CDN * Generate credentials that can be used to read from the backup CDN
* *
@ -377,7 +343,7 @@ public class BackupManager {
// Try to swap out the backupDir for the user // Try to swap out the backupDir for the user
.scheduleBackupDeletion(backupUser) .scheduleBackupDeletion(backupUser)
// If there was already a pending swap, try to delete the cdn objects directly // If there was already a pending swap, try to delete the cdn objects directly
.exceptionallyCompose(ExceptionUtils.exceptionallyHandler(BackupsDb.PendingDeletionException.class,e -> .exceptionallyCompose(ExceptionUtils.exceptionallyHandler(BackupsDb.PendingDeletionException.class, e ->
AsyncTimerUtil.record(SYNCHRONOUS_DELETE_TIMER, () -> AsyncTimerUtil.record(SYNCHRONOUS_DELETE_TIMER, () ->
deletePrefix(backupUser.backupDir(), DELETION_CONCURRENCY)))); deletePrefix(backupUser.backupDir(), DELETION_CONCURRENCY))));
} }

102
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypter.java
View File

@ -1,102 +0,0 @@
package org.whispersystems.textsecuregcm.backup;
import java.nio.ByteBuffer;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.concurrent.Flow;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.Mac;
import javax.crypto.NoSuchPaddingException;
import org.reactivestreams.FlowAdapters;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class BackupMediaEncrypter {
private final Cipher cipher;
private final Mac mac;
public BackupMediaEncrypter(final MediaEncryptionParameters encryptionParameters) {
cipher = initializeCipher(encryptionParameters);
mac = initializeMac(encryptionParameters);
}
public int outputSize(final int inputSize) {
return cipher.getIV().length + cipher.getOutputSize(inputSize) + mac.getMacLength();
}
/**
* Perform streaming encryption
*
* @param sourceBody A source of ByteBuffers, typically from an asynchronous HttpResponse
* @return A publisher that returns IV + AES/CBC/PKCS5Padding encrypted source + HMAC(IV + encrypted source) suitable
* to write with an asynchronous HttpRequest
*/
public Flow.Publisher<ByteBuffer> encryptBody(Flow.Publisher<List<ByteBuffer>> sourceBody) {
// Write IV, encrypted payload, mac
final Flux<ByteBuffer> encryptedBody = Flux.concat(
Mono.fromSupplier(() -> {
mac.update(cipher.getIV());
return ByteBuffer.wrap(cipher.getIV());
}),
Flux.from(FlowAdapters.toPublisher(sourceBody))
.concatMap(Flux::fromIterable)
.concatMap(byteBuffer -> {
final byte[] copy = new byte[byteBuffer.remaining()];
byteBuffer.get(copy);
final byte[] res = cipher.update(copy);
if (res == null) {
return Mono.empty();
} else {
mac.update(res);
return Mono.just(ByteBuffer.wrap(res));
}
}),
Mono.fromSupplier(() -> {
try {
final byte[] finalBytes = cipher.doFinal();
mac.update(finalBytes);
return ByteBuffer.wrap(finalBytes);
} catch (IllegalBlockSizeException | BadPaddingException e) {
throw ExceptionUtils.wrap(e);
}
}),
Mono.fromSupplier(() -> ByteBuffer.wrap(mac.doFinal())));
return FlowAdapters.toFlowPublisher(encryptedBody);
}
private static Mac initializeMac(final MediaEncryptionParameters encryptionParameters) {
try {
final Mac mac = Mac.getInstance("HmacSHA256");
mac.init(encryptionParameters.hmacSHA256Key());
return mac;
} catch (NoSuchAlgorithmException e) {
throw new AssertionError(e);
} catch (InvalidKeyException e) {
throw new IllegalArgumentException(e);
}
}
private static Cipher initializeCipher(final MediaEncryptionParameters encryptionParameters) {
try {
final Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(
Cipher.ENCRYPT_MODE,
encryptionParameters.aesEncryptionKey(),
encryptionParameters.iv());
return cipher;
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
throw new AssertionError(e);
} catch (InvalidAlgorithmParameterException | InvalidKeyException e) {
throw new IllegalArgumentException(e);
}
}
}

177
service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java
View File

@ -1,5 +1,6 @@
package org.whispersystems.textsecuregcm.backup; package org.whispersystems.textsecuregcm.backup;
import com.fasterxml.jackson.core.JsonProcessingException;
import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.Timer;
import java.io.IOException; import java.io.IOException;
@ -8,30 +9,24 @@ import java.net.URI;
import java.net.http.HttpClient; import java.net.http.HttpClient;
import java.net.http.HttpRequest; import java.net.http.HttpRequest;
import java.net.http.HttpResponse; import java.net.http.HttpResponse;
import java.nio.ByteBuffer;
import java.security.cert.CertificateException;
import java.time.Duration; import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletionException; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage; import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Flow;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.stream.Stream;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import javax.validation.constraints.NotNull; import javax.validation.constraints.NotNull;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration; import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient; import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil; import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.util.ExceptionUtils; import org.whispersystems.textsecuregcm.util.ExceptionUtils;
@ -42,20 +37,15 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
private static final Logger logger = LoggerFactory.getLogger(Cdn3RemoteStorageManager.class); private static final Logger logger = LoggerFactory.getLogger(Cdn3RemoteStorageManager.class);
private final FaultTolerantHttpClient cdnHttpClient;
private final FaultTolerantHttpClient storageManagerHttpClient; private final FaultTolerantHttpClient storageManagerHttpClient;
private final String storageManagerBaseUrl; private final String storageManagerBaseUrl;
private final String clientId; private final String clientId;
private final String clientSecret; private final String clientSecret;
private final Map<Integer, String> sourceSchemes;
static final String CLIENT_ID_HEADER = "CF-Access-Client-Id"; static final String CLIENT_ID_HEADER = "CF-Access-Client-Id";
static final String CLIENT_SECRET_HEADER = "CF-Access-Client-Secret"; static final String CLIENT_SECRET_HEADER = "CF-Access-Client-Secret";
private static String TUS_UPLOAD_LENGTH_HEADER = "Upload-Length";
private static String TUS_UPLOAD_OFFSET_HEADER = "Upload-Offset";
private static String TUS_VERSION_HEADER = "Tus-Resumable";
private static String TUS_VERSION = "1.0.0";
private static String TUS_CONTENT_TYPE = "application/offset+octet-stream";
private static final String STORAGE_MANAGER_STATUS_COUNTER_NAME = MetricsUtil.name(Cdn3RemoteStorageManager.class, private static final String STORAGE_MANAGER_STATUS_COUNTER_NAME = MetricsUtil.name(Cdn3RemoteStorageManager.class,
"storageManagerStatus"); "storageManagerStatus");
@ -66,41 +56,25 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
public Cdn3RemoteStorageManager( public Cdn3RemoteStorageManager(
final ScheduledExecutorService retryExecutor, final ScheduledExecutorService retryExecutor,
final CircuitBreakerConfiguration circuitBreakerConfiguration, final Cdn3StorageManagerConfiguration configuration) {
final RetryConfiguration retryConfiguration,
final List<String> cdnCaCertificates,
final Cdn3StorageManagerConfiguration configuration) throws CertificateException {
// strip trailing "/" for easier URI construction // strip trailing "/" for easier URI construction
this.storageManagerBaseUrl = StringUtils.removeEnd(configuration.baseUri(), "/"); this.storageManagerBaseUrl = StringUtils.removeEnd(configuration.baseUri(), "/");
this.clientId = configuration.clientId(); this.clientId = configuration.clientId();
this.clientSecret = configuration.clientSecret().value(); this.clientSecret = configuration.clientSecret().value();
// Client used to read/write to cdn
this.cdnHttpClient = FaultTolerantHttpClient.newBuilder()
.withName("cdn-client")
.withCircuitBreaker(circuitBreakerConfiguration)
.withExecutor(Executors.newCachedThreadPool())
.withRetryExecutor(retryExecutor)
.withRetry(retryConfiguration)
.withConnectTimeout(Duration.ofSeconds(10))
.withVersion(HttpClient.Version.HTTP_2)
.withTrustedServerCertificates(cdnCaCertificates.toArray(new String[0]))
.withNumClients(configuration.numHttpClients())
.build();
// Client used for calls to storage-manager // Client used for calls to storage-manager
// storage-manager has an external CA so uses a different client
this.storageManagerHttpClient = FaultTolerantHttpClient.newBuilder() this.storageManagerHttpClient = FaultTolerantHttpClient.newBuilder()
.withName("cdn3-storage-manager") .withName("cdn3-storage-manager")
.withCircuitBreaker(circuitBreakerConfiguration) .withCircuitBreaker(configuration.circuitBreaker())
.withExecutor(Executors.newCachedThreadPool()) .withExecutor(Executors.newCachedThreadPool())
.withRetryExecutor(retryExecutor) .withRetryExecutor(retryExecutor)
.withRetry(retryConfiguration) .withRetry(configuration.retry())
.withConnectTimeout(Duration.ofSeconds(10)) .withConnectTimeout(Duration.ofSeconds(10))
.withVersion(HttpClient.Version.HTTP_2) .withVersion(HttpClient.Version.HTTP_2)
.withNumClients(configuration.numHttpClients()) .withNumClients(configuration.numHttpClients())
.build(); .build();
this.sourceSchemes = configuration.sourceSchemes();
} }
@Override @Override
@ -110,85 +84,70 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
@Override @Override
public CompletionStage<Void> copy( public CompletionStage<Void> copy(
final URI sourceUri, final int sourceCdn,
final String sourceKey,
final int expectedSourceLength, final int expectedSourceLength,
final MediaEncryptionParameters encryptionParameters, final MediaEncryptionParameters encryptionParameters,
final BackupUploadDescriptor uploadDescriptor) { final String destinationKey) {
final String sourceScheme = this.sourceSchemes.get(sourceCdn);
if (uploadDescriptor.cdn() != cdnNumber()) { if (sourceScheme == null) {
throw new IllegalArgumentException("Cdn3RemoteStorageManager can only copy to cdn3"); return CompletableFuture.failedFuture(
new SourceObjectNotFoundException("Cdn3RemoteStorageManager cannot copy from " + sourceCdn));
} }
final String requestBody = new Cdn3CopyRequest(
encryptionParameters,
new Cdn3CopyRequest.SourceDescriptor(sourceScheme, sourceKey),
expectedSourceLength,
destinationKey).json();
final Timer.Sample sample = Timer.start(); final Timer.Sample sample = Timer.start();
final BackupMediaEncrypter encrypter = new BackupMediaEncrypter(encryptionParameters); final HttpRequest request = HttpRequest.newBuilder()
final HttpRequest request = HttpRequest.newBuilder().GET().uri(sourceUri).build(); .PUT(HttpRequest.BodyPublishers.ofString(requestBody))
return cdnHttpClient.sendAsync(request, HttpResponse.BodyHandlers.ofPublisher()).thenCompose(response -> { .uri(URI.create(copyUrl()))
try { .header("Content-Type", "application/json")
return cdnHttpClient.sendAsync( .header(CLIENT_ID_HEADER, clientId)
createCopyRequest(expectedSourceLength, uploadDescriptor, encrypter, response), .header(CLIENT_SECRET_HEADER, clientSecret)
HttpResponse.BodyHandlers.discarding()); .build();
} catch (Exception e) { return this.storageManagerHttpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
// Discard the response body so we don't hold the http2 stream open
response.body().subscribe(CancelSubscriber.INSTANCE);
throw ExceptionUtils.wrap(e);
}
})
.thenAccept(response -> { .thenAccept(response -> {
if (response.statusCode() != Response.Status.CREATED.getStatusCode() && if (response.statusCode() == Response.Status.NOT_FOUND.getStatusCode()) {
response.statusCode() != Response.Status.OK.getStatusCode()) { throw ExceptionUtils.wrap(new SourceObjectNotFoundException());
throw new CompletionException(new IOException("Failed to copy object: " + response.statusCode())); } else if (response.statusCode() == Response.Status.CONFLICT.getStatusCode()) {
} throw ExceptionUtils.wrap(new InvalidLengthException(response.body()));
long uploadOffset = response.headers().firstValueAsLong(TUS_UPLOAD_OFFSET_HEADER) } else if (!HttpUtils.isSuccessfulResponse(response.statusCode())) {
.orElseThrow(() -> new CompletionException(new IOException("Tus server did not return Upload-Offset"))); logger.info("Failed to copy via storage-manager {} {}", response.statusCode(), response.body());
final int expectedEncryptedLength = encrypter.outputSize(expectedSourceLength); throw ExceptionUtils.wrap(new IOException("Failed to copy object: " + response.statusCode()));
if (uploadOffset != expectedEncryptedLength) {
throw new CompletionException(new IOException(
"Expected to upload %d bytes, uploaded %d".formatted(expectedEncryptedLength, uploadOffset)));
} }
}) })
.whenComplete((ignored, ignoredException) -> .whenComplete((ignored, ignoredException) ->
sample.stop(Metrics.timer(STORAGE_MANAGER_TIMER_NAME, OPERATION_TAG_NAME, "copy"))); sample.stop(Metrics.timer(STORAGE_MANAGER_TIMER_NAME, OPERATION_TAG_NAME, "copy")));
} }
private HttpRequest createCopyRequest( /**
final int expectedSourceLength, * Serialized copy request for cdn3 storage manager
final BackupUploadDescriptor uploadDescriptor, */
BackupMediaEncrypter encrypter, record Cdn3CopyRequest(
HttpResponse<Flow.Publisher<List<ByteBuffer>>> response) throws IOException { String encryptionKey, String hmacKey, String iv,
if (response.statusCode() == Response.Status.NOT_FOUND.getStatusCode()) { SourceDescriptor source, int expectedSourceLength,
throw new SourceObjectNotFoundException(); String dst) {
} else if (response.statusCode() != Response.Status.OK.getStatusCode()) {
throw new IOException("error reading from source: " + response.statusCode()); Cdn3CopyRequest(MediaEncryptionParameters parameters, SourceDescriptor source, int expectedSourceLength,
String dst) {
this(Base64.getEncoder().encodeToString(parameters.aesEncryptionKey().getEncoded()),
Base64.getEncoder().encodeToString(parameters.hmacSHA256Key().getEncoded()),
Base64.getEncoder().encodeToString(parameters.iv().getIV()),
source, expectedSourceLength, dst);
} }
final int actualSourceLength = Math.toIntExact(response.headers().firstValueAsLong("Content-Length") record SourceDescriptor(String scheme, String key) {}
.orElseThrow(() -> new IOException("upstream missing Content-Length")));
if (actualSourceLength != expectedSourceLength) { String json() {
throw new InvalidLengthException( try {
"Provided sourceLength " + expectedSourceLength + " was " + actualSourceLength); return SystemMapper.jsonMapper().writeValueAsString(this);
} catch (JsonProcessingException e) {
throw new IllegalStateException("Could not serialize copy request", e);
}
} }
final int expectedEncryptedLength = encrypter.outputSize(expectedSourceLength);
final HttpRequest.BodyPublisher encryptedBody = HttpRequest.BodyPublishers.fromPublisher(
encrypter.encryptBody(response.body()), expectedEncryptedLength);
final String[] headers = Stream.concat(
uploadDescriptor.headers().entrySet()
.stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue())),
Stream.of(
TUS_VERSION_HEADER, TUS_VERSION,
TUS_UPLOAD_LENGTH_HEADER, Integer.toString(expectedEncryptedLength),
HttpHeaders.CONTENT_TYPE, TUS_CONTENT_TYPE))
.toArray(String[]::new);
return HttpRequest.newBuilder()
.uri(URI.create(uploadDescriptor.signedUploadLocation()))
.headers(headers)
.POST(encryptedBody)
.build();
} }
@Override @Override
@ -338,25 +297,7 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
return "%s/%s/".formatted(storageManagerBaseUrl, Cdn3BackupCredentialGenerator.CDN_PATH); return "%s/%s/".formatted(storageManagerBaseUrl, Cdn3BackupCredentialGenerator.CDN_PATH);
} }
private static class CancelSubscriber implements Flow.Subscriber<List<ByteBuffer>> { private String copyUrl() {
return "%s/copy".formatted(storageManagerBaseUrl);
private static CancelSubscriber INSTANCE = new CancelSubscriber();
@Override
public void onSubscribe(final Flow.Subscription subscription) {
subscription.cancel();
}
@Override
public void onNext(final List<ByteBuffer> item) {
}
@Override
public void onError(final Throwable throwable) {
}
@Override
public void onComplete() {
}
} }
} }

11
service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java
View File

@ -1,6 +1,5 @@
package org.whispersystems.textsecuregcm.backup; package org.whispersystems.textsecuregcm.backup;
import java.net.URI;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletionStage; import java.util.concurrent.CompletionStage;
@ -18,12 +17,13 @@ public interface RemoteStorageManager {
/** /**
* Copy and the object from a remote source into the backup, adding an additional layer of encryption * Copy and the object from a remote source into the backup, adding an additional layer of encryption
* *
* @param sourceUri The location of the object to copy * @param sourceCdn The cdn number where the source attachment is stored
* @param sourceKey The key of the source attachment within the attachment cdn
* @param expectedSourceLength The length of the source object, should match the content-length of the object returned * @param expectedSourceLength The length of the source object, should match the content-length of the object returned
* from the sourceUri. * from the sourceUri.
* @param encryptionParameters The encryption keys that should be used to apply an additional layer of encryption to * @param encryptionParameters The encryption keys that should be used to apply an additional layer of encryption to
* the object * the object
* @param uploadDescriptor The destination, which must be in the cdn returned by {@link #cdnNumber()} * @param dstKey The key within the backup cdn where the copied object will be written
* @return A stage that completes successfully when the source has been successfully re-encrypted and copied into * @return A stage that completes successfully when the source has been successfully re-encrypted and copied into
* uploadDescriptor. The returned CompletionStage can be completed exceptionally with the following exceptions. * uploadDescriptor. The returned CompletionStage can be completed exceptionally with the following exceptions.
* <ul> * <ul>
@ -33,10 +33,11 @@ public interface RemoteStorageManager {
* </ul> * </ul>
*/ */
CompletionStage<Void> copy( CompletionStage<Void> copy(
URI sourceUri, int sourceCdn,
String sourceKey,
int expectedSourceLength, int expectedSourceLength,
MediaEncryptionParameters encryptionParameters, MediaEncryptionParameters encryptionParameters,
BackupUploadDescriptor uploadDescriptor); String dstKey);
/** /**
* Result of a {@link #list} operation * Result of a {@link #list} operation

31
service/src/main/java/org/whispersystems/textsecuregcm/configuration/Cdn3StorageManagerConfiguration.java
View File

@ -1,17 +1,44 @@
package org.whispersystems.textsecuregcm.configuration; package org.whispersystems.textsecuregcm.configuration;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretString; import java.util.Collections;
import java.util.Map;
import javax.validation.constraints.NotNull; import javax.validation.constraints.NotNull;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
/**
* Configuration for the cdn3 storage manager
*
* @param baseUri The base URI of the storage manager
* @param clientId The cloudflare client ID to use to authenticate to the storage manager
* @param clientSecret The cloudflare client secret to use to authenticate to the storage manager
* @param sourceSchemes A map of cdn id to a retrieval scheme understood by the storage-manager. This is used by the
* storage-manager when copying to determine how to read a source object. Current schemes are
* 'gcs' and 'r2'
* @param numHttpClients The number http clients to use with the storage-manager to support request striping
* @param circuitBreaker A circuit breaker configuration for the storage-manager http client
* @param retry A retry configuration for the storage-manager http client
*/
public record Cdn3StorageManagerConfiguration( public record Cdn3StorageManagerConfiguration(
@NotNull String baseUri, @NotNull String baseUri,
@NotNull String clientId, @NotNull String clientId,
@NotNull SecretString clientSecret, @NotNull SecretString clientSecret,
@NotNull Integer numHttpClients) { @NotNull Map<Integer, String> sourceSchemes,
@NotNull Integer numHttpClients,
@NotNull CircuitBreakerConfiguration circuitBreaker,
@NotNull RetryConfiguration retry) {
public Cdn3StorageManagerConfiguration { public Cdn3StorageManagerConfiguration {
if (numHttpClients == null) { if (numHttpClients == null) {
numHttpClients = 2; numHttpClients = 2;
} }
if (sourceSchemes == null) {
sourceSchemes = Collections.emptyMap();
}
if (circuitBreaker == null) {
circuitBreaker = new CircuitBreakerConfiguration();
}
if (retry == null) {
retry = new RetryConfiguration();
}
} }
} }

22
service/src/main/java/org/whispersystems/textsecuregcm/configuration/ClientCdnConfiguration.java
View File

@ -13,20 +13,6 @@ import java.util.Map;
*/ */
public class ClientCdnConfiguration { public class ClientCdnConfiguration {
/**
* Map from cdn number to the base url for attachments.
* <p>
* For example, if an attachment with the id 'abc' can be retrieved from cdn 2 at https://example.org/attachments/abc,
* the attachment url for 2 should https://example.org/attachments
*/
@JsonProperty
@NotNull
Map<Integer, @NotBlank String> attachmentUrls;
@JsonProperty
@NotNull
@NotEmpty List<@NotBlank String> caCertificates = new ArrayList<>();
@JsonProperty @JsonProperty
@NotNull @NotNull
CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration(); CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
@ -35,10 +21,6 @@ public class ClientCdnConfiguration {
@NotNull @NotNull
RetryConfiguration retry = new RetryConfiguration(); RetryConfiguration retry = new RetryConfiguration();
public List<String> getCaCertificates() {
return caCertificates;
}
public CircuitBreakerConfiguration getCircuitBreaker() { public CircuitBreakerConfiguration getCircuitBreaker() {
return circuitBreaker; return circuitBreaker;
} }
@ -46,8 +28,4 @@ public class ClientCdnConfiguration {
public RetryConfiguration getRetry() { public RetryConfiguration getRetry() {
return retry; return retry;
} }
public Map<Integer, String> getAttachmentUrls() {
return attachmentUrls;
}
} }

8
service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java
View File

@ -211,13 +211,7 @@ record CommandDependencies(
rateLimiters, rateLimiters,
new TusAttachmentGenerator(configuration.getTus()), new TusAttachmentGenerator(configuration.getTus()),
new Cdn3BackupCredentialGenerator(configuration.getTus()), new Cdn3BackupCredentialGenerator(configuration.getTus()),
new Cdn3RemoteStorageManager( new Cdn3RemoteStorageManager(remoteStorageExecutor, configuration.getCdn3StorageManagerConfiguration()),
remoteStorageExecutor,
configuration.getClientCdnConfiguration().getCircuitBreaker(),
configuration.getClientCdnConfiguration().getRetry(),
configuration.getClientCdnConfiguration().getCaCertificates(),
configuration.getCdn3StorageManagerConfiguration()),
configuration.getClientCdnConfiguration().getAttachmentUrls(),
clock); clock);
environment.lifecycle().manage(messagesCache); environment.lifecycle().manage(messagesCache);

18
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java
View File

@ -102,6 +102,8 @@ public class BackupManagerTest {
final RateLimiters rateLimiters = mock(RateLimiters.class); final RateLimiters rateLimiters = mock(RateLimiters.class);
when(rateLimiters.forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT)).thenReturn(mediaUploadLimiter); when(rateLimiters.forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT)).thenReturn(mediaUploadLimiter);
when(remoteStorageManager.cdnNumber()).thenReturn(3);
this.backupsDb = new BackupsDb( this.backupsDb = new BackupsDb(
DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(), DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(),
DynamoDbExtensionSchema.Tables.BACKUPS.tableName(), DynamoDbExtensionSchema.Tables.BACKUPS.tableName(),
@ -113,7 +115,6 @@ public class BackupManagerTest {
tusAttachmentGenerator, tusAttachmentGenerator,
tusCredentialGenerator, tusCredentialGenerator,
remoteStorageManager, remoteStorageManager,
Map.of(3, "cdn3.example.org/attachments"),
testClock); testClock);
} }
@ -352,7 +353,7 @@ public class BackupManagerTest {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA); final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
when(tusCredentialGenerator.generateUpload(any())) when(tusCredentialGenerator.generateUpload(any()))
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); .thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any())) when(remoteStorageManager.copy(eq(3), eq("abc"), eq(100), any(), any()))
.thenReturn(CompletableFuture.completedFuture(null)); .thenReturn(CompletableFuture.completedFuture(null));
final MediaEncryptionParameters encryptionParams = new MediaEncryptionParameters( final MediaEncryptionParameters encryptionParams = new MediaEncryptionParameters(
TestRandomUtil.nextBytes(32), TestRandomUtil.nextBytes(32),
@ -378,7 +379,7 @@ public class BackupManagerTest {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA); final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
when(tusCredentialGenerator.generateUpload(any())) when(tusCredentialGenerator.generateUpload(any()))
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), "")); .thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any())) when(remoteStorageManager.copy(eq(3), eq("abc"), eq(100), any(), any()))
.thenReturn(CompletableFuture.failedFuture(new SourceObjectNotFoundException())); .thenReturn(CompletableFuture.failedFuture(new SourceObjectNotFoundException()));
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class, CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
@ -394,17 +395,6 @@ public class BackupManagerTest {
assertThat(AttributeValues.getLong(backup, BackupsDb.ATTR_MEDIA_COUNT, -1L)).isEqualTo(0L); assertThat(AttributeValues.getLong(backup, BackupsDb.ATTR_MEDIA_COUNT, -1L)).isEqualTo(0L);
} }
@Test
public void unknownSourceCdn() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
backupManager.copyToBackup(
backupUser,
0, "abc", 100,
mock(MediaEncryptionParameters.class),
"def".getBytes(StandardCharsets.UTF_8)));
}
@Test @Test
public void quotaEnforcementNoRecalculation() { public void quotaEnforcementNoRecalculation() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA); final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);

20
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypterTest.java
View File

@ -1,20 +0,0 @@
package org.whispersystems.textsecuregcm.backup;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.whispersystems.textsecuregcm.util.TestRandomUtil;
public class BackupMediaEncrypterTest {
@ParameterizedTest
@ValueSource(ints = {0, 1, 2, 15, 16, 17, 63, 64, 65, 1023, 1024, 1025})
public void sizeCalc() {
final MediaEncryptionParameters params = new MediaEncryptionParameters(
TestRandomUtil.nextBytes(32),
TestRandomUtil.nextBytes(32), TestRandomUtil.nextBytes(16));
final BackupMediaEncrypter encrypter = new BackupMediaEncrypter(params);
assertThat(params.outputSize(1)).isEqualTo(encrypter.outputSize(1));
}
}

181
service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java
View File

@ -3,37 +3,23 @@ package org.whispersystems.textsecuregcm.backup;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.put;
import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.assertj.core.api.Assertions.assertThatNoException;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.client.WireMock;
import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import javax.crypto.BadPaddingException; import javax.ws.rs.core.HttpHeaders;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.Mac;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
@ -60,150 +46,83 @@ public class Cdn3RemoteStorageManagerTest {
.options(wireMockConfig().dynamicPort()) .options(wireMockConfig().dynamicPort())
.build(); .build();
private static final String SMALL_CDN2 = "a small object from cdn2";
private static final String SMALL_CDN3 = "a small object from cdn3";
private static final String LARGE = "a".repeat(1024 * 1024 * 5);
private RemoteStorageManager remoteStorageManager; private RemoteStorageManager remoteStorageManager;
@BeforeEach @BeforeEach
public void init() throws CertificateException { public void init() {
remoteStorageManager = new Cdn3RemoteStorageManager( remoteStorageManager = new Cdn3RemoteStorageManager(
Executors.newSingleThreadScheduledExecutor(), Executors.newSingleThreadScheduledExecutor(),
new CircuitBreakerConfiguration(),
new RetryConfiguration(),
Collections.emptyList(),
new Cdn3StorageManagerConfiguration( new Cdn3StorageManagerConfiguration(
wireMock.url("storage-manager/"), wireMock.url("storage-manager/"),
"clientId", "clientId",
new SecretString("clientSecret"), new SecretString("clientSecret"),
2)); Map.of(2, "gcs", 3, "r2"),
2,
wireMock.stubFor(get(urlEqualTo("/cdn2/source/small")) new CircuitBreakerConfiguration(),
.willReturn(aResponse() new RetryConfiguration()));
.withHeader("Content-Length", Integer.toString(SMALL_CDN2.length()))
.withBody(SMALL_CDN2)));
wireMock.stubFor(get(urlEqualTo("/cdn3/source/small"))
.willReturn(aResponse()
.withHeader("Content-Length", Integer.toString(SMALL_CDN3.length()))
.withBody(SMALL_CDN3)));
wireMock.stubFor(get(urlEqualTo("/cdn3/source/large"))
.willReturn(aResponse()
.withHeader("Content-Length", Integer.toString(LARGE.length()))
.withBody(LARGE)));
wireMock.stubFor(get(urlEqualTo("/cdn3/source/missing"))
.willReturn(aResponse().withStatus(404)));
} }
@ParameterizedTest @ParameterizedTest
@ValueSource(ints = {2, 3}) @ValueSource(ints = {2, 3})
public void copySmall(final int sourceCdn) public void copy(final int sourceCdn) throws JsonProcessingException {
throws InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException { final MediaEncryptionParameters encryptionParameters = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
final String scheme = switch (sourceCdn) {
final String expectedSource = switch (sourceCdn) { case 2 -> "gcs";
case 2 -> SMALL_CDN2; case 3 -> "r2";
case 3 -> SMALL_CDN3;
default -> throw new AssertionError(); default -> throw new AssertionError();
}; };
final Cdn3RemoteStorageManager.Cdn3CopyRequest expectedCopyRequest = new Cdn3RemoteStorageManager.Cdn3CopyRequest(
final MediaEncryptionParameters encryptionParameters = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
final long expectedEncryptedLength = encryptionParameters.outputSize(expectedSource.length());
wireMock.stubFor(post(urlEqualTo("/cdn3/dest"))
.withHeader("Content-Length", equalTo(Long.toString(expectedEncryptedLength)))
.withHeader("Upload-Length", equalTo(Long.toString(expectedEncryptedLength)))
.withHeader("Content-Type", equalTo("application/offset+octet-stream"))
.willReturn(aResponse()
.withStatus(201)
.withHeader("Upload-Offset", Long.toString(expectedEncryptedLength))));
remoteStorageManager.copy(
URI.create(wireMock.url("/cdn" + sourceCdn + "/source/small")),
expectedSource.length(),
encryptionParameters, encryptionParameters,
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) new Cdn3RemoteStorageManager.Cdn3CopyRequest.SourceDescriptor(scheme, "a/test/source"),
.toCompletableFuture().join(); 100,
"a/destination");
final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).getFirst().getBody(); wireMock.stubFor(put(urlEqualTo("/storage-manager/copy"))
assertThat(new String(decrypt(destBody), StandardCharsets.UTF_8)) .withHeader(HttpHeaders.CONTENT_TYPE, equalTo("application/json"))
.isEqualTo(expectedSource); .withRequestBody(WireMock.equalToJson(SystemMapper.jsonMapper().writeValueAsString(expectedCopyRequest)))
} .willReturn(aResponse().withStatus(204)));
assertThatNoException().isThrownBy(() ->
@Test
public void copyLarge()
throws InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException {
final MediaEncryptionParameters params = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
final long expectedEncryptedLength = params.outputSize(LARGE.length());
wireMock.stubFor(post(urlEqualTo("/cdn3/dest"))
.withHeader("Content-Length", equalTo(Long.toString(expectedEncryptedLength)))
.withHeader("Upload-Length", equalTo(Long.toString(expectedEncryptedLength)))
.withHeader("Content-Type", equalTo("application/offset+octet-stream"))
.willReturn(aResponse()
.withStatus(201)
.withHeader("Upload-Offset", Long.toString(expectedEncryptedLength))));
remoteStorageManager.copy( remoteStorageManager.copy(
URI.create(wireMock.url("/cdn3/source/large")), sourceCdn,
LARGE.length(), "a/test/source",
params, 100,
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) encryptionParameters,
.toCompletableFuture().join(); "a/destination")
.toCompletableFuture().join());
final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).getFirst().getBody();
assertThat(destBody.length)
.isEqualTo(new BackupMediaEncrypter(params).outputSize(LARGE.length()))
.isEqualTo(params.outputSize(LARGE.length()));
assertThat(new String(decrypt(destBody), StandardCharsets.UTF_8)).isEqualTo(LARGE);
} }
@Test @Test
public void incorrectLength() { public void copyIncorrectLength() {
wireMock.stubFor(put(urlPathEqualTo("/storage-manager/copy")).willReturn(aResponse().withStatus(409)));
CompletableFutureTestUtil.assertFailsWithCause(InvalidLengthException.class, CompletableFutureTestUtil.assertFailsWithCause(InvalidLengthException.class,
remoteStorageManager.copy( remoteStorageManager.copy(
URI.create(wireMock.url("/cdn3/source/small")), 2,
SMALL_CDN3.length() - 1, "a/test/source",
100,
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV), new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) "a/destination").toCompletableFuture());
.toCompletableFuture());
} }
@Test @Test
public void sourceMissing() { public void copySourceMissing() {
wireMock.stubFor(put(urlPathEqualTo("/storage-manager/copy")).willReturn(aResponse().withStatus(404)));
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class, CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
remoteStorageManager.copy( remoteStorageManager.copy(
URI.create(wireMock.url("/cdn3/source/missing")), 2,
1, "a/test/source",
100,
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV), new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest"))) "a/destination").toCompletableFuture());
.toCompletableFuture());
} }
private byte[] decrypt(final byte[] encrypted) @Test
throws InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException { public void copyUnknownCdn() {
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
final Mac mac; remoteStorageManager.copy(
try { 0,
mac = Mac.getInstance("HmacSHA256"); "a/test/source",
} catch (NoSuchAlgorithmException e) { 100,
throw new AssertionError(e); new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
} "a/destination").toCompletableFuture());
mac.init(new SecretKeySpec(HMAC_KEY, "HmacSHA256"));
mac.update(encrypted, 0, encrypted.length - mac.getMacLength());
assertArrayEquals(mac.doFinal(),
Arrays.copyOfRange(encrypted, encrypted.length - mac.getMacLength(), encrypted.length));
assertArrayEquals(IV, Arrays.copyOf(encrypted, 16));
final Cipher cipher;
try {
cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
throw new AssertionError(e);
}
cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(AES_KEY, "AES"), new IvParameterSpec(IV));
return cipher.doFinal(encrypted, IV.length, encrypted.length - IV.length - mac.getMacLength());
} }
@Test @Test

30
service/src/test/resources/config/test.yml
View File

@ -249,37 +249,13 @@ cdn:
secretAccessKey: secret://cdn.accessSecret secretAccessKey: secret://cdn.accessSecret
region: us-west-2 # AWS region region: us-west-2 # AWS region
clientCdn:
attachmentUrls:
2: https://cdn2.example.com/attachments/
caCertificates:
- |
-----BEGIN CERTIFICATE-----
MIIDazCCAlOgAwIBAgIUW5lcNWkuynRVc8Rq5pO6mHQBuZAwDQYJKoZIhvcNAQEL
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDAzMjUwMzE4MTNaFw0yOTAz
MjQwMzE4MTNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
AQUAA4IBDwAwggEKAoIBAQCfH4Um+fv2r4KudhD37/UXp8duRLTmp4XvpBTpDHpD
2HF8p2yThVKlJnMkP/9Ey1Rb0vhxO7DCltLdW8IYcxJuHoyMvyhGUEtxxkOZbrk8
ciUR9jTZ37x7vXRGj/RxcdlS6iD0MeF0D/LAkImt4T/kiKwDbENrVEnYWJmipCKP
ribxWky7HqxDCoYMQr0zatxB3A9mx5stH+H3kbw3CZcm+ugF9ZIKDEVHb0lf28gq
llmD120q/vs9YV3rzVL7sBGDqf6olkulvHQJKElZg2rdcHWFcngSlU2BjR04oyuH
c/SSiLSB3YB0tdFGta5uorXyV1y7RElPeBfOfvEjsG3TAgMBAAGjUzBRMB0GA1Ud
DgQWBBQX+xlgSWWbDjv0SrJ+h67xauJ80zAfBgNVHSMEGDAWgBQX+xlgSWWbDjv0
SrJ+h67xauJ80zAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAw
ZG2MCCjscn6h/QOoJU+IDfa68OqLq0I37gMnLMde4yEhAmm//miePIq4Uz9GRJ+h
rAmdEnspKgyQ93PjF7Xpk/JdJA4B1bIrsOl/cSwqx2sFhRt8Kt1DHGlGWXqOaHRP
UkZ86MyRL3sXly6WkxEYxZJeQaOzMy2XmQh7grzrlTBuSI+0xf7vsRRDipxr6LVQ
6qGWyGODLLc2JD1IXj/1HpRVT2LoGGlKMuyxACQAm4oak1vvJ9mGxgfd9AU+eo58
O/esB2Eaf+QqMPELdFSZQfG2jvp+3WQTZK8fDKHyLr076G3UetEMy867F6fzTSZd
9Kxq0DY7RCEpdHMCKcOL
-----END CERTIFICATE-----
cdn3StorageManager: cdn3StorageManager:
baseUri: https://storage-manager.example.com baseUri: https://storage-manager.example.com
clientId: example clientId: example
clientSecret: secret://cdn3StorageManager.clientSecret clientSecret: secret://cdn3StorageManager.clientSecret
sourceSchemes:
2: gcs
3: r2
dogstatsd: dogstatsd:
type: nowait type: nowait