MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use storage-manager's copy implementation

This commit is contained in:
Ravi Khadiwala 2024-05-02 17:10:29 -05:00 committed by ravi-signal
parent 843151859d
commit fc097db2a0
14 changed files with 167 additions and 536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
service/config/sample.yml
View File

@ -254,37 +254,13 @@ cdn:
secretAccessKey: secret://cdn.accessSecret
region: us-west-2 # AWS region
clientCdn:
attachmentUrls:
2: https://cdn2.example.com/attachments/
caCertificates:
- |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
cdn3StorageManager:
baseUri: https://storage-manager.example.com
clientId: example
clientSecret: secret://cdn3StorageManager.clientSecret
sourceSchemes:
2: gcs
3: r2
dogstatsd:
environment: dev

9
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java
View File

@ -112,11 +112,6 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private CdnConfiguration cdn;
@NotNull
@Valid
@JsonProperty
private ClientCdnConfiguration clientCdn;
@NotNull
@Valid
@JsonProperty
@ -450,10 +445,6 @@ public class WhisperServerConfiguration extends Configuration {
return cdn;
}
public ClientCdnConfiguration getClientCdnConfiguration() {
return clientCdn;
}
public Cdn3StorageManagerConfiguration getCdn3StorageManagerConfiguration() {
return cdn3StorageManager;
}

8
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
View File

@ -703,13 +703,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
rateLimiters,
tusAttachmentGenerator,
cdn3BackupCredentialGenerator,
new Cdn3RemoteStorageManager(
remoteStorageExecutor,
config.getClientCdnConfiguration().getCircuitBreaker(),
config.getClientCdnConfiguration().getRetry(),
config.getClientCdnConfiguration().getCaCertificates(),
config.getCdn3StorageManagerConfiguration()),
config.getClientCdnConfiguration().getAttachmentUrls(),
new Cdn3RemoteStorageManager(remoteStorageExecutor, config.getCdn3StorageManagerConfiguration()),
clock);
final DynamicConfigTurnRouter configTurnRouter = new DynamicConfigTurnRouter(dynamicConfigurationManager);

48
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupManager.java
View File

@ -9,8 +9,7 @@ import com.google.common.annotations.VisibleForTesting;
import io.grpc.Status;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics;
import java.io.IOException;
import java.net.URI;
import io.micrometer.core.instrument.Timer;
import java.security.SecureRandom;
import java.time.Clock;
import java.time.Duration;
@ -22,9 +21,6 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.stream.Collectors;
import io.micrometer.core.instrument.Timer;
import org.apache.commons.lang3.StringUtils;
import org.signal.libsignal.protocol.ecc.Curve;
import org.signal.libsignal.protocol.ecc.ECPublicKey;
import org.signal.libsignal.zkgroup.GenericServerSecretParams;
@ -80,7 +76,6 @@ public class BackupManager {
private final TusAttachmentGenerator tusAttachmentGenerator;
private final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator;
private final RemoteStorageManager remoteStorageManager;
private final Map<Integer, String> attachmentCdnBaseUris;
private final SecureRandom secureRandom = new SecureRandom();
private final Clock clock;
@ -92,7 +87,6 @@ public class BackupManager {
final TusAttachmentGenerator tusAttachmentGenerator,
final Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator,
final RemoteStorageManager remoteStorageManager,
final Map<Integer, String> attachmentCdnBaseUris,
final Clock clock) {
this.backupsDb = backupsDb;
this.serverSecretParams = serverSecretParams;
@ -101,11 +95,6 @@ public class BackupManager {
this.cdn3BackupCredentialGenerator = cdn3BackupCredentialGenerator;
this.remoteStorageManager = remoteStorageManager;
this.clock = clock;
// strip trailing "/" for easier URI construction
this.attachmentCdnBaseUris = attachmentCdnBaseUris.entrySet().stream().collect(Collectors.toMap(
Map.Entry::getKey,
entry -> StringUtils.removeEnd(entry.getValue(), "/")
));
}
@ -271,23 +260,15 @@ public class BackupManager {
.asRuntimeException();
}
final URI sourceUri;
try {
sourceUri = attachmentReadUri(sourceCdn, sourceKey);
} catch (IOException e) {
return CompletableFuture.failedFuture(e);
}
final BackupUploadDescriptor dst = cdn3BackupCredentialGenerator.generateUpload(
cdnMediaPath(backupUser, destinationMediaId));
final String destination = cdnMediaPath(backupUser, destinationMediaId);
final int destinationLength = encryptionParameters.outputSize(sourceLength);
return this.backupsDb
// Write the ddb updates before actually updating backing storage
.trackMedia(backupUser, 1, destinationLength)
// Actually copy the objects. If the copy fails, our estimated quota usage may not be exact
.thenComposeAsync(ignored -> remoteStorageManager.copy(sourceUri, sourceLength, encryptionParameters, dst))
.thenComposeAsync(ignored ->
remoteStorageManager.copy(sourceCdn, sourceKey, sourceLength, encryptionParameters, destination))
.exceptionallyCompose(throwable -> {
final Throwable unwrapped = ExceptionUtils.unwrap(throwable);
if (!(unwrapped instanceof SourceObjectNotFoundException) && !(unwrapped instanceof InvalidLengthException)) {
@ -299,25 +280,10 @@ public class BackupManager {
});
})
// indicates where the backup was stored
.thenApply(ignore -> new StorageDescriptor(dst.cdn(), destinationMediaId));
.thenApply(ignore -> new StorageDescriptor(remoteStorageManager.cdnNumber(), destinationMediaId));
}
/**
* Construct the URI for an attachment with the specified key
*
* @param cdn where the attachment is located
* @param key the attachment key
* @return A {@link URI} where the attachment can be retrieved
*/
private URI attachmentReadUri(final int cdn, final String key) throws IOException {
final String baseUri = attachmentCdnBaseUris.get(cdn);
if (baseUri == null) {
throw new SourceObjectNotFoundException("Unknown attachment cdn " + cdn);
}
return URI.create("%s/%s".formatted(baseUri, key));
}
/**
* Generate credentials that can be used to read from the backup CDN
*
@ -377,7 +343,7 @@ public class BackupManager {
// Try to swap out the backupDir for the user
.scheduleBackupDeletion(backupUser)
// If there was already a pending swap, try to delete the cdn objects directly
.exceptionallyCompose(ExceptionUtils.exceptionallyHandler(BackupsDb.PendingDeletionException.class,e ->
.exceptionallyCompose(ExceptionUtils.exceptionallyHandler(BackupsDb.PendingDeletionException.class, e ->
AsyncTimerUtil.record(SYNCHRONOUS_DELETE_TIMER, () ->
deletePrefix(backupUser.backupDir(), DELETION_CONCURRENCY))));
}
@ -590,7 +556,7 @@ public class BackupManager {
/**
* Check that the authenticated backup user is authorized to use the provided backupLevel
*
* @param backupUser The backup user to check
* @param backupUser The backup user to check
* @param backupLevel The authorization level to verify the backupUser has access to
* @throws {@link Status#PERMISSION_DENIED} error if the backup user is not authorized to access {@code backupLevel}
*/

102
service/src/main/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypter.java
View File

@ -1,102 +0,0 @@
package org.whispersystems.textsecuregcm.backup;
import java.nio.ByteBuffer;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.concurrent.Flow;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.Mac;
import javax.crypto.NoSuchPaddingException;
import org.reactivestreams.FlowAdapters;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class BackupMediaEncrypter {
private final Cipher cipher;
private final Mac mac;
public BackupMediaEncrypter(final MediaEncryptionParameters encryptionParameters) {
cipher = initializeCipher(encryptionParameters);
mac = initializeMac(encryptionParameters);
}
public int outputSize(final int inputSize) {
return cipher.getIV().length + cipher.getOutputSize(inputSize) + mac.getMacLength();
}
/**
* Perform streaming encryption
*
* @param sourceBody A source of ByteBuffers, typically from an asynchronous HttpResponse
* @return A publisher that returns IV + AES/CBC/PKCS5Padding encrypted source + HMAC(IV + encrypted source) suitable
* to write with an asynchronous HttpRequest
*/
public Flow.Publisher<ByteBuffer> encryptBody(Flow.Publisher<List<ByteBuffer>> sourceBody) {
// Write IV, encrypted payload, mac
final Flux<ByteBuffer> encryptedBody = Flux.concat(
Mono.fromSupplier(() -> {
mac.update(cipher.getIV());
return ByteBuffer.wrap(cipher.getIV());
}),
Flux.from(FlowAdapters.toPublisher(sourceBody))
.concatMap(Flux::fromIterable)
.concatMap(byteBuffer -> {
final byte[] copy = new byte[byteBuffer.remaining()];
byteBuffer.get(copy);
final byte[] res = cipher.update(copy);
if (res == null) {
return Mono.empty();
} else {
mac.update(res);
return Mono.just(ByteBuffer.wrap(res));
}
}),
Mono.fromSupplier(() -> {
try {
final byte[] finalBytes = cipher.doFinal();
mac.update(finalBytes);
return ByteBuffer.wrap(finalBytes);
} catch (IllegalBlockSizeException | BadPaddingException e) {
throw ExceptionUtils.wrap(e);
}
}),
Mono.fromSupplier(() -> ByteBuffer.wrap(mac.doFinal())));
return FlowAdapters.toFlowPublisher(encryptedBody);
}
private static Mac initializeMac(final MediaEncryptionParameters encryptionParameters) {
try {
final Mac mac = Mac.getInstance("HmacSHA256");
mac.init(encryptionParameters.hmacSHA256Key());
return mac;
} catch (NoSuchAlgorithmException e) {
throw new AssertionError(e);
} catch (InvalidKeyException e) {
throw new IllegalArgumentException(e);
}
}
private static Cipher initializeCipher(final MediaEncryptionParameters encryptionParameters) {
try {
final Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(
Cipher.ENCRYPT_MODE,
encryptionParameters.aesEncryptionKey(),
encryptionParameters.iv());
return cipher;
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
throw new AssertionError(e);
} catch (InvalidAlgorithmParameterException | InvalidKeyException e) {
throw new IllegalArgumentException(e);
}
}
}

177
service/src/main/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManager.java
View File

@ -1,5 +1,6 @@
package org.whispersystems.textsecuregcm.backup;
import com.fasterxml.jackson.core.JsonProcessingException;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import java.io.IOException;
@ -8,30 +9,24 @@ import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.ByteBuffer;
import java.security.cert.CertificateException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletionException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executors;
import java.util.concurrent.Flow;
import java.util.concurrent.ScheduledExecutorService;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import javax.validation.constraints.NotNull;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.Cdn3StorageManagerConfiguration;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.configuration.RetryConfiguration;
import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.util.ExceptionUtils;
@ -42,20 +37,15 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
private static final Logger logger = LoggerFactory.getLogger(Cdn3RemoteStorageManager.class);
private final FaultTolerantHttpClient cdnHttpClient;
private final FaultTolerantHttpClient storageManagerHttpClient;
private final String storageManagerBaseUrl;
private final String clientId;
private final String clientSecret;
private final Map<Integer, String> sourceSchemes;
static final String CLIENT_ID_HEADER = "CF-Access-Client-Id";
static final String CLIENT_SECRET_HEADER = "CF-Access-Client-Secret";
private static String TUS_UPLOAD_LENGTH_HEADER = "Upload-Length";
private static String TUS_UPLOAD_OFFSET_HEADER = "Upload-Offset";
private static String TUS_VERSION_HEADER = "Tus-Resumable";
private static String TUS_VERSION = "1.0.0";
private static String TUS_CONTENT_TYPE = "application/offset+octet-stream";
private static final String STORAGE_MANAGER_STATUS_COUNTER_NAME = MetricsUtil.name(Cdn3RemoteStorageManager.class,
"storageManagerStatus");
@ -66,41 +56,25 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
public Cdn3RemoteStorageManager(
final ScheduledExecutorService retryExecutor,
final CircuitBreakerConfiguration circuitBreakerConfiguration,
final RetryConfiguration retryConfiguration,
final List<String> cdnCaCertificates,
final Cdn3StorageManagerConfiguration configuration) throws CertificateException {
final Cdn3StorageManagerConfiguration configuration) {
// strip trailing "/" for easier URI construction
this.storageManagerBaseUrl = StringUtils.removeEnd(configuration.baseUri(), "/");
this.clientId = configuration.clientId();
this.clientSecret = configuration.clientSecret().value();
// Client used to read/write to cdn
this.cdnHttpClient = FaultTolerantHttpClient.newBuilder()
.withName("cdn-client")
.withCircuitBreaker(circuitBreakerConfiguration)
.withExecutor(Executors.newCachedThreadPool())
.withRetryExecutor(retryExecutor)
.withRetry(retryConfiguration)
.withConnectTimeout(Duration.ofSeconds(10))
.withVersion(HttpClient.Version.HTTP_2)
.withTrustedServerCertificates(cdnCaCertificates.toArray(new String[0]))
.withNumClients(configuration.numHttpClients())
.build();
// Client used for calls to storage-manager
// storage-manager has an external CA so uses a different client
this.storageManagerHttpClient = FaultTolerantHttpClient.newBuilder()
.withName("cdn3-storage-manager")
.withCircuitBreaker(circuitBreakerConfiguration)
.withCircuitBreaker(configuration.circuitBreaker())
.withExecutor(Executors.newCachedThreadPool())
.withRetryExecutor(retryExecutor)
.withRetry(retryConfiguration)
.withRetry(configuration.retry())
.withConnectTimeout(Duration.ofSeconds(10))
.withVersion(HttpClient.Version.HTTP_2)
.withNumClients(configuration.numHttpClients())
.build();
this.sourceSchemes = configuration.sourceSchemes();
}
@Override
@ -110,85 +84,70 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
@Override
public CompletionStage<Void> copy(
final URI sourceUri,
final int sourceCdn,
final String sourceKey,
final int expectedSourceLength,
final MediaEncryptionParameters encryptionParameters,
final BackupUploadDescriptor uploadDescriptor) {
if (uploadDescriptor.cdn() != cdnNumber()) {
throw new IllegalArgumentException("Cdn3RemoteStorageManager can only copy to cdn3");
final String destinationKey) {
final String sourceScheme = this.sourceSchemes.get(sourceCdn);
if (sourceScheme == null) {
return CompletableFuture.failedFuture(
new SourceObjectNotFoundException("Cdn3RemoteStorageManager cannot copy from " + sourceCdn));
}
final String requestBody = new Cdn3CopyRequest(
encryptionParameters,
new Cdn3CopyRequest.SourceDescriptor(sourceScheme, sourceKey),
expectedSourceLength,
destinationKey).json();
final Timer.Sample sample = Timer.start();
final BackupMediaEncrypter encrypter = new BackupMediaEncrypter(encryptionParameters);
final HttpRequest request = HttpRequest.newBuilder().GET().uri(sourceUri).build();
return cdnHttpClient.sendAsync(request, HttpResponse.BodyHandlers.ofPublisher()).thenCompose(response -> {
try {
return cdnHttpClient.sendAsync(
createCopyRequest(expectedSourceLength, uploadDescriptor, encrypter, response),
HttpResponse.BodyHandlers.discarding());
} catch (Exception e) {
// Discard the response body so we don't hold the http2 stream open
response.body().subscribe(CancelSubscriber.INSTANCE);
throw ExceptionUtils.wrap(e);
}
})
final HttpRequest request = HttpRequest.newBuilder()
.PUT(HttpRequest.BodyPublishers.ofString(requestBody))
.uri(URI.create(copyUrl()))
.header("Content-Type", "application/json")
.header(CLIENT_ID_HEADER, clientId)
.header(CLIENT_SECRET_HEADER, clientSecret)
.build();
return this.storageManagerHttpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenAccept(response -> {
if (response.statusCode() != Response.Status.CREATED.getStatusCode() &&
response.statusCode() != Response.Status.OK.getStatusCode()) {
throw new CompletionException(new IOException("Failed to copy object: " + response.statusCode()));
}
long uploadOffset = response.headers().firstValueAsLong(TUS_UPLOAD_OFFSET_HEADER)
.orElseThrow(() -> new CompletionException(new IOException("Tus server did not return Upload-Offset")));
final int expectedEncryptedLength = encrypter.outputSize(expectedSourceLength);
if (uploadOffset != expectedEncryptedLength) {
throw new CompletionException(new IOException(
"Expected to upload %d bytes, uploaded %d".formatted(expectedEncryptedLength, uploadOffset)));
if (response.statusCode() == Response.Status.NOT_FOUND.getStatusCode()) {
throw ExceptionUtils.wrap(new SourceObjectNotFoundException());
} else if (response.statusCode() == Response.Status.CONFLICT.getStatusCode()) {
throw ExceptionUtils.wrap(new InvalidLengthException(response.body()));
} else if (!HttpUtils.isSuccessfulResponse(response.statusCode())) {
logger.info("Failed to copy via storage-manager {} {}", response.statusCode(), response.body());
throw ExceptionUtils.wrap(new IOException("Failed to copy object: " + response.statusCode()));
}
})
.whenComplete((ignored, ignoredException) ->
sample.stop(Metrics.timer(STORAGE_MANAGER_TIMER_NAME, OPERATION_TAG_NAME, "copy")));
}
private HttpRequest createCopyRequest(
final int expectedSourceLength,
final BackupUploadDescriptor uploadDescriptor,
BackupMediaEncrypter encrypter,
HttpResponse<Flow.Publisher<List<ByteBuffer>>> response) throws IOException {
if (response.statusCode() == Response.Status.NOT_FOUND.getStatusCode()) {
throw new SourceObjectNotFoundException();
} else if (response.statusCode() != Response.Status.OK.getStatusCode()) {
throw new IOException("error reading from source: " + response.statusCode());
/**
* Serialized copy request for cdn3 storage manager
*/
record Cdn3CopyRequest(
String encryptionKey, String hmacKey, String iv,
SourceDescriptor source, int expectedSourceLength,
String dst) {
Cdn3CopyRequest(MediaEncryptionParameters parameters, SourceDescriptor source, int expectedSourceLength,
String dst) {
this(Base64.getEncoder().encodeToString(parameters.aesEncryptionKey().getEncoded()),
Base64.getEncoder().encodeToString(parameters.hmacSHA256Key().getEncoded()),
Base64.getEncoder().encodeToString(parameters.iv().getIV()),
source, expectedSourceLength, dst);
}
final int actualSourceLength = Math.toIntExact(response.headers().firstValueAsLong("Content-Length")
.orElseThrow(() -> new IOException("upstream missing Content-Length")));
record SourceDescriptor(String scheme, String key) {}
if (actualSourceLength != expectedSourceLength) {
throw new InvalidLengthException(
"Provided sourceLength " + expectedSourceLength + " was " + actualSourceLength);
String json() {
try {
return SystemMapper.jsonMapper().writeValueAsString(this);
} catch (JsonProcessingException e) {
throw new IllegalStateException("Could not serialize copy request", e);
}
}
final int expectedEncryptedLength = encrypter.outputSize(expectedSourceLength);
final HttpRequest.BodyPublisher encryptedBody = HttpRequest.BodyPublishers.fromPublisher(
encrypter.encryptBody(response.body()), expectedEncryptedLength);
final String[] headers = Stream.concat(
uploadDescriptor.headers().entrySet()
.stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue())),
Stream.of(
TUS_VERSION_HEADER, TUS_VERSION,
TUS_UPLOAD_LENGTH_HEADER, Integer.toString(expectedEncryptedLength),
HttpHeaders.CONTENT_TYPE, TUS_CONTENT_TYPE))
.toArray(String[]::new);
return HttpRequest.newBuilder()
.uri(URI.create(uploadDescriptor.signedUploadLocation()))
.headers(headers)
.POST(encryptedBody)
.build();
}
@Override
@ -338,25 +297,7 @@ public class Cdn3RemoteStorageManager implements RemoteStorageManager {
return "%s/%s/".formatted(storageManagerBaseUrl, Cdn3BackupCredentialGenerator.CDN_PATH);
}
private static class CancelSubscriber implements Flow.Subscriber<List<ByteBuffer>> {
private static CancelSubscriber INSTANCE = new CancelSubscriber();
@Override
public void onSubscribe(final Flow.Subscription subscription) {
subscription.cancel();
}
@Override
public void onNext(final List<ByteBuffer> item) {
}
@Override
public void onError(final Throwable throwable) {
}
@Override
public void onComplete() {
}
private String copyUrl() {
return "%s/copy".formatted(storageManagerBaseUrl);
}
}

11
service/src/main/java/org/whispersystems/textsecuregcm/backup/RemoteStorageManager.java
View File

@ -1,6 +1,5 @@
package org.whispersystems.textsecuregcm.backup;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletionStage;
@ -18,12 +17,13 @@ public interface RemoteStorageManager {
/**
* Copy and the object from a remote source into the backup, adding an additional layer of encryption
*
* @param sourceUri The location of the object to copy
* @param sourceCdn The cdn number where the source attachment is stored
* @param sourceKey The key of the source attachment within the attachment cdn
* @param expectedSourceLength The length of the source object, should match the content-length of the object returned
* from the sourceUri.
* @param encryptionParameters The encryption keys that should be used to apply an additional layer of encryption to
* the object
* @param uploadDescriptor The destination, which must be in the cdn returned by {@link #cdnNumber()}
* @param dstKey The key within the backup cdn where the copied object will be written
* @return A stage that completes successfully when the source has been successfully re-encrypted and copied into
* uploadDescriptor. The returned CompletionStage can be completed exceptionally with the following exceptions.
* <ul>
@ -33,10 +33,11 @@ public interface RemoteStorageManager {
* </ul>
*/
CompletionStage<Void> copy(
URI sourceUri,
int sourceCdn,
String sourceKey,
int expectedSourceLength,
MediaEncryptionParameters encryptionParameters,
BackupUploadDescriptor uploadDescriptor);
String dstKey);
/**
* Result of a {@link #list} operation

31
service/src/main/java/org/whispersystems/textsecuregcm/configuration/Cdn3StorageManagerConfiguration.java
View File

@ -1,17 +1,44 @@
package org.whispersystems.textsecuregcm.configuration;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
import java.util.Collections;
import java.util.Map;
import javax.validation.constraints.NotNull;
import org.whispersystems.textsecuregcm.configuration.secrets.SecretString;
/**
* Configuration for the cdn3 storage manager
*
* @param baseUri The base URI of the storage manager
* @param clientId The cloudflare client ID to use to authenticate to the storage manager
* @param clientSecret The cloudflare client secret to use to authenticate to the storage manager
* @param sourceSchemes A map of cdn id to a retrieval scheme understood by the storage-manager. This is used by the
* storage-manager when copying to determine how to read a source object. Current schemes are
* 'gcs' and 'r2'
* @param numHttpClients The number http clients to use with the storage-manager to support request striping
* @param circuitBreaker A circuit breaker configuration for the storage-manager http client
* @param retry A retry configuration for the storage-manager http client
*/
public record Cdn3StorageManagerConfiguration(
@NotNull String baseUri,
@NotNull String clientId,
@NotNull SecretString clientSecret,
@NotNull Integer numHttpClients) {
@NotNull Map<Integer, String> sourceSchemes,
@NotNull Integer numHttpClients,
@NotNull CircuitBreakerConfiguration circuitBreaker,
@NotNull RetryConfiguration retry) {
public Cdn3StorageManagerConfiguration {
if (numHttpClients == null) {
numHttpClients = 2;
}
if (sourceSchemes == null) {
sourceSchemes = Collections.emptyMap();
}
if (circuitBreaker == null) {
circuitBreaker = new CircuitBreakerConfiguration();
}
if (retry == null) {
retry = new RetryConfiguration();
}
}
}

22
service/src/main/java/org/whispersystems/textsecuregcm/configuration/ClientCdnConfiguration.java
View File

@ -13,20 +13,6 @@ import java.util.Map;
*/
public class ClientCdnConfiguration {
/**
* Map from cdn number to the base url for attachments.
* <p>
* For example, if an attachment with the id 'abc' can be retrieved from cdn 2 at https://example.org/attachments/abc,
* the attachment url for 2 should https://example.org/attachments
*/
@JsonProperty
@NotNull
Map<Integer, @NotBlank String> attachmentUrls;
@JsonProperty
@NotNull
@NotEmpty List<@NotBlank String> caCertificates = new ArrayList<>();
@JsonProperty
@NotNull
CircuitBreakerConfiguration circuitBreaker = new CircuitBreakerConfiguration();
@ -35,10 +21,6 @@ public class ClientCdnConfiguration {
@NotNull
RetryConfiguration retry = new RetryConfiguration();
public List<String> getCaCertificates() {
return caCertificates;
}
public CircuitBreakerConfiguration getCircuitBreaker() {
return circuitBreaker;
}
@ -46,8 +28,4 @@ public class ClientCdnConfiguration {
public RetryConfiguration getRetry() {
return retry;
}
public Map<Integer, String> getAttachmentUrls() {
return attachmentUrls;
}
}

8
service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java
View File

@ -211,13 +211,7 @@ record CommandDependencies(
rateLimiters,
new TusAttachmentGenerator(configuration.getTus()),
new Cdn3BackupCredentialGenerator(configuration.getTus()),
new Cdn3RemoteStorageManager(
remoteStorageExecutor,
configuration.getClientCdnConfiguration().getCircuitBreaker(),
configuration.getClientCdnConfiguration().getRetry(),
configuration.getClientCdnConfiguration().getCaCertificates(),
configuration.getCdn3StorageManagerConfiguration()),
configuration.getClientCdnConfiguration().getAttachmentUrls(),
new Cdn3RemoteStorageManager(remoteStorageExecutor, configuration.getCdn3StorageManagerConfiguration()),
clock);
environment.lifecycle().manage(messagesCache);

18
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupManagerTest.java
View File

@ -101,6 +101,8 @@ public class BackupManagerTest {
final RateLimiters rateLimiters = mock(RateLimiters.class);
when(rateLimiters.forDescriptor(RateLimiters.For.BACKUP_ATTACHMENT)).thenReturn(mediaUploadLimiter);
when(remoteStorageManager.cdnNumber()).thenReturn(3);
this.backupsDb = new BackupsDb(
DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(),
@ -113,7 +115,6 @@ public class BackupManagerTest {
tusAttachmentGenerator,
tusCredentialGenerator,
remoteStorageManager,
Map.of(3, "cdn3.example.org/attachments"),
testClock);
}
@ -352,7 +353,7 @@ public class BackupManagerTest {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
when(tusCredentialGenerator.generateUpload(any()))
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any()))
when(remoteStorageManager.copy(eq(3), eq("abc"), eq(100), any(), any()))
.thenReturn(CompletableFuture.completedFuture(null));
final MediaEncryptionParameters encryptionParams = new MediaEncryptionParameters(
TestRandomUtil.nextBytes(32),
@ -378,7 +379,7 @@ public class BackupManagerTest {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
when(tusCredentialGenerator.generateUpload(any()))
.thenReturn(new BackupUploadDescriptor(3, "def", Collections.emptyMap(), ""));
when(remoteStorageManager.copy(eq(URI.create("cdn3.example.org/attachments/abc")), eq(100), any(), any()))
when(remoteStorageManager.copy(eq(3), eq("abc"), eq(100), any(), any()))
.thenReturn(CompletableFuture.failedFuture(new SourceObjectNotFoundException()));
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
@ -394,17 +395,6 @@ public class BackupManagerTest {
assertThat(AttributeValues.getLong(backup, BackupsDb.ATTR_MEDIA_COUNT, -1L)).isEqualTo(0L);
}
@Test
public void unknownSourceCdn() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
backupManager.copyToBackup(
backupUser,
0, "abc", 100,
mock(MediaEncryptionParameters.class),
"def".getBytes(StandardCharsets.UTF_8)));
}
@Test
public void quotaEnforcementNoRecalculation() {
final AuthenticatedBackupUser backupUser = backupUser(TestRandomUtil.nextBytes(16), BackupLevel.MEDIA);

20
service/src/test/java/org/whispersystems/textsecuregcm/backup/BackupMediaEncrypterTest.java
View File

@ -1,20 +0,0 @@
package org.whispersystems.textsecuregcm.backup;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.whispersystems.textsecuregcm.util.TestRandomUtil;
public class BackupMediaEncrypterTest {
@ParameterizedTest
@ValueSource(ints = {0, 1, 2, 15, 16, 17, 63, 64, 65, 1023, 1024, 1025})
public void sizeCalc() {
final MediaEncryptionParameters params = new MediaEncryptionParameters(
TestRandomUtil.nextBytes(32),
TestRandomUtil.nextBytes(32), TestRandomUtil.nextBytes(16));
final BackupMediaEncrypter encrypter = new BackupMediaEncrypter(params);
assertThat(params.outputSize(1)).isEqualTo(encrypter.outputSize(1));
}
}

189
service/src/test/java/org/whispersystems/textsecuregcm/backup/Cdn3RemoteStorageManagerTest.java
View File

@ -3,37 +3,23 @@ package org.whispersystems.textsecuregcm.backup;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.post;
import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
import static com.github.tomakehurst.wiremock.client.WireMock.put;
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.assertj.core.api.Assertions.assertThatNoException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.github.tomakehurst.wiremock.client.WireMock;
import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Executors;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.Mac;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import javax.ws.rs.core.HttpHeaders;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@ -60,150 +46,83 @@ public class Cdn3RemoteStorageManagerTest {
.options(wireMockConfig().dynamicPort())
.build();
private static final String SMALL_CDN2 = "a small object from cdn2";
private static final String SMALL_CDN3 = "a small object from cdn3";
private static final String LARGE = "a".repeat(1024 * 1024 * 5);
private RemoteStorageManager remoteStorageManager;
@BeforeEach
public void init() throws CertificateException {
public void init() {
remoteStorageManager = new Cdn3RemoteStorageManager(
Executors.newSingleThreadScheduledExecutor(),
new CircuitBreakerConfiguration(),
new RetryConfiguration(),
Collections.emptyList(),
new Cdn3StorageManagerConfiguration(
wireMock.url("storage-manager/"),
"clientId",
new SecretString("clientSecret"),
2));
wireMock.stubFor(get(urlEqualTo("/cdn2/source/small"))
.willReturn(aResponse()
.withHeader("Content-Length", Integer.toString(SMALL_CDN2.length()))
.withBody(SMALL_CDN2)));
wireMock.stubFor(get(urlEqualTo("/cdn3/source/small"))
.willReturn(aResponse()
.withHeader("Content-Length", Integer.toString(SMALL_CDN3.length()))
.withBody(SMALL_CDN3)));
wireMock.stubFor(get(urlEqualTo("/cdn3/source/large"))
.willReturn(aResponse()
.withHeader("Content-Length", Integer.toString(LARGE.length()))
.withBody(LARGE)));
wireMock.stubFor(get(urlEqualTo("/cdn3/source/missing"))
.willReturn(aResponse().withStatus(404)));
Map.of(2, "gcs", 3, "r2"),
2,
new CircuitBreakerConfiguration(),
new RetryConfiguration()));
}
@ParameterizedTest
@ValueSource(ints = {2, 3})
public void copySmall(final int sourceCdn)
throws InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
final String expectedSource = switch (sourceCdn) {
case 2 -> SMALL_CDN2;
case 3 -> SMALL_CDN3;
public void copy(final int sourceCdn) throws JsonProcessingException {
final MediaEncryptionParameters encryptionParameters = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
final String scheme = switch (sourceCdn) {
case 2 -> "gcs";
case 3 -> "r2";
default -> throw new AssertionError();
};
final MediaEncryptionParameters encryptionParameters = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
final long expectedEncryptedLength = encryptionParameters.outputSize(expectedSource.length());
wireMock.stubFor(post(urlEqualTo("/cdn3/dest"))
.withHeader("Content-Length", equalTo(Long.toString(expectedEncryptedLength)))
.withHeader("Upload-Length", equalTo(Long.toString(expectedEncryptedLength)))
.withHeader("Content-Type", equalTo("application/offset+octet-stream"))
.willReturn(aResponse()
.withStatus(201)
.withHeader("Upload-Offset", Long.toString(expectedEncryptedLength))));
remoteStorageManager.copy(
URI.create(wireMock.url("/cdn" + sourceCdn + "/source/small")),
expectedSource.length(),
encryptionParameters,
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest")))
.toCompletableFuture().join();
final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).getFirst().getBody();
assertThat(new String(decrypt(destBody), StandardCharsets.UTF_8))
.isEqualTo(expectedSource);
final Cdn3RemoteStorageManager.Cdn3CopyRequest expectedCopyRequest = new Cdn3RemoteStorageManager.Cdn3CopyRequest(
encryptionParameters,
new Cdn3RemoteStorageManager.Cdn3CopyRequest.SourceDescriptor(scheme, "a/test/source"),
100,
"a/destination");
wireMock.stubFor(put(urlEqualTo("/storage-manager/copy"))
.withHeader(HttpHeaders.CONTENT_TYPE, equalTo("application/json"))
.withRequestBody(WireMock.equalToJson(SystemMapper.jsonMapper().writeValueAsString(expectedCopyRequest)))
.willReturn(aResponse().withStatus(204)));
assertThatNoException().isThrownBy(() ->
remoteStorageManager.copy(
sourceCdn,
"a/test/source",
100,
encryptionParameters,
"a/destination")
.toCompletableFuture().join());
}
@Test
public void copyLarge()
throws InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException {
final MediaEncryptionParameters params = new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV);
final long expectedEncryptedLength = params.outputSize(LARGE.length());
wireMock.stubFor(post(urlEqualTo("/cdn3/dest"))
.withHeader("Content-Length", equalTo(Long.toString(expectedEncryptedLength)))
.withHeader("Upload-Length", equalTo(Long.toString(expectedEncryptedLength)))
.withHeader("Content-Type", equalTo("application/offset+octet-stream"))
.willReturn(aResponse()
.withStatus(201)
.withHeader("Upload-Offset", Long.toString(expectedEncryptedLength))));
remoteStorageManager.copy(
URI.create(wireMock.url("/cdn3/source/large")),
LARGE.length(),
params,
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest")))
.toCompletableFuture().join();
final byte[] destBody = wireMock.findAll(postRequestedFor(urlEqualTo("/cdn3/dest"))).getFirst().getBody();
assertThat(destBody.length)
.isEqualTo(new BackupMediaEncrypter(params).outputSize(LARGE.length()))
.isEqualTo(params.outputSize(LARGE.length()));
assertThat(new String(decrypt(destBody), StandardCharsets.UTF_8)).isEqualTo(LARGE);
}
@Test
public void incorrectLength() {
public void copyIncorrectLength() {
wireMock.stubFor(put(urlPathEqualTo("/storage-manager/copy")).willReturn(aResponse().withStatus(409)));
CompletableFutureTestUtil.assertFailsWithCause(InvalidLengthException.class,
remoteStorageManager.copy(
URI.create(wireMock.url("/cdn3/source/small")),
SMALL_CDN3.length() - 1,
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest")))
.toCompletableFuture());
2,
"a/test/source",
100,
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
"a/destination").toCompletableFuture());
}
@Test
public void sourceMissing() {
public void copySourceMissing() {
wireMock.stubFor(put(urlPathEqualTo("/storage-manager/copy")).willReturn(aResponse().withStatus(404)));
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
remoteStorageManager.copy(
URI.create(wireMock.url("/cdn3/source/missing")),
1,
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
new BackupUploadDescriptor(3, "test", Collections.emptyMap(), wireMock.url("/cdn3/dest")))
.toCompletableFuture());
2,
"a/test/source",
100,
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
"a/destination").toCompletableFuture());
}
private byte[] decrypt(final byte[] encrypted)
throws InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException, BadPaddingException {
final Mac mac;
try {
mac = Mac.getInstance("HmacSHA256");
} catch (NoSuchAlgorithmException e) {
throw new AssertionError(e);
}
mac.init(new SecretKeySpec(HMAC_KEY, "HmacSHA256"));
mac.update(encrypted, 0, encrypted.length - mac.getMacLength());
assertArrayEquals(mac.doFinal(),
Arrays.copyOfRange(encrypted, encrypted.length - mac.getMacLength(), encrypted.length));
assertArrayEquals(IV, Arrays.copyOf(encrypted, 16));
final Cipher cipher;
try {
cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
throw new AssertionError(e);
}
cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(AES_KEY, "AES"), new IvParameterSpec(IV));
return cipher.doFinal(encrypted, IV.length, encrypted.length - IV.length - mac.getMacLength());
@Test
public void copyUnknownCdn() {
CompletableFutureTestUtil.assertFailsWithCause(SourceObjectNotFoundException.class,
remoteStorageManager.copy(
0,
"a/test/source",
100,
new MediaEncryptionParameters(AES_KEY, HMAC_KEY, IV),
"a/destination").toCompletableFuture());
}
@Test

30
service/src/test/resources/config/test.yml
View File

@ -249,37 +249,13 @@ cdn:
secretAccessKey: secret://cdn.accessSecret
region: us-west-2 # AWS region
clientCdn:
attachmentUrls:
2: https://cdn2.example.com/attachments/
caCertificates:
- |
-----BEGIN CERTIFICATE-----
MIIDazCCAlOgAwIBAgIUW5lcNWkuynRVc8Rq5pO6mHQBuZAwDQYJKoZIhvcNAQEL
BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDAzMjUwMzE4MTNaFw0yOTAz
MjQwMzE4MTNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
AQUAA4IBDwAwggEKAoIBAQCfH4Um+fv2r4KudhD37/UXp8duRLTmp4XvpBTpDHpD
2HF8p2yThVKlJnMkP/9Ey1Rb0vhxO7DCltLdW8IYcxJuHoyMvyhGUEtxxkOZbrk8
ciUR9jTZ37x7vXRGj/RxcdlS6iD0MeF0D/LAkImt4T/kiKwDbENrVEnYWJmipCKP
ribxWky7HqxDCoYMQr0zatxB3A9mx5stH+H3kbw3CZcm+ugF9ZIKDEVHb0lf28gq
llmD120q/vs9YV3rzVL7sBGDqf6olkulvHQJKElZg2rdcHWFcngSlU2BjR04oyuH
c/SSiLSB3YB0tdFGta5uorXyV1y7RElPeBfOfvEjsG3TAgMBAAGjUzBRMB0GA1Ud
DgQWBBQX+xlgSWWbDjv0SrJ+h67xauJ80zAfBgNVHSMEGDAWgBQX+xlgSWWbDjv0
SrJ+h67xauJ80zAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAw
ZG2MCCjscn6h/QOoJU+IDfa68OqLq0I37gMnLMde4yEhAmm//miePIq4Uz9GRJ+h
rAmdEnspKgyQ93PjF7Xpk/JdJA4B1bIrsOl/cSwqx2sFhRt8Kt1DHGlGWXqOaHRP
UkZ86MyRL3sXly6WkxEYxZJeQaOzMy2XmQh7grzrlTBuSI+0xf7vsRRDipxr6LVQ
6qGWyGODLLc2JD1IXj/1HpRVT2LoGGlKMuyxACQAm4oak1vvJ9mGxgfd9AU+eo58
O/esB2Eaf+QqMPELdFSZQfG2jvp+3WQTZK8fDKHyLr076G3UetEMy867F6fzTSZd
9Kxq0DY7RCEpdHMCKcOL
-----END CERTIFICATE-----
cdn3StorageManager:
baseUri: https://storage-manager.example.com
clientId: example
clientSecret: secret://cdn3StorageManager.clientSecret
sourceSchemes:
2: gcs
3: r2
dogstatsd:
type: nowait