Delete from SVR3 in account deletion flow

service/config/sample.yml

@@ -215,7 +215,9 @@ svr2:
       -----END CERTIFICATE-----
 
 svr3:
-  uri: svr3.example.com
+  backend1Uri: backend1.example.com
+  backend2Uri: backend2.example.com
+  backend3Uri: backend3.example.com
   userAuthenticationTokenSharedSecret: secret://svr3.userAuthenticationTokenSharedSecret
   userIdTokenSharedSecret: secret://svr3.userIdTokenSharedSecret
   svrCaCertificates:

service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java

@@ -206,6 +206,7 @@ import org.whispersystems.textsecuregcm.s3.PolicySigner;
 import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.spam.ChallengeConstraintChecker;
 import org.whispersystems.textsecuregcm.spam.RegistrationFraudChecker;
 import org.whispersystems.textsecuregcm.spam.RegistrationRecoveryChecker;
@@ -478,8 +479,10 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
         .maxThreads(1).minThreads(1).build();
     ExecutorService fcmSenderExecutor = environment.lifecycle().executorService(name(getClass(), "fcmSender-%d"))
         .maxThreads(32).minThreads(32).workQueue(fcmSenderQueue).build();
-    ExecutorService secureValueRecoveryServiceExecutor = environment.lifecycle()
+    ExecutorService secureValueRecovery2ServiceExecutor = environment.lifecycle()
-        .executorService(name(getClass(), "secureValueRecoveryService-%d")).maxThreads(1).minThreads(1).build();
+        .executorService(name(getClass(), "secureValueRecoveryService2-%d")).maxThreads(1).minThreads(1).build();
+    ExecutorService secureValueRecovery3ServiceExecutor = environment.lifecycle()
+        .executorService(name(getClass(), "secureValueRecoveryService3-%d")).maxThreads(1).minThreads(1).build();
     ExecutorService storageServiceExecutor = environment.lifecycle()
         .executorService(name(getClass(), "storageService-%d")).maxThreads(1).minThreads(1).build();
     ExecutorService virtualThreadEventLoggerExecutor = environment.lifecycle()
@@ -602,7 +605,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
         config.getKeyTransparencyServiceConfiguration().clientPrivateKey().value(),
         keyTransparencyCallbackExecutor);
     SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(svr2CredentialsGenerator,
-        secureValueRecoveryServiceExecutor, secureValueRecoveryServiceRetryExecutor, config.getSvr2Configuration());
+        secureValueRecovery2ServiceExecutor, secureValueRecoveryServiceRetryExecutor, config.getSvr2Configuration());
+    SecureValueRecovery3Client secureValueRecovery3Client = new SecureValueRecovery3Client(svr3CredentialsGenerator,
+        secureValueRecovery3ServiceExecutor, secureValueRecoveryServiceRetryExecutor, config.getSvr3Configuration());
     SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator,
         storageServiceExecutor, storageServiceRetryExecutor, config.getSecureStorageServiceConfiguration());
     DisconnectionRequestManager disconnectionRequestManager = new DisconnectionRequestManager(pubsubClient, disconnectionRequestListenerExecutor);
@@ -623,7 +628,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
         new ClientPublicKeysManager(clientPublicKeys, accountLockManager, accountLockExecutor);
     AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster,
         pubsubClient, accountLockManager, keysManager, messagesManager, profilesManager,
-        secureStorageClient, secureValueRecovery2Client, disconnectionRequestManager,
+        secureStorageClient, secureValueRecovery2Client, secureValueRecovery3Client, disconnectionRequestManager,
         registrationRecoveryPasswordsManager, clientPublicKeysManager, accountLockExecutor, messagePollExecutor,
         clock, config.getLinkDeviceSecretConfiguration().secret().value(), dynamicConfigurationManager);
     RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs);

service/src/main/java/org/whispersystems/textsecuregcm/configuration/SecureValueRecovery3Configuration.java

@@ -13,7 +13,9 @@ import org.whispersystems.textsecuregcm.configuration.secrets.SecretBytes;
 import org.whispersystems.textsecuregcm.util.ExactlySize;
 
 public record SecureValueRecovery3Configuration(
-    @NotBlank String uri,
+    @NotBlank String backend1Uri,
+    @NotBlank String backend2Uri,
+    @NotBlank String backend3Uri,
     @ExactlySize(32) SecretBytes userAuthenticationTokenSharedSecret,
     @ExactlySize(32) SecretBytes userIdTokenSharedSecret,
     @NotEmpty List<@NotBlank String> svrCaCertificates,

service/src/main/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery3Client.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.securevaluerecovery;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HttpHeaders;
+import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
+import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
+import org.whispersystems.textsecuregcm.configuration.SecureValueRecovery3Configuration;
+import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient;
+import org.whispersystems.textsecuregcm.util.ExceptionUtils;
+import org.whispersystems.textsecuregcm.util.HttpUtils;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.security.cert.CertificateException;
+import java.time.Duration;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.stream.Stream;
+
+import static org.whispersystems.textsecuregcm.util.HeaderUtils.basicAuthHeader;
+
+/**
+ * A client for sending requests to Signal's secure value recovery v3 service on behalf of authenticated users.
+ */
+public class SecureValueRecovery3Client {
+  private final ExternalServiceCredentialsGenerator secureValueRecoveryCredentialsGenerator;
+  private final URI backend1Uri;
+  private final URI backend2Uri;
+  private final URI backend3Uri;
+
+  private final FaultTolerantHttpClient httpClient;
+
+  @VisibleForTesting
+  static final String DELETE_PATH = "/v1/delete";
+
+  public SecureValueRecovery3Client(final ExternalServiceCredentialsGenerator secureValueRecoveryCredentialsGenerator,
+      final Executor executor, final ScheduledExecutorService retryExecutor,
+      final SecureValueRecovery3Configuration configuration)
+      throws CertificateException {
+    this.secureValueRecoveryCredentialsGenerator = secureValueRecoveryCredentialsGenerator;
+    this.backend1Uri = URI.create(configuration.backend1Uri()).resolve(DELETE_PATH);
+    this.backend2Uri = URI.create(configuration.backend2Uri()).resolve(DELETE_PATH);
+    this.backend3Uri = URI.create(configuration.backend3Uri()).resolve(DELETE_PATH);
+    this.httpClient = FaultTolerantHttpClient.newBuilder()
+        .withCircuitBreaker(configuration.circuitBreaker())
+        .withRetry(configuration.retry())
+        .withRetryExecutor(retryExecutor)
+        .withVersion(HttpClient.Version.HTTP_1_1)
+        .withConnectTimeout(Duration.ofSeconds(10))
+        .withRedirect(HttpClient.Redirect.NEVER)
+        .withExecutor(executor)
+        .withName("secure-value-recovery3")
+        .withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_2)
+        .withTrustedServerCertificates(configuration.svrCaCertificates().toArray(String[]::new))
+        .build();
+  }
+
+  public CompletableFuture<Void> deleteBackups(final UUID accountUuid) {
+    final ExternalServiceCredentials credentials = secureValueRecoveryCredentialsGenerator.generateForUuid(accountUuid);
+    final List<CompletableFuture<HttpResponse<String>>> futures = Stream.of(backend1Uri, backend2Uri, backend3Uri)
+        .map(uri -> HttpRequest.newBuilder()
+            .uri(uri)
+            .DELETE()
+            .header(HttpHeaders.AUTHORIZATION, basicAuthHeader(credentials))
+            .build())
+        .map(request -> httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()))
+        .toList();
+
+    return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new))
+        .thenApply(ignored -> futures.stream().map(CompletableFuture::join).toList())
+        .thenAccept(responses -> responses.forEach(response -> {
+          if (!HttpUtils.isSuccessfulResponse(response.statusCode())) {
+            throw new SecureValueRecoveryException(String.format("Failed to delete backup in %s", response.uri()), String.valueOf(response.statusCode()));
+          }
+        }));
+  }
+}

service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java

@@ -82,6 +82,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecoveryException;
 import org.whispersystems.textsecuregcm.util.DestinationDeviceValidator;
 import org.whispersystems.textsecuregcm.util.ExceptionUtils;
@@ -125,6 +126,8 @@ public class AccountsManager extends RedisPubSubAdapter<String, String> implemen
   private final ProfilesManager profilesManager;
   private final SecureStorageClient secureStorageClient;
   private final SecureValueRecovery2Client secureValueRecovery2Client;
+  private final SecureValueRecovery3Client secureValueRecovery3Client;
+
   private final DisconnectionRequestManager disconnectionRequestManager;
   private final RegistrationRecoveryPasswordsManager registrationRecoveryPasswordsManager;
   private final ClientPublicKeysManager clientPublicKeysManager;
@@ -207,6 +210,7 @@ public class AccountsManager extends RedisPubSubAdapter<String, String> implemen
       final ProfilesManager profilesManager,
       final SecureStorageClient secureStorageClient,
       final SecureValueRecovery2Client secureValueRecovery2Client,
+      final SecureValueRecovery3Client secureValueRecovery3Client,
       final DisconnectionRequestManager disconnectionRequestManager,
       final RegistrationRecoveryPasswordsManager registrationRecoveryPasswordsManager,
       final ClientPublicKeysManager clientPublicKeysManager,
@@ -225,6 +229,7 @@ public class AccountsManager extends RedisPubSubAdapter<String, String> implemen
     this.profilesManager = profilesManager;
     this.secureStorageClient = secureStorageClient;
     this.secureValueRecovery2Client = secureValueRecovery2Client;
+    this.secureValueRecovery3Client = secureValueRecovery3Client;
     this.disconnectionRequestManager = disconnectionRequestManager;
     this.registrationRecoveryPasswordsManager = requireNonNull(registrationRecoveryPasswordsManager);
     this.clientPublicKeysManager = clientPublicKeysManager;
@@ -1248,19 +1253,27 @@ public class AccountsManager extends RedisPubSubAdapter<String, String> implemen
                     account.getIdentifier(IdentityType.ACI),
                     device.getId())))
         .toList();
-    CompletableFuture<Void> deleteBackupFuture = secureValueRecovery2Client.deleteBackups(account.getUuid())
+    final CompletableFuture<Void> svr2DeleteBackupFuture = secureValueRecovery2Client.deleteBackups(account.getUuid())
         .exceptionally(ExceptionUtils.exceptionallyHandler(SecureValueRecoveryException.class, exception -> {
           final List<String> svrStatusCodesToIgnore = dynamicConfigurationManager.getConfiguration().getSvrStatusCodesToIgnoreForAccountDeletion();
           if (svrStatusCodesToIgnore.contains(exception.getStatusCode())) {
-            logger.warn("Failed to delete backup for account: " + account.getUuid(), exception);
+            logger.warn("Ignoring failure to delete svr2 backup for account: " + account.getUuid(), exception);
             return null;
           }
+          logger.warn("Failed to delete svr2 backup for account: " + account.getUuid(), exception);
           throw new CompletionException(exception);
         }));
 
+    final CompletableFuture<Void> svr3DeleteBackupFuture = secureValueRecovery3Client.deleteBackups(account.getUuid())
+        .exceptionally(exception -> {
+          // We don't care about errors from SVR3 because we're not currently using it
+          return null;
+        });
+
     return CompletableFuture.allOf(
             secureStorageClient.deleteStoredData(account.getUuid()),
-            deleteBackupFuture,
+            svr2DeleteBackupFuture,
+            svr3DeleteBackupFuture,
             keysManager.deleteSingleUsePreKeys(account.getUuid()),
             keysManager.deleteSingleUsePreKeys(account.getPhoneNumberIdentifier()),
             messagesManager.clear(account.getUuid()),

service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java

@@ -32,6 +32,7 @@ import org.whispersystems.textsecuregcm.backup.Cdn3RemoteStorageManager;
 import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
 import org.whispersystems.textsecuregcm.controllers.SecureStorageController;
 import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery2Controller;
+import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery3Controller;
 import org.whispersystems.textsecuregcm.experiment.PushNotificationExperimentSamples;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
 import org.whispersystems.textsecuregcm.metrics.MicrometerAwsSdkMetricPublisher;
@@ -44,6 +45,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.storage.AccountLockManager;
 import org.whispersystems.textsecuregcm.storage.Accounts;
 import org.whispersystems.textsecuregcm.storage.AccountsManager;
@@ -153,8 +155,10 @@ record CommandDependencies(
 
     ExternalServiceCredentialsGenerator storageCredentialsGenerator = SecureStorageController.credentialsGenerator(
         configuration.getSecureStorageServiceConfiguration());
-    ExternalServiceCredentialsGenerator secureValueRecoveryCredentialsGenerator = SecureValueRecovery2Controller.credentialsGenerator(
+    ExternalServiceCredentialsGenerator secureValueRecovery2CredentialsGenerator = SecureValueRecovery2Controller.credentialsGenerator(
         configuration.getSvr2Configuration());
+    ExternalServiceCredentialsGenerator secureValueRecovery3CredentialsGenerator = SecureValueRecovery3Controller.credentialsGenerator(
+        configuration.getSvr3Configuration());
 
     final ExecutorService awsSdkMetricsExecutor = environment.lifecycle()
         .virtualExecutorService(MetricRegistry.name(CommandDependencies.class, "awsSdkMetrics-%d"));
@@ -204,9 +208,13 @@ record CommandDependencies(
     FaultTolerantRedisClusterClient rateLimitersCluster = configuration.getRateLimitersCluster().build("rate_limiters",
         redisClientResourcesBuilder);
     SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(
-        secureValueRecoveryCredentialsGenerator, secureValueRecoveryServiceExecutor,
+        secureValueRecovery2CredentialsGenerator, secureValueRecoveryServiceExecutor,
         secureValueRecoveryServiceRetryExecutor,
         configuration.getSvr2Configuration());
+    SecureValueRecovery3Client secureValueRecovery3Client = new SecureValueRecovery3Client(
+        secureValueRecovery3CredentialsGenerator, secureValueRecoveryServiceExecutor,
+        secureValueRecoveryServiceRetryExecutor,
+        configuration.getSvr3Configuration());
     SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator,
         storageServiceExecutor, storageServiceRetryExecutor, configuration.getSecureStorageServiceConfiguration());
     DisconnectionRequestManager disconnectionRequestManager = new DisconnectionRequestManager(pubsubClient, disconnectionRequestListenerExecutor);
@@ -228,7 +236,7 @@ record CommandDependencies(
         new RegistrationRecoveryPasswordsManager(registrationRecoveryPasswords);
     AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster,
         pubsubClient, accountLockManager, keys, messagesManager, profilesManager,
-        secureStorageClient, secureValueRecovery2Client, disconnectionRequestManager,
+        secureStorageClient, secureValueRecovery2Client, secureValueRecovery3Client, disconnectionRequestManager,
         registrationRecoveryPasswordsManager, clientPublicKeysManager, accountLockExecutor, messagePollExecutor,
         clock, configuration.getLinkDeviceSecretConfiguration().secret().value(), dynamicConfigurationManager);
     RateLimiters rateLimiters = RateLimiters.createAndValidate(configuration.getLimitsConfiguration(),

service/src/test/java/org/whispersystems/textsecuregcm/controllers/SecureValueRecovery3ControllerTest.java

@@ -55,6 +55,8 @@ import org.whispersystems.textsecuregcm.util.TestRandomUtil;
 public class SecureValueRecovery3ControllerTest extends SecureValueRecoveryControllerBaseTest {
 
   private static final SecureValueRecovery3Configuration CFG = new SecureValueRecovery3Configuration(
+      "",
+      "",
       "",
       randomSecretBytes(32),
       randomSecretBytes(32),

service/src/test/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery3ClientTest.java

@@ -0,0 +1,196 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.securevaluerecovery;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.delete;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes;
+
+import com.github.tomakehurst.wiremock.junit5.WireMockExtension;
+import java.security.cert.CertificateException;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
+import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator;
+import org.whispersystems.textsecuregcm.configuration.SecureValueRecovery3Configuration;
+
+class SecureValueRecovery3ClientTest {
+
+  private UUID accountUuid;
+  private ExternalServiceCredentialsGenerator credentialsGenerator;
+  private ExecutorService httpExecutor;
+  private ScheduledExecutorService retryExecutor;
+
+  private SecureValueRecovery3Client secureValueRecovery3Client;
+
+  @RegisterExtension
+  private static final WireMockExtension backend1WireMock = WireMockExtension.newInstance()
+      .options(wireMockConfig().dynamicPort().dynamicHttpsPort())
+      .build();
+
+  @RegisterExtension
+  private static final WireMockExtension backend2WireMock = WireMockExtension.newInstance()
+      .options(wireMockConfig().dynamicPort().dynamicHttpsPort())
+      .build();
+
+  @RegisterExtension
+  private static final WireMockExtension backend3WireMock = WireMockExtension.newInstance()
+      .options(wireMockConfig().dynamicPort().dynamicHttpsPort())
+      .build();
+
+  @BeforeEach
+  void setUp() throws CertificateException {
+    accountUuid = UUID.randomUUID();
+    credentialsGenerator = mock(ExternalServiceCredentialsGenerator.class);
+    httpExecutor = Executors.newSingleThreadExecutor();
+    retryExecutor = Executors.newSingleThreadScheduledExecutor();
+
+    final SecureValueRecovery3Configuration config = new SecureValueRecovery3Configuration(
+        "http://localhost:" + backend1WireMock.getPort(),
+        "http://localhost:" + backend2WireMock.getPort(),
+        "http://localhost:" + backend3WireMock.getPort(),
+        randomSecretBytes(32),
+        randomSecretBytes(32),
+        // This is a randomly-generated, throwaway certificate that's not actually connected to anything
+        List.of("""
+            -----BEGIN CERTIFICATE-----
+            MIICZDCCAc2gAwIBAgIBADANBgkqhkiG9w0BAQ0FADBPMQswCQYDVQQGEwJ1czEL
+            MAkGA1UECAwCVVMxHjAcBgNVBAoMFVNpZ25hbCBNZXNzZW5nZXIsIExMQzETMBEG
+            A1UEAwwKc2lnbmFsLm9yZzAeFw0yMDEyMjMyMjQ3NTlaFw0zMDEyMjEyMjQ3NTla
+            ME8xCzAJBgNVBAYTAnVzMQswCQYDVQQIDAJVUzEeMBwGA1UECgwVU2lnbmFsIE1l
+            c3NlbmdlciwgTExDMRMwEQYDVQQDDApzaWduYWwub3JnMIGfMA0GCSqGSIb3DQEB
+            AQUAA4GNADCBiQKBgQCfSLcZNHYqbxSsgWp4JvbPRHjQTrlsrKrgD2q7f/OY6O3Y
+            /X0QNcNSOJpliN8rmzwslfsrXHO3q1diGRw4xHogUJZ/7NQrHiP/zhN0VTDh49pD
+            ZpjXVyUbayLS/6qM5arKxBspzEFBb5v8cF6bPr76SO/rpGXiI0j6yJKX6fRiKwID
+            AQABo1AwTjAdBgNVHQ4EFgQU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwHwYDVR0jBBgw
+            FoAU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B
+            AQ0FAAOBgQB+5d5+NtzLILfrc9QmJdIO1YeDP64JmFwTER0kEUouRsb9UwknVWZa
+            y7MTM4NoBV1k0zb5LAk89SIDPr/maW5AsLtEomzjnEiomjoMBUdNe3YCgQReoLnr
+            R/QaUNbrCjTGYfBsjGbIzmkWPUyTec2ZdRyJ8JiVl386+6CZkxnndQ==
+            -----END CERTIFICATE-----
+            """, """
+            -----BEGIN CERTIFICATE-----
+            MIIEpDCCAowCCQC43PUTWSADVjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+            b2NhbGhvc3QwHhcNMjIxMDE3MjA0NTM0WhcNMjMxMDE3MjA0NTM0WjAUMRIwEAYD
+            VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDV
+            x1cdEd2ffQTlTXWRiCHGcrlYf4RJnctt9sw/BuHWTLXBu5LhyJSGn5LRszO/NCXK
+            Z/cmGR7pLj366RtiwL+Qo3nhvDCK7T9xZeNIusM6XMcMK9D/DGCYPqtjQz8NXd9V
+            ajBBe6nwTDTa+oqX8Mt89foWNkg5Il/lY62u9Dr18LRZ2W9zzYi3Q9/K0CbIX6pM
+            yVlPIO5rITOR2IsbeyqsO9jufgX5lP4ZKLLBAP1b7usjC4YdvWacjQg/rK5aay1x
+            jC2HCDgo/4N30QVXzSA9nFfSe6AE/xkStK4819JqOkY5JsJCbef1P3hOOdSLEjbp
+            xq3MjOs6G6dOgteaAGs10vx7dHxDWETTIiD7BIZ9zRYgOF5bkCaIUO+JfySE1MHD
+            KBAFLoRuvmRev5Ln5R0MCHpUMSmMNgJqz+RWZV3g/gpYbuWiHgJOwL1393eK50Bg
+            W7SXQ8EjJj2yXZSH+1gPzN0DRoJZiaBoTPnCL2qUgvwFpW1PJsM5FDyUJFUoK5kK
+            HLBBSKAPt6ZlSrUe2nBgJv7EF1GK+fTU08LXgW33OpLceGPa0zTShkukQUMtUtZ8
+            GqhO12ohMzEupIu5Xurthq4VVUrzHUdj1ZZRMhAbfLU36sd03MMyL/xBqTN6dzCa
+            GDGIPGpYjAllZ5xMRt2kZdv+Kr6oo3u2nLUIsqI7KQIDAQABMA0GCSqGSIb3DQEB
+            CwUAA4ICAQCB5s43YF35ssf5YONW5iAaifGpi1o0866xfeOybtohFGvQ7V2W34i9
+            TYBCt8+0hgatMcvZ08f0vqig1i7nrvYcE1hnhL7JNkU8qm0s9ytHZt6j62nB0kd/
+            uqE2hOEQalTf/2TGPV0CCgiqLyd8lEUQvQeA38wktwUeZpVnErlzHeMR2CvV3K8R
+            u4vV6SnBcf+TAt56RKYZkPyvZj5llQPo14Glyoo8qZES7Ky1SHmM0GL+baPRBjRW
+            3KgSt98Wyu4yr9qu21JpnbAnLhBfzfSKjSeCRgFElUE1GIaFGRZ7ypA74dUKeLnb
+            /VUWrszmUhGaEjV9dpI6x6B/kSpQMtIQqBaKRY2ALUeEujS/rURi4iMDwSU+GkSH
+            cyEvZKS97OA/dWeXfLXdo4beDBRG93bI4rQnDg5+VdlBOkQSLueb8x6/VThMoC5d
+            vZiotFQHseljQAdTkNa6tBu6c4XDYPCKB3CfkMYOlCfTS7Acn5G6dxTPKBtLGBnL
+            nQfYyzuwYkN09+2PVzt6auBHr3To7uoclkxX+hxyvPIwIZ0N6b4tQR1FCAkvg29Q
+            WIOjZOKGW690ESKCKOnFjUHVO0HpuWnT81URTuY62FXsYdVc2wE4v0E04mEbqQ0P
+            lY6ZKNA81Lm3YADYtObmK1IUrOPo9BeIaPy0UM08SmN880Vunqa91Q==
+            -----END CERTIFICATE-----
+            """),
+        null, null);
+
+    secureValueRecovery3Client = new SecureValueRecovery3Client(credentialsGenerator, httpExecutor, retryExecutor,
+        config);
+  }
+
+  @AfterEach
+  void tearDown() throws InterruptedException {
+    httpExecutor.shutdown();
+    httpExecutor.awaitTermination(1, TimeUnit.SECONDS);
+    retryExecutor.shutdown();
+    retryExecutor.awaitTermination(1, TimeUnit.SECONDS);
+  }
+
+  @Test
+  void deleteStoredData() {
+    final String username = RandomStringUtils.secure().nextAlphabetic(16);
+    final String password = RandomStringUtils.secure().nextAlphanumeric(32);
+
+    when(credentialsGenerator.generateForUuid(accountUuid)).thenReturn(
+        new ExternalServiceCredentials(username, password));
+
+    backend1WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH))
+        .withBasicAuth(username, password)
+        .willReturn(aResponse().withStatus(202)));
+
+    backend2WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH))
+        .withBasicAuth(username, password)
+        .willReturn(aResponse().withStatus(202)));
+
+    backend3WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH))
+        .withBasicAuth(username, password)
+        .willReturn(aResponse().withStatus(202)));
+
+    assertDoesNotThrow(() -> secureValueRecovery3Client.deleteBackups(accountUuid).join());
+  }
+
+  @ParameterizedTest
+  @MethodSource
+  void deleteStoredDataFailure(final int backend1Status, final int backend2Status, final int backend3Status) {
+    final String username = RandomStringUtils.secure().nextAlphabetic(16);
+    final String password = RandomStringUtils.secure().nextAlphanumeric(32);
+
+    when(credentialsGenerator.generateForUuid(accountUuid)).thenReturn(
+        new ExternalServiceCredentials(username, password));
+
+    backend1WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH))
+        .withBasicAuth(username, password)
+        .willReturn(aResponse().withStatus(backend1Status)));
+
+    backend2WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH))
+        .withBasicAuth(username, password)
+        .willReturn(aResponse().withStatus(backend2Status)));
+
+    backend3WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH))
+        .withBasicAuth(username, password)
+        .willReturn(aResponse().withStatus(backend3Status)));
+
+    final CompletionException completionException = assertThrows(CompletionException.class,
+        () -> secureValueRecovery3Client.deleteBackups(accountUuid).join());
+
+    assertInstanceOf(SecureValueRecoveryException.class, completionException.getCause());
+  }
+
+  private static Stream<Arguments> deleteStoredDataFailure() {
+    return Stream.of(
+        Arguments.of(400, 202, 202),
+        Arguments.of(202, 400, 202),
+        Arguments.of(202, 202, 400)
+    );
+  }
+
+}

service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountCreationDeletionIntegrationTest.java

@@ -49,6 +49,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.tests.util.KeysHelper;
 
 public class AccountCreationDeletionIntegrationTest {
@@ -125,6 +126,9 @@ public class AccountCreationDeletionIntegrationTest {
     final SecureValueRecovery2Client svr2Client = mock(SecureValueRecovery2Client.class);
     when(svr2Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null));
 
+    final SecureValueRecovery3Client svr3Client = mock(SecureValueRecovery3Client.class);
+    when(svr3Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null));
+
     final PhoneNumberIdentifiers phoneNumberIdentifiers =
         new PhoneNumberIdentifiers(DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(),
             DynamoDbExtensionSchema.Tables.PNI.tableName());
@@ -155,6 +159,7 @@ public class AccountCreationDeletionIntegrationTest {
         profilesManager,
         secureStorageClient,
         svr2Client,
+        svr3Client,
         disconnectionRequestManager,
         registrationRecoveryPasswordsManager,
         clientPublicKeysManager,

service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java

@@ -41,6 +41,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.storage.DynamoDbExtensionSchema.Tables;
 import org.whispersystems.textsecuregcm.tests.util.AccountsHelper;
 import org.whispersystems.textsecuregcm.tests.util.KeysHelper;
@@ -117,6 +118,9 @@ class AccountsManagerChangeNumberIntegrationTest {
       final SecureValueRecovery2Client svr2Client = mock(SecureValueRecovery2Client.class);
       when(svr2Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null));
 
+      final SecureValueRecovery3Client svr3Client = mock(SecureValueRecovery3Client.class);
+      when(svr3Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null));
+
       disconnectionRequestManager = mock(DisconnectionRequestManager.class);
 
       final PhoneNumberIdentifiers phoneNumberIdentifiers =
@@ -145,6 +149,7 @@ class AccountsManagerChangeNumberIntegrationTest {
           profilesManager,
           secureStorageClient,
           svr2Client,
+          svr3Client,
           disconnectionRequestManager,
           registrationRecoveryPasswordsManager,
           clientPublicKeysManager,

service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java

@@ -54,6 +54,7 @@ import org.whispersystems.textsecuregcm.identity.IdentityType;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.storage.DynamoDbExtensionSchema.Tables;
 import org.whispersystems.textsecuregcm.tests.util.DevicesHelper;
 import org.whispersystems.textsecuregcm.tests.util.JsonHelpers;
@@ -136,6 +137,7 @@ class AccountsManagerConcurrentModificationIntegrationTest {
           mock(ProfilesManager.class),
           mock(SecureStorageClient.class),
           mock(SecureValueRecovery2Client.class),
+          mock(SecureValueRecovery3Client.class),
           mock(DisconnectionRequestManager.class),
           mock(RegistrationRecoveryPasswordsManager.class),
           mock(ClientPublicKeysManager.class),

service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerDeviceTransferIntegrationTest.java

@@ -21,6 +21,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.redis.RedisServerExtension;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import java.nio.charset.StandardCharsets;
 import java.time.Clock;
 import java.time.Duration;
@@ -65,6 +66,7 @@ public class AccountsManagerDeviceTransferIntegrationTest {
         mock(ProfilesManager.class),
         mock(SecureStorageClient.class),
         mock(SecureValueRecovery2Client.class),
+        mock(SecureValueRecovery3Client.class),
         mock(DisconnectionRequestManager.class),
         mock(RegistrationRecoveryPasswordsManager.class),
         mock(ClientPublicKeysManager.class),

service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerTest.java

@@ -87,6 +87,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecoveryException;
 import org.whispersystems.textsecuregcm.storage.AccountsManager.UsernameReservation;
 import org.whispersystems.textsecuregcm.tests.util.AccountsHelper;
@@ -130,6 +131,7 @@ class AccountsManagerTest {
   private RedisAdvancedClusterAsyncCommands<String, String> asyncClusterCommands;
   private AccountsManager accountsManager;
   private SecureValueRecovery2Client svr2Client;
+  private SecureValueRecovery3Client svr3Client;
   private DynamicConfiguration dynamicConfiguration;
 
   private static final Answer<?> ACCOUNT_UPDATE_ANSWER = (answer) -> {
@@ -193,6 +195,9 @@ class AccountsManagerTest {
     svr2Client = mock(SecureValueRecovery2Client.class);
     when(svr2Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null));
 
+    svr3Client = mock(SecureValueRecovery3Client.class);
+    when(svr3Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null));
+
     final PhoneNumberIdentifiers phoneNumberIdentifiers = mock(PhoneNumberIdentifiers.class);
     phoneNumberIdentifiersByE164 = new HashMap<>();
 
@@ -254,6 +259,7 @@ class AccountsManagerTest {
         profilesManager,
         storageClient,
         svr2Client,
+        svr3Client,
         disconnectionRequestManager,
         registrationRecoveryPasswordsManager,
         clientPublicKeysManager,
@@ -266,7 +272,7 @@ class AccountsManagerTest {
 
   @ParameterizedTest
   @MethodSource
-  void testDeleteWithSvrErrorStatusCodes(final String statusCode, final boolean expectError) throws InterruptedException {
+  void testDeleteWithSvr2ErrorStatusCodes(final String statusCode, final boolean expectError) throws InterruptedException {
     when(svr2Client.deleteBackups(any())).thenReturn(
         CompletableFuture.failedFuture(new SecureValueRecoveryException("Failed to delete backup", statusCode)));
     when(dynamicConfiguration.getSvrStatusCodesToIgnoreForAccountDeletion()).thenReturn(List.of("500"));
@@ -282,13 +288,26 @@ class AccountsManagerTest {
     }
   }
 
-  private static Stream<Arguments> testDeleteWithSvrErrorStatusCodes() {
+  private static Stream<Arguments> testDeleteWithSvr2ErrorStatusCodes() {
     return Stream.of(
         Arguments.of("500", false),
         Arguments.of("429", true)
     );
   }
 
+  @ParameterizedTest
+  @ValueSource(strings = {"500", "429"})
+  void testDeleteWithSvr3ErrorStatusCodes(final String statusCode) throws InterruptedException {
+    when(svr3Client.deleteBackups(any())).thenReturn(
+        CompletableFuture.failedFuture(new SecureValueRecoveryException("Failed to delete backup", statusCode)));
+
+    final AccountAttributes attributes = new AccountAttributes(false, 1, 2, null, null, true, null);
+
+    final Account createdAccount = createAccount("+18005550123", attributes);
+
+    assertDoesNotThrow(() -> accountsManager.delete(createdAccount, AccountsManager.DeletionReason.USER_REQUEST).toCompletableFuture().join());
+  }
+
   @Test
   void testGetByServiceIdentifier() {
     final UUID aci = UUID.randomUUID();

service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java

@@ -41,6 +41,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient;
 import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.storage.DynamoDbExtensionSchema.Tables;
 import org.whispersystems.textsecuregcm.tests.util.AccountsHelper;
 import org.whispersystems.textsecuregcm.util.AttributeValues;
@@ -150,6 +151,7 @@ class AccountsManagerUsernameIntegrationTest {
         profileManager,
         mock(SecureStorageClient.class),
         mock(SecureValueRecovery2Client.class),
+        mock(SecureValueRecovery3Client.class),
         disconnectionRequestManager,
         mock(RegistrationRecoveryPasswordsManager.class),
         mock(ClientPublicKeysManager.class),

service/src/test/java/org/whispersystems/textsecuregcm/storage/AddRemoveDeviceIntegrationTest.java

@@ -41,6 +41,7 @@ import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
 import org.whispersystems.textsecuregcm.redis.RedisServerExtension;
 import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
 import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client;
+import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client;
 import org.whispersystems.textsecuregcm.tests.util.AccountsHelper;
 import org.whispersystems.textsecuregcm.tests.util.KeysHelper;
 import org.whispersystems.textsecuregcm.util.Pair;
@@ -125,6 +126,9 @@ public class AddRemoveDeviceIntegrationTest {
     final SecureValueRecovery2Client svr2Client = mock(SecureValueRecovery2Client.class);
     when(svr2Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null));
 
+    final SecureValueRecovery3Client svr3Client = mock(SecureValueRecovery3Client.class);
+    when(svr3Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null));
+
     final PhoneNumberIdentifiers phoneNumberIdentifiers =
         new PhoneNumberIdentifiers(DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(),
             DynamoDbExtensionSchema.Tables.PNI.tableName());
@@ -157,6 +161,7 @@ public class AddRemoveDeviceIntegrationTest {
         profilesManager,
         secureStorageClient,
         svr2Client,
+        svr3Client,
         mock(DisconnectionRequestManager.class),
         mock(RegistrationRecoveryPasswordsManager.class),
         clientPublicKeysManager,

service/src/test/resources/config/test.yml

@@ -213,34 +213,36 @@ svr2:
       -----END CERTIFICATE-----
 
 svr3:
-  uri: svr3.example.com
+  backend1Uri: backend1.example.com
+  backend2Uri: backend2.example.com
+  backend3Uri: backend3.example.com
   userAuthenticationTokenSharedSecret: secret://svr3.userAuthenticationTokenSharedSecret
   userIdTokenSharedSecret: secret://svr3.userIdTokenSharedSecret
   svrCaCertificates:
+    # This is a randomly generated test certificate
     - |
       -----BEGIN CERTIFICATE-----
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      MIIDazCCAlOgAwIBAgIUW5lcNWkuynRVc8Rq5pO6mHQBuZAwDQYJKoZIhvcNAQEL
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDAzMjUwMzE4MTNaFw0yOTAz
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      MjQwMzE4MTNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      AQUAA4IBDwAwggEKAoIBAQCfH4Um+fv2r4KudhD37/UXp8duRLTmp4XvpBTpDHpD
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      2HF8p2yThVKlJnMkP/9Ey1Rb0vhxO7DCltLdW8IYcxJuHoyMvyhGUEtxxkOZbrk8
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ciUR9jTZ37x7vXRGj/RxcdlS6iD0MeF0D/LAkImt4T/kiKwDbENrVEnYWJmipCKP
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ribxWky7HqxDCoYMQr0zatxB3A9mx5stH+H3kbw3CZcm+ugF9ZIKDEVHb0lf28gq
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      llmD120q/vs9YV3rzVL7sBGDqf6olkulvHQJKElZg2rdcHWFcngSlU2BjR04oyuH
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      c/SSiLSB3YB0tdFGta5uorXyV1y7RElPeBfOfvEjsG3TAgMBAAGjUzBRMB0GA1Ud
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      DgQWBBQX+xlgSWWbDjv0SrJ+h67xauJ80zAfBgNVHSMEGDAWgBQX+xlgSWWbDjv0
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      SrJ+h67xauJ80zAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAw
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ZG2MCCjscn6h/QOoJU+IDfa68OqLq0I37gMnLMde4yEhAmm//miePIq4Uz9GRJ+h
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      rAmdEnspKgyQ93PjF7Xpk/JdJA4B1bIrsOl/cSwqx2sFhRt8Kt1DHGlGWXqOaHRP
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      UkZ86MyRL3sXly6WkxEYxZJeQaOzMy2XmQh7grzrlTBuSI+0xf7vsRRDipxr6LVQ
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      6qGWyGODLLc2JD1IXj/1HpRVT2LoGGlKMuyxACQAm4oak1vvJ9mGxgfd9AU+eo58
-      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      O/esB2Eaf+QqMPELdFSZQfG2jvp+3WQTZK8fDKHyLr076G3UetEMy867F6fzTSZd
-      AAAAAAAAAAAAAAAAAAAA
+      9Kxq0DY7RCEpdHMCKcOL
       -----END CERTIFICATE-----
 
-
 messageCache: # Redis server configuration for message store cache
   persistDelayMinutes: 1
   cluster: