From 92f6a79e1f3958b8a5079e47e501d942e15f8971 Mon Sep 17 00:00:00 2001 From: Moxie Marlinspike Date: Mon, 1 Feb 2021 08:01:58 -0800 Subject: [PATCH] Add a dynamic configuration manager --- service/pom.xml | 9 +- .../WhisperServerConfiguration.java | 10 ++ .../textsecuregcm/WhisperServerService.java | 4 + .../configuration/AppConfigConfiguration.java | 32 +++++ .../dynamic/DynamicConfiguration.java | 5 + .../storage/DynamicConfigurationManager.java | 132 ++++++++++++++++++ .../DynamicConfigurationManagerTest.java | 43 ++++++ 7 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/AppConfigConfiguration.java create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicConfiguration.java create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/DynamicConfigurationManager.java create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamicConfigurationManagerTest.java diff --git a/service/pom.xml b/service/pom.xml index b30d64dc2..d7547cdbd 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -83,12 +83,17 @@ com.amazonaws aws-java-sdk-s3 - 1.11.366 + 1.11.939 com.amazonaws aws-java-sdk-sqs - 1.11.366 + 1.11.939 + + + com.amazonaws + aws-java-sdk-appconfig + 1.11.939 diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index 23e687748..fdf20a7c1 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -9,6 +9,7 @@ import io.dropwizard.Configuration; import io.dropwizard.client.JerseyClientConfiguration; import org.whispersystems.textsecuregcm.configuration.AccountDatabaseCrawlerConfiguration; import org.whispersystems.textsecuregcm.configuration.ApnConfiguration; +import org.whispersystems.textsecuregcm.configuration.AppConfigConfiguration; import org.whispersystems.textsecuregcm.configuration.AwsAttachmentsConfiguration; import org.whispersystems.textsecuregcm.configuration.CdnConfiguration; import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration; @@ -216,6 +217,11 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private RemoteConfigConfiguration remoteConfig; + @Valid + @NotNull + @JsonProperty + private AppConfigConfiguration appConfig; + private Map transparentDataIndex = new HashMap<>(); public RecaptchaConfiguration getRecaptchaConfiguration() { @@ -371,4 +377,8 @@ public class WhisperServerConfiguration extends Configuration { public RemoteConfigConfiguration getRemoteConfigConfiguration() { return remoteConfig; } + + public AppConfigConfiguration getAppConfig() { + return appConfig; + } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index d264e3cae..52b99786a 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -116,6 +116,7 @@ import org.whispersystems.textsecuregcm.storage.ActiveUserCounter; import org.whispersystems.textsecuregcm.storage.DirectoryManager; import org.whispersystems.textsecuregcm.storage.DirectoryReconciler; import org.whispersystems.textsecuregcm.storage.DirectoryReconciliationClient; +import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; import org.whispersystems.textsecuregcm.storage.FeatureFlags; import org.whispersystems.textsecuregcm.storage.FeatureFlagsManager; @@ -328,6 +329,8 @@ public class WhisperServerService extends Application configuration = new AtomicReference<>(); + private final AtomicBoolean running = new AtomicBoolean(true); + private final Logger logger = LoggerFactory.getLogger(DynamicConfigurationManager.class); + + private GetConfigurationResult lastConfigResult; + + private boolean initialized = false; + + public DynamicConfigurationManager(String application, String environment, String configurationName) { + this(AmazonAppConfigClient.builder() + .withClientConfiguration(new ClientConfiguration().withClientExecutionTimeout(10000).withRequestTimeout(10000)) + .build(), + application, environment, configurationName, UUID.randomUUID().toString()); + } + + @VisibleForTesting + public DynamicConfigurationManager(AmazonAppConfig appConfigClient, String application, String environment, String configurationName, String clientId) { + this.appConfigClient = appConfigClient; + this.application = application; + this.environment = environment; + this.configurationName = configurationName; + this.clientId = clientId; + } + + public DynamicConfiguration getConfiguration() { + synchronized (this) { + while (!initialized) Util.wait(this); + } + + return configuration.get(); + } + + @Override + public void start() { + configuration.set(retrieveInitialDynamicConfiguration()); + + synchronized (this) { + this.initialized = true; + this.notifyAll(); + } + + new Thread(() -> { + while (running.get()) { + try { + retrieveDynamicConfiguration().ifPresent(configuration::set); + } catch (Throwable t) { + logger.warn("Error retrieving dynamic configuration", t); + } + + Util.sleep(5000); + } + }).start(); + } + + @Override + public void stop() { + running.set(false); + } + + private Optional retrieveDynamicConfiguration() throws JsonProcessingException { + final String previousVersion = lastConfigResult != null ? lastConfigResult.getConfigurationVersion() : null; + + lastConfigResult = appConfigClient.getConfiguration(new GetConfigurationRequest().withApplication(application) + .withEnvironment(environment) + .withConfiguration(configurationName) + .withClientId(clientId) + .withClientConfigurationVersion(previousVersion)); + + final Optional maybeDynamicConfiguration; + + if (!StringUtils.equals(lastConfigResult.getConfigurationVersion(), previousVersion)) { + logger.info("Received new config version: {}", lastConfigResult.getConfigurationVersion()); + maybeDynamicConfiguration = Optional.of(mapper.readValue(StandardCharsets.UTF_8.decode(lastConfigResult.getContent().asReadOnlyBuffer()).toString(), DynamicConfiguration.class)); + } else { + // No change since last version + maybeDynamicConfiguration = Optional.empty(); + } + + return maybeDynamicConfiguration; + } + + private DynamicConfiguration retrieveInitialDynamicConfiguration() { + for (;;) { + try { + final Optional maybeDynamicConfiguration = retrieveDynamicConfiguration(); + + if (maybeDynamicConfiguration.isPresent()) { + return maybeDynamicConfiguration.get(); + } else { + throw new IllegalStateException("No initial configuration available"); + } + } catch (Throwable t) { + logger.warn("Error retrieving initial dynamic configuration", t); + Util.sleep(1000); + } + } + } +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamicConfigurationManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamicConfigurationManagerTest.java new file mode 100644 index 000000000..200df2782 --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/DynamicConfigurationManagerTest.java @@ -0,0 +1,43 @@ +package org.whispersystems.textsecuregcm.storage; + +import com.amazonaws.services.appconfig.AmazonAppConfig; +import com.amazonaws.services.appconfig.model.GetConfigurationRequest; +import com.amazonaws.services.appconfig.model.GetConfigurationResult; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.nio.ByteBuffer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DynamicConfigurationManagerTest { + + private DynamicConfigurationManager dynamicConfigurationManager; + private AmazonAppConfig appConfig; + + @Before + public void setup() { + this.appConfig = mock(AmazonAppConfig.class); + this.dynamicConfigurationManager = new DynamicConfigurationManager(appConfig, "foo", "bar", "baz", "poof"); + } + + @Test + public void testGetConfig() { + ArgumentCaptor captor = ArgumentCaptor.forClass(GetConfigurationRequest.class); + when(appConfig.getConfiguration(captor.capture())).thenReturn(new GetConfigurationResult().withContent(ByteBuffer.wrap("test: true".getBytes())) + .withConfigurationVersion("1")); + + dynamicConfigurationManager.start(); + + assertThat(captor.getValue().getApplication()).isEqualTo("foo"); + assertThat(captor.getValue().getEnvironment()).isEqualTo("bar"); + assertThat(captor.getValue().getConfiguration()).isEqualTo("baz"); + assertThat(captor.getValue().getClientId()).isEqualTo("poof"); + + assertThat(dynamicConfigurationManager.getConfiguration()).isNotNull(); + } +}