MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce a DynamoDB-backed remote config store

This commit is contained in:
Jon Chambers 2021-11-29 12:44:37 -05:00 committed by Jon Chambers
parent 4eb7dde1c8
commit 23bc11f3b6
10 changed files with 275 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
service/config/sample.yml
View File

@ -24,6 +24,8 @@ dynamoDbTables:
tableName: Example_Subscriptions
profiles:
tableName: Example_Profiles
remoteConfig:
tableName: Example_RemoteConfig
twilio: # Twilio gateway configuration
accountId: unset

11
service/src/main/java/org/whispersystems/textsecuregcm/configuration/DynamoDbTables.java
View File

@ -50,17 +50,20 @@ public class DynamoDbTables {
private final TableWithExpiration redeemedReceipts;
private final Table subscriptions;
private final Table profiles;
private final Table remoteConfig;
@JsonCreator
public DynamoDbTables(
@JsonProperty("issuedReceipts") final IssuedReceiptsTableConfiguration issuedReceipts,
@JsonProperty("redeemedReceipts") final TableWithExpiration redeemedReceipts,
@JsonProperty("subscriptions") final Table subscriptions,
@JsonProperty("profiles") final Table profiles) {
@JsonProperty("profiles") final Table profiles,
@JsonProperty("remoteConfig") final Table remoteConfig) {
this.issuedReceipts = issuedReceipts;
this.redeemedReceipts = redeemedReceipts;
this.subscriptions = subscriptions;
this.profiles = profiles;
this.remoteConfig = remoteConfig;
}
@Valid
@ -86,4 +89,10 @@ public class DynamoDbTables {
public Table getProfiles() {
return profiles;
}
@Valid
@NotNull
public Table getRemoteConfig() {
return remoteConfig;
}
}

17
service/src/main/java/org/whispersystems/textsecuregcm/storage/RemoteConfigStore.java Normal file
View File

@ -0,0 +1,17 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import java.util.List;
public interface RemoteConfigStore {
void set(RemoteConfig remoteConfig);
List<RemoteConfig> getAll();
void delete(String name);
}

5
service/src/main/java/org/whispersystems/textsecuregcm/storage/RemoteConfigs.java
View File

@ -15,7 +15,7 @@ import java.util.UUID;
import org.whispersystems.textsecuregcm.storage.mappers.RemoteConfigRowMapper;
import org.whispersystems.textsecuregcm.util.Constants;
public class RemoteConfigs {
public class RemoteConfigs implements RemoteConfigStore {
public static final String ID = "id";
public static final String NAME = "name";
@ -38,6 +38,7 @@ public class RemoteConfigs {
this.database.getDatabase().registerArrayType(UUID.class, "uuid");
}
@Override
public void set(RemoteConfig remoteConfig) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = setTimer.time()) {
@ -53,6 +54,7 @@ public class RemoteConfigs {
}));
}
@Override
public List<RemoteConfig> getAll() {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = getAllTimer.time()) {
@ -63,6 +65,7 @@ public class RemoteConfigs {
}));
}
@Override
public void delete(String name) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = deleteTimer.time()) {

117
service/src/main/java/org/whispersystems/textsecuregcm/storage/RemoteConfigsDynamoDb.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
public class RemoteConfigsDynamoDb implements RemoteConfigStore {
private final DynamoDbClient dynamoDbClient;
private final String tableName;
// Config name; string
static final String KEY_NAME = "N";
// Rollout percentage; integer
private static final String ATTR_PERCENTAGE = "P";
// Enrolled UUIDs (ACIs); list of byte arrays
private static final String ATTR_UUIDS = "U";
// Default value; string
private static final String ATTR_DEFAULT_VALUE = "D";
// Value when enrolled; string
private static final String ATTR_VALUE = "V";
// Hash key; string
private static final String ATTR_HASH_KEY = "H";
public RemoteConfigsDynamoDb(final DynamoDbClient dynamoDbClient, final String tableName) {
this.dynamoDbClient = dynamoDbClient;
this.tableName = tableName;
}
@Override
public void set(final RemoteConfig remoteConfig) {
final Map<String, AttributeValue> item = new HashMap<>(Map.of(
KEY_NAME, AttributeValues.fromString(remoteConfig.getName()),
ATTR_PERCENTAGE, AttributeValues.fromInt(remoteConfig.getPercentage())));
if (remoteConfig.getUuids() != null && !remoteConfig.getUuids().isEmpty()) {
final List<SdkBytes> uuidByteSets = remoteConfig.getUuids().stream()
.map(UUIDUtil::toByteBuffer)
.map(SdkBytes::fromByteBuffer)
.collect(Collectors.toList());
item.put(ATTR_UUIDS, AttributeValue.builder().bs(uuidByteSets).build());
}
if (remoteConfig.getDefaultValue() != null) {
item.put(ATTR_DEFAULT_VALUE, AttributeValues.fromString(remoteConfig.getDefaultValue()));
}
if (remoteConfig.getValue() != null) {
item.put(ATTR_VALUE, AttributeValues.fromString(remoteConfig.getValue()));
}
if (remoteConfig.getHashKey() != null) {
item.put(ATTR_HASH_KEY, AttributeValues.fromString(remoteConfig.getHashKey()));
}
dynamoDbClient.putItem(PutItemRequest.builder()
.tableName(tableName)
.item(item)
.build());
}
@Override
public List<RemoteConfig> getAll() {
return dynamoDbClient.scanPaginator(ScanRequest.builder()
.tableName(tableName)
.consistentRead(true)
.build())
.items()
.stream()
.map(item -> {
final String name = AttributeValues.getString(item, KEY_NAME, null);
final int percentage = AttributeValues.getInt(item, ATTR_PERCENTAGE, 0);
final String defaultValue = AttributeValues.getString(item, ATTR_DEFAULT_VALUE, null);
final String value = AttributeValues.getString(item, ATTR_VALUE, null);
final String hashKey = AttributeValues.getString(item, ATTR_HASH_KEY, null);
final Set<UUID> uuids;
if (item.containsKey(ATTR_UUIDS)) {
uuids = item.get(ATTR_UUIDS).bs().stream()
.map(sdkBytes -> UUIDUtil.fromByteBuffer(sdkBytes.asByteBuffer()))
.collect(Collectors.toSet());
} else {
uuids = Collections.emptySet();
}
return new RemoteConfig(name, percentage, uuids, defaultValue, value, hashKey);
})
.collect(Collectors.toList());
}
@Override
public void delete(final String name) {
dynamoDbClient.deleteItem(DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_NAME, AttributeValues.fromString(name)))
.build());
}
}

4
service/src/main/java/org/whispersystems/textsecuregcm/storage/RemoteConfigsManager.java
View File

@ -12,11 +12,11 @@ import java.util.function.Supplier;
public class RemoteConfigsManager {
private final RemoteConfigs remoteConfigs;
private final RemoteConfigStore remoteConfigs;
private final Supplier<List<RemoteConfig>> remoteConfigSupplier;
public RemoteConfigsManager(RemoteConfigs remoteConfigs) {
public RemoteConfigsManager(RemoteConfigStore remoteConfigs) {
this.remoteConfigs = remoteConfigs;
remoteConfigSupplier =

38
service/src/test/java/org/whispersystems/textsecuregcm/storage/RemoteConfigsDynamoDbTest.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.RegisterExtension;
import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
class RemoteConfigsDynamoDbTest extends RemoteConfigsTest {
private static final String REMOTE_CONFIGS_TABLE_NAME = "remote_configs_test";
@RegisterExtension
static DynamoDbExtension dynamoDbExtension = DynamoDbExtension.builder()
.tableName(REMOTE_CONFIGS_TABLE_NAME)
.hashKey(RemoteConfigsDynamoDb.KEY_NAME)
.attributeDefinition(AttributeDefinition.builder()
.attributeName(RemoteConfigsDynamoDb.KEY_NAME)
.attributeType(ScalarAttributeType.S)
.build())
.build();
private RemoteConfigsDynamoDb remoteConfigs;
@BeforeEach
void setUp() {
remoteConfigs = new RemoteConfigsDynamoDb(dynamoDbExtension.getDynamoDbClient(), REMOTE_CONFIGS_TABLE_NAME);
}
@Override
protected RemoteConfigStore getRemoteConfigStore() {
return remoteConfigs;
}
}

76
service/src/test/java/org/whispersystems/textsecuregcm/storage/RemoteConfigsManagerTest.java
View File

@ -5,62 +5,50 @@
package org.whispersystems.textsecuregcm.storage;
import com.opentable.db.postgres.embedded.LiquibasePreparer;
import com.opentable.db.postgres.junit.EmbeddedPostgresRules;
import com.opentable.db.postgres.junit.PreparedDbRule;
import org.jdbi.v3.core.Jdbi;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
import java.util.List;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
public class RemoteConfigsManagerTest {
@Rule
public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml"));
private RemoteConfigsManager remoteConfigs;
private RemoteConfigStore remoteConfigs;
private RemoteConfigsManager remoteConfigsManager;
@Before
public void setup() {
this.remoteConfigs = new RemoteConfigsManager(new RemoteConfigs(
new FaultTolerantDatabase("remote_configs-test", Jdbi.create(db.getTestDatabase()), new CircuitBreakerConfiguration())));
this.remoteConfigs = mock(RemoteConfigStore.class);
this.remoteConfigsManager = new RemoteConfigsManager(remoteConfigs);
}
@Test
public void testUpdate() {
remoteConfigs.set(new RemoteConfig("android.stickers", 50, Set.of(AuthHelper.VALID_UUID), "FALSE", "TRUE", null));
remoteConfigs.set(new RemoteConfig("value.sometimes", 50, Set.of(), "bar", "baz", null));
remoteConfigs.set(new RemoteConfig("ios.stickers", 50, Set.of(), "FALSE", "TRUE", null));
remoteConfigs.set(new RemoteConfig("ios.stickers", 75, Set.of(), "FALSE", "TRUE", null));
remoteConfigs.set(new RemoteConfig("value.sometimes", 25, Set.of(AuthHelper.VALID_UUID), "abc", "def", null));
List<RemoteConfig> results = remoteConfigs.getAll();
assertThat(results.size()).isEqualTo(3);
assertThat(results.get(0).getName()).isEqualTo("android.stickers");
assertThat(results.get(0).getPercentage()).isEqualTo(50);
assertThat(results.get(0).getUuids().size()).isEqualTo(1);
assertThat(results.get(0).getUuids().contains(AuthHelper.VALID_UUID)).isTrue();
assertThat(results.get(1).getName()).isEqualTo("ios.stickers");
assertThat(results.get(1).getPercentage()).isEqualTo(75);
assertThat(results.get(1).getUuids()).isEmpty();
assertThat(results.get(2).getName()).isEqualTo("value.sometimes");
assertThat(results.get(2).getUuids()).hasSize(1);
assertThat(results.get(2).getUuids()).contains(AuthHelper.VALID_UUID);
assertThat(results.get(2).getPercentage()).isEqualTo(25);
assertThat(results.get(2).getDefaultValue()).isEqualTo("abc");
assertThat(results.get(2).getValue()).isEqualTo("def");
public void testGetAll() {
remoteConfigsManager.getAll();
remoteConfigsManager.getAll();
// A memoized supplier should prevent multiple calls to the underlying data source
verify(remoteConfigs, times(1)).getAll();
}
@Test
public void testSet() {
final RemoteConfig remoteConfig = mock(RemoteConfig.class);
remoteConfigsManager.set(remoteConfig);
remoteConfigsManager.set(remoteConfig);
verify(remoteConfigs, times(2)).set(remoteConfig);
}
@Test
public void testDelete() {
final String name = "name";
remoteConfigsManager.delete(name);
remoteConfigsManager.delete(name);
verify(remoteConfigs, times(2)).delete(name);
}
}

38
service/src/test/java/org/whispersystems/textsecuregcm/storage/RemoteConfigsPostgresTest.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import com.opentable.db.postgres.embedded.LiquibasePreparer;
import com.opentable.db.postgres.junit5.EmbeddedPostgresExtension;
import com.opentable.db.postgres.junit5.PreparedDbExtension;
import org.jdbi.v3.core.Jdbi;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
public class RemoteConfigsPostgresTest extends RemoteConfigsTest {
@RegisterExtension
static PreparedDbExtension ACCOUNTS_POSTGRES_EXTENSION =
EmbeddedPostgresExtension.preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml"));
private RemoteConfigs remoteConfigs;
@BeforeEach
void setUp() {
final FaultTolerantDatabase remoteConfigDatabase = new FaultTolerantDatabase("remote_configs-test", Jdbi.create(ACCOUNTS_POSTGRES_EXTENSION.getTestDatabase()), new CircuitBreakerConfiguration());
remoteConfigDatabase.use(jdbi -> jdbi.useHandle(handle ->
handle.createUpdate("DELETE FROM remote_config").execute()));
this.remoteConfigs = new RemoteConfigs(remoteConfigDatabase);
}
@Override
protected RemoteConfigStore getRemoteConfigStore() {
return remoteConfigs;
}
}

49
service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/RemoteConfigsTest.java → service/src/test/java/org/whispersystems/textsecuregcm/storage/RemoteConfigsTest.java
View File

@ -1,48 +1,25 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.tests.storage;
package org.whispersystems.textsecuregcm.storage;
import org.junit.jupiter.api.Test;
import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
import java.util.List;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
import com.codahale.metrics.Timer;
import com.opentable.db.postgres.embedded.LiquibasePreparer;
import com.opentable.db.postgres.junit5.EmbeddedPostgresExtension;
import com.opentable.db.postgres.junit5.PreparedDbExtension;
import java.util.List;
import java.util.Set;
import org.jdbi.v3.core.Jdbi;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
import org.whispersystems.textsecuregcm.storage.RemoteConfig;
import org.whispersystems.textsecuregcm.storage.RemoteConfigs;
import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
abstract class RemoteConfigsTest {
public class RemoteConfigsTest {
@RegisterExtension
static PreparedDbExtension ACCOUNTS_POSTGRES_EXTENSION =
EmbeddedPostgresExtension.preparedDatabase(LiquibasePreparer.forClasspathLocation("accountsdb.xml"));
private RemoteConfigs remoteConfigs;
@BeforeEach
void setUp() {
final FaultTolerantDatabase remoteConfigDatabase = new FaultTolerantDatabase("remote_configs-test", Jdbi.create(ACCOUNTS_POSTGRES_EXTENSION.getTestDatabase()), new CircuitBreakerConfiguration());
remoteConfigDatabase.use(jdbi -> jdbi.useHandle(handle ->
handle.createUpdate("DELETE FROM remote_config").execute()));
this.remoteConfigs = new RemoteConfigs(remoteConfigDatabase);
}
protected abstract RemoteConfigStore getRemoteConfigStore();
@Test
void testStore() {
final RemoteConfigStore remoteConfigs = getRemoteConfigStore();
remoteConfigs.set(new RemoteConfig("android.stickers", 50, Set.of(AuthHelper.VALID_UUID, AuthHelper.VALID_UUID_TWO), "FALSE", "TRUE", null));
remoteConfigs.set(new RemoteConfig("value.sometimes", 25, Set.of(AuthHelper.VALID_UUID_TWO), "default", "custom", null));
@ -71,6 +48,8 @@ public class RemoteConfigsTest {
@Test
void testUpdate() {
final RemoteConfigStore remoteConfigs = getRemoteConfigStore();
remoteConfigs.set(new RemoteConfig("android.stickers", 50, Set.of(), "FALSE", "TRUE", null));
remoteConfigs.set(new RemoteConfig("value.sometimes", 22, Set.of(), "def", "!", null));
remoteConfigs.set(new RemoteConfig("ios.stickers", 50, Set.of(AuthHelper.DISABLED_UUID), "FALSE", "TRUE", null));
@ -102,6 +81,8 @@ public class RemoteConfigsTest {
@Test
void testDelete() {
final RemoteConfigStore remoteConfigs = getRemoteConfigStore();
remoteConfigs.set(new RemoteConfig("android.stickers", 50, Set.of(AuthHelper.VALID_UUID), "FALSE", "TRUE", null));
remoteConfigs.set(new RemoteConfig("ios.stickers", 50, Set.of(), "FALSE", "TRUE", null));
remoteConfigs.set(new RemoteConfig("ios.stickers", 75, Set.of(), "FALSE", "TRUE", null));