diff --git a/service/config/sample.yml b/service/config/sample.yml index 2c7292df5..88c2d1060 100644 --- a/service/config/sample.yml +++ b/service/config/sample.yml @@ -221,12 +221,6 @@ abuseDatabase: # Postgresql database configuration password: password url: jdbc:postgresql://example.com:5432/abusedb -newAbuseDatabase: # Postgresql database configuration - driverClass: org.postgresql.Driver - user: example - password: password - url: jdbc:postgresql://new.example.com:5432/abusedb - accountDatabaseCrawler: chunkSize: 10 # accounts per run chunkIntervalMs: 60000 # time per run diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index 05b7c0a4f..f60b92628 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -215,11 +215,6 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private DatabaseConfiguration abuseDatabase; - @Valid - @NotNull - @JsonProperty - private DatabaseConfiguration newAbuseDatabase; - @Valid @NotNull @JsonProperty @@ -461,10 +456,6 @@ public class WhisperServerConfiguration extends Configuration { return abuseDatabase; } - public DatabaseConfiguration getNewAbuseDatabaseConfiguration() { - return newAbuseDatabase; - } - public RateLimitsConfiguration getLimitsConfiguration() { return limits; } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 73f274981..c1cef9d4d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -214,7 +214,6 @@ import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator; import org.whispersystems.textsecuregcm.workers.CertificateCommand; import org.whispersystems.textsecuregcm.workers.CheckDynamicConfigurationCommand; import org.whispersystems.textsecuregcm.workers.DeleteUserCommand; -import org.whispersystems.textsecuregcm.workers.MigrateAbusiveHostRulesCommand; import org.whispersystems.textsecuregcm.workers.ReserveUsernameCommand; import org.whispersystems.textsecuregcm.workers.ServerVersionCommand; import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask; @@ -243,7 +242,6 @@ public class WhisperServerService extends Application("abusedb", "abusedb.xml") { @Override @@ -312,10 +310,6 @@ public class WhisperServerService extends Application getExperimentEnrollmentConfiguration( final String experimentName) { return Optional.ofNullable(experiments.get(experimentName)); @@ -118,11 +110,4 @@ public class DynamicConfiguration { return pushLatency; } - public DynamicProfileMigrationConfiguration getProfileMigrationConfiguration() { - return profileMigration; - } - - public DynamicAbusiveHostRulesMigrationConfiguration getAbusiveHostRulesMigrationConfiguration() { - return abusiveHostRulesMigration; - } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicProfileMigrationConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicProfileMigrationConfiguration.java deleted file mode 100644 index 42e4cef01..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/dynamic/DynamicProfileMigrationConfiguration.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2013-2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.configuration.dynamic; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class DynamicProfileMigrationConfiguration { - - @JsonProperty - private boolean dynamoDbDeleteEnabled = false; - - @JsonProperty - private boolean dynamoDbWriteEnabled = false; - - @JsonProperty - private boolean dynamoDbReadForComparisonEnabled = false; - - @JsonProperty - private boolean dynamoDbReadPrimary = false; - - @JsonProperty - private boolean logMismatches = false; - - public boolean isDynamoDbDeleteEnabled() { - return dynamoDbDeleteEnabled; - } - - public boolean isDynamoDbWriteEnabled() { - return dynamoDbWriteEnabled; - } - - public boolean isDynamoDbReadForComparisonEnabled() { - return dynamoDbReadForComparisonEnabled; - } - - public boolean isDynamoDbReadPrimary() { - return dynamoDbReadPrimary; - } - - public boolean isLogMismatches() { - return logMismatches; - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRules.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRules.java index bd4d39d96..781acd49c 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRules.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRules.java @@ -10,17 +10,11 @@ import static com.codahale.metrics.MetricRegistry.name; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.SharedMetricRegistries; import com.codahale.metrics.Timer; -import com.google.common.base.Suppliers; import java.util.List; -import java.util.function.BiConsumer; -import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; -import org.whispersystems.textsecuregcm.experiment.Experiment; import org.whispersystems.textsecuregcm.storage.mappers.AbusiveHostRuleRowMapper; import org.whispersystems.textsecuregcm.util.Constants; -import org.whispersystems.textsecuregcm.util.Pair; public class AbusiveHostRules { @@ -36,26 +30,16 @@ public class AbusiveHostRules { private final Timer getTimer = metricRegistry.timer(name(AbusiveHostRules.class, "get")); private final Timer insertTimer = metricRegistry.timer(name(AbusiveHostRules.class, "setBlockedHost")); - private final FaultTolerantDatabase oldDatabase; - private final FaultTolerantDatabase newDatabase; + private final FaultTolerantDatabase database; - private final DynamicConfigurationManager dynamicConfigurationManager; - private final Experiment migrationExperiment = new Experiment("abusiveHostRulesMigration"); + public AbusiveHostRules(FaultTolerantDatabase database) { - public AbusiveHostRules(FaultTolerantDatabase oldDatabase, FaultTolerantDatabase newDatabase, - DynamicConfigurationManager dynamicConfigurationManager) { - - this.oldDatabase = oldDatabase; - this.oldDatabase.getDatabase().registerRowMapper(new AbusiveHostRuleRowMapper()); - - this.newDatabase = newDatabase; - this.newDatabase.getDatabase().registerRowMapper(new AbusiveHostRuleRowMapper()); - - this.dynamicConfigurationManager = dynamicConfigurationManager; + this.database = database; + this.database.getDatabase().registerRowMapper(new AbusiveHostRuleRowMapper()); } public List getAbusiveHostRulesFor(String host) { - final List oldDbRules = oldDatabase.with(jdbi -> jdbi.withHandle(handle -> { + return database.with(jdbi -> jdbi.withHandle(handle -> { try (Timer.Context timer = getTimer.time()) { return handle.createQuery("SELECT * FROM abusive_host_rules WHERE :host::inet <<= " + HOST) .bind("host", host) @@ -63,25 +47,10 @@ public class AbusiveHostRules { .list(); } })); - - final Supplier> newDbRules = Suppliers.memoize( - () -> newDatabase.with(jdbi -> jdbi.withHandle( - handle -> handle.createQuery("SELECT * FROM abusive_host_rules WHERE :host::inet <<= " + HOST) - .bind("host", host) - .mapTo(AbusiveHostRule.class) - .list()))); - - if (dynamicConfigurationManager.getConfiguration().getAbusiveHostRulesMigrationConfiguration().isNewReadEnabled()) { - migrationExperiment.compareSupplierResult(oldDbRules, newDbRules); - } - - return dynamicConfigurationManager.getConfiguration().getAbusiveHostRulesMigrationConfiguration().isNewPrimary() - ? newDbRules.get() - : oldDbRules; } public void setBlockedHost(String host, String notes) { - oldDatabase.use(jdbi -> jdbi.useHandle(handle -> { + database.use(jdbi -> jdbi.useHandle(handle -> { try (Timer.Context timer = insertTimer.time()) { handle.createUpdate( "INSERT INTO abusive_host_rules(host, blocked, notes) VALUES(:host::inet, :blocked, :notes) ON CONFLICT DO NOTHING") @@ -91,47 +60,6 @@ public class AbusiveHostRules { .execute(); } })); - - if (dynamicConfigurationManager.getConfiguration().getAbusiveHostRulesMigrationConfiguration() - .isNewWriteEnabled()) { - try { - newDatabase.use(jdbi -> jdbi.useHandle(handle -> handle.createUpdate( - "INSERT INTO abusive_host_rules(host, blocked, notes) VALUES(:host::inet, :blocked, :notes) ON CONFLICT DO NOTHING") - .bind("host", host) - .bind("blocked", 1) - .bind("notes", notes) - .execute())); - } catch (final Exception e) { - logger.warn("Failed to insert rule in new database", e); - } - } } - public int migrateAbusiveHostRule(AbusiveHostRule rule, String notes) { - return newDatabase.with(jdbi -> jdbi.withHandle(handle -> { - try (Timer.Context timer = insertTimer.time()) { - return handle.createUpdate( - "INSERT INTO abusive_host_rules(host, blocked, notes, regions) VALUES(:host::inet, :blocked, :notes, :regions) ON CONFLICT DO NOTHING") - .bind("host", rule.host()) - .bind("blocked", rule.blocked() ? 1 : 0) - .bind("notes", notes) - .bind("regions", String.join(",", rule.regions())) - .execute(); - } - })); - } - - public void forEachInOldDatabase(final BiConsumer consumer, final int fetchSize) { - final AbusiveHostRuleRowMapper rowMapper = new AbusiveHostRuleRowMapper(); - - oldDatabase.use(jdbi -> jdbi.useHandle(handle -> handle.useTransaction(transactionHandle -> - transactionHandle.createQuery("SELECT * FROM abusive_host_rules") - .setFetchSize(fetchSize) - .map((resultSet, ctx) -> { - AbusiveHostRule rule = rowMapper.map(resultSet, ctx); - String notes = resultSet.getString(NOTES); - return new Pair<>(rule, notes); - }) - .forEach(pair -> consumer.accept(pair.first(), pair.second()))))); - } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/MigrateAbusiveHostRulesCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/MigrateAbusiveHostRulesCommand.java deleted file mode 100644 index c819e1484..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/MigrateAbusiveHostRulesCommand.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2021 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.workers; - -import com.codahale.metrics.jdbi3.strategies.DefaultNameStrategy; -import io.dropwizard.Application; -import io.dropwizard.cli.EnvironmentCommand; -import io.dropwizard.jdbi3.JdbiFactory; -import io.dropwizard.setup.Environment; -import net.sourceforge.argparse4j.inf.Namespace; -import net.sourceforge.argparse4j.inf.Subparser; -import org.jdbi.v3.core.Jdbi; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.whispersystems.textsecuregcm.WhisperServerConfiguration; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; -import org.whispersystems.textsecuregcm.storage.AbusiveHostRules; -import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; -import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; -import java.util.concurrent.atomic.AtomicInteger; - -public class MigrateAbusiveHostRulesCommand extends EnvironmentCommand { - - private static final Logger log = LoggerFactory.getLogger(MigrateAbusiveHostRulesCommand.class); - - public MigrateAbusiveHostRulesCommand() { - super(new Application<>() { - @Override - public void run(WhisperServerConfiguration configuration, Environment environment) { - } - }, "migrate-abusive-host-rules", "Migrate abusive host rules from one Postgres to another"); - } - - @Override - public void configure(Subparser subparser) { - super.configure(subparser); - - subparser.addArgument("-s", "--fetch-size") - .dest("fetchSize") - .type(Integer.class) - .required(false) - .setDefault(512) - .help("The number of rules to fetch from Postgres at once"); - } - - @Override - protected void run(final Environment environment, final Namespace namespace, - final WhisperServerConfiguration config) throws Exception { - - DynamicConfigurationManager dynamicConfigurationManager = - new DynamicConfigurationManager<>(config.getAppConfig().getApplication(), - config.getAppConfig().getEnvironment(), - config.getAppConfig().getConfigurationName(), - DynamicConfiguration.class); - - JdbiFactory jdbiFactory = new JdbiFactory(DefaultNameStrategy.CHECK_EMPTY); - Jdbi abuseJdbi = jdbiFactory.build(environment, config.getAbuseDatabaseConfiguration(), "abusedb"); - - FaultTolerantDatabase abuseDatabase = new FaultTolerantDatabase("abuse_database", abuseJdbi, - config.getAbuseDatabaseConfiguration().getCircuitBreakerConfiguration()); - - Jdbi newAbuseJdbi = jdbiFactory.build(environment, config.getNewAbuseDatabaseConfiguration(), "abusedb2"); - FaultTolerantDatabase newAbuseDatabase = new FaultTolerantDatabase("abuse_database2", newAbuseJdbi, - config.getNewAbuseDatabaseConfiguration().getCircuitBreakerConfiguration()); - - log.info("Beginning migration"); - - AbusiveHostRules abusiveHostRules = new AbusiveHostRules(abuseDatabase, newAbuseDatabase, - dynamicConfigurationManager); - - final int fetchSize = namespace.getInt("fetchSize"); - - final AtomicInteger rulesMigrated = new AtomicInteger(0); - - abusiveHostRules.forEachInOldDatabase((rule, notes) -> { - - abusiveHostRules.migrateAbusiveHostRule(rule, notes); - - int migrated = rulesMigrated.incrementAndGet(); - - if (migrated % 1_000 == 0) { - log.info("Migrated {} rules", migrated); - } - }, fetchSize); - - log.info("Migration complete ({} total rules)", rulesMigrated.get()); - } - -} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AbusiveHostRulesTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AbusiveHostRulesTest.java index 09b23399b..b86558b45 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AbusiveHostRulesTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AbusiveHostRulesTest.java @@ -6,8 +6,6 @@ package org.whispersystems.textsecuregcm.tests.storage; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import io.zonky.test.db.postgres.embedded.LiquibasePreparer; import io.zonky.test.db.postgres.junit5.EmbeddedPostgresExtension; @@ -22,11 +20,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicAbusiveHostRulesMigrationConfiguration; -import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.storage.AbusiveHostRule; import org.whispersystems.textsecuregcm.storage.AbusiveHostRules; -import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; class AbusiveHostRulesTest { @@ -43,20 +38,9 @@ class AbusiveHostRulesTest { @BeforeEach void setup() { - //noinspection unchecked - final DynamicConfigurationManager dynamicConfigurationManager = mock( - DynamicConfigurationManager.class); - final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class); - when(dynamicConfigurationManager.getConfiguration()).thenReturn(dynamicConfiguration); - when(dynamicConfiguration.getAbusiveHostRulesMigrationConfiguration()).thenReturn( - new DynamicAbusiveHostRulesMigrationConfiguration()); - this.abusiveHostRules = new AbusiveHostRules( new FaultTolerantDatabase("abusive_hosts-test", Jdbi.create(db.getTestDatabase()), - new CircuitBreakerConfiguration()), - new FaultTolerantDatabase("abusive_hosts-test", Jdbi.create(newDb.getTestDatabase()), - new CircuitBreakerConfiguration()), - dynamicConfigurationManager); + new CircuitBreakerConfiguration())); } @Test @@ -147,30 +131,4 @@ class AbusiveHostRulesTest { assertThat(resultSet.getString("notes")).isEqualTo("Testing one two"); } - @Test - void testMigrate() throws Exception { - final int rules = 20; - for (int i = 1; i <= rules; i++) { - abusiveHostRules.setBlockedHost("172.17.0." + i, "Testing one two " + i); - } - - PreparedStatement statement = newDb.getTestDatabase().getConnection() - .prepareStatement("SELECT * from abusive_host_rules"); - - assertThat(queryResultSize(statement.executeQuery())).isEqualTo(0); - - abusiveHostRules.forEachInOldDatabase((rule, host) -> abusiveHostRules.migrateAbusiveHostRule(rule, host), 5); - - assertThat(queryResultSize(statement.executeQuery())).isEqualTo(rules); - } - - private int queryResultSize(ResultSet resultSet) throws SQLException { - int migrated = 0; - while (resultSet.next()) { - migrated++; - } - - return migrated; - } - }