diff --git a/pom.xml b/pom.xml index c5a1f6893..f9393e163 100644 --- a/pom.xml +++ b/pom.xml @@ -61,7 +61,6 @@ 4.3.1 4.1.65.Final 1.2.0 - 42.3.3 3.19.4 0.15.1 1.5.0 @@ -224,12 +223,6 @@ ${opentest4j.version} test - - org.postgresql - postgresql - ${postgresql.version} - runtime - org.slf4j slf4j-api @@ -278,13 +271,6 @@ libsignal-server 0.16.0 - - io.zonky.test.postgres - embedded-postgres-binaries-bom - 11.13.0 - pom - import - org.apache.logging.log4j log4j-bom diff --git a/service/config/sample.yml b/service/config/sample.yml index faf683410..26b0f34fc 100644 --- a/service/config/sample.yml +++ b/service/config/sample.yml @@ -185,12 +185,6 @@ gcpAttachments: # GCP Storage configuration AAAAAAAA -----END PRIVATE KEY----- -abuseDatabase: # Postgresql database configuration - driverClass: org.postgresql.Driver - user: example - password: password - url: jdbc:postgresql://example.com:5432/abusedb - accountDatabaseCrawler: chunkSize: 10 # accounts per run chunkIntervalMs: 60000 # time per run diff --git a/service/pom.xml b/service/pom.xml index 5f9ddaada..7088e27d2 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -128,11 +128,6 @@ jdbi3-core - - org.liquibase - liquibase-core - - io.dropwizard.metrics metrics-core @@ -305,12 +300,6 @@ lettuce-core - - org.postgresql - postgresql - runtime - - com.eatthepath pushy @@ -372,13 +361,6 @@ - - io.zonky.test - embedded-postgres - 1.3.1 - test - - com.almworks.sqlite4java sqlite4java diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index 54d86ba31..3d76a4f99 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -143,11 +143,6 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private RedisClusterConfiguration clientPresenceCluster; - @Valid - @NotNull - @JsonProperty - private DatabaseConfiguration abuseDatabase; - @Valid @NotNull @JsonProperty @@ -337,10 +332,6 @@ public class WhisperServerConfiguration extends Configuration { return rateLimitersCluster; } - public DatabaseConfiguration getAbuseDatabaseConfiguration() { - return abuseDatabase; - } - public RateLimitsConfiguration getLimitsConfiguration() { return limits; } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 5b89060c0..124a48a32 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -114,7 +114,6 @@ import org.whispersystems.textsecuregcm.limits.PushChallengeManager; import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager; import org.whispersystems.textsecuregcm.limits.RateLimitChallengeOptionManager; import org.whispersystems.textsecuregcm.limits.RateLimiters; -import org.whispersystems.textsecuregcm.liquibase.NameableMigrationsBundle; import org.whispersystems.textsecuregcm.mappers.CompletionExceptionMapper; import org.whispersystems.textsecuregcm.mappers.DeviceLimitExceededExceptionMapper; import org.whispersystems.textsecuregcm.mappers.IOExceptionMapper; @@ -244,13 +243,6 @@ public class WhisperServerService extends Application("abusedb", "abusedb.xml") { - @Override - public PooledDataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) { - return configuration.getAbuseDatabaseConfiguration(); - } - }); } @Override @@ -308,12 +300,6 @@ public class WhisperServerService extends Application getExperimentEnrollmentConfiguration( final String experimentName) { return Optional.ofNullable(experiments.get(experimentName)); @@ -117,4 +121,7 @@ public class DynamicConfiguration { return turn; } + public DynamicAbusiveHostRulesConfiguration getAbusiveHostRules() { + return abusiveHostRules; + } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/AccountController.java b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/AccountController.java index 5af27d312..2ae19bbe9 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/AccountController.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/AccountController.java @@ -82,7 +82,6 @@ import org.whispersystems.textsecuregcm.push.GcmMessage; import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient; import org.whispersystems.textsecuregcm.sms.SmsSender; import org.whispersystems.textsecuregcm.sms.TwilioVerifyExperimentEnrollmentManager; -import org.whispersystems.textsecuregcm.storage.AbusiveHostRule; import org.whispersystems.textsecuregcm.storage.AbusiveHostRules; import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.AccountsManager; @@ -108,9 +107,7 @@ public class AccountController { private final Logger logger = LoggerFactory.getLogger(AccountController.class); private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME); private final Meter blockedHostMeter = metricRegistry.meter(name(AccountController.class, "blocked_host" )); - private final Meter blockedPrefixMeter = metricRegistry.meter(name(AccountController.class, "blocked_prefix" )); private final Meter countryFilterApplicable = metricRegistry.meter(name(AccountController.class, "country_filter_applicable")); - private final Meter filteredHostMeter = metricRegistry.meter(name(AccountController.class, "filtered_host" )); private final Meter countryFilteredHostMeter = metricRegistry.meter(name(AccountController.class, "country_limited_host" )); private final Meter rateLimitedHostMeter = metricRegistry.meter(name(AccountController.class, "rate_limited_host" )); private final Meter rateLimitedPrefixMeter = metricRegistry.meter(name(AccountController.class, "rate_limited_prefix" )); @@ -246,7 +243,7 @@ public class AccountController { if (requirement.isAutoBlock() && shouldAutoBlock(sourceHost)) { logger.info("Auto-block: {}", sourceHost); - abusiveHostRules.setBlockedHost(sourceHost, "Auto-Block"); + abusiveHostRules.setBlockedHost(sourceHost); } return Response.status(402).build(); @@ -780,7 +777,10 @@ public class AccountController { DynamicCaptchaConfiguration captchaConfig = dynamicConfigurationManager.getConfiguration() .getCaptchaConfiguration(); boolean countryFiltered = captchaConfig.getSignupCountryCodes().contains(countryCode); - if (shouldBlock(transport, forwardedFor, sourceHost, number)) { + + if (abusiveHostRules.isBlocked(sourceHost)) { + blockedHostMeter.mark(); + logger.info("Blocked host: {}, {}, {} ({})", transport, number, sourceHost, forwardedFor); if (countryFiltered) { // this host was caught in the abusiveHostRules filter, but // would be caught by country filter as well @@ -813,33 +813,6 @@ public class AccountController { return new CaptchaRequirement(false, false); } - private boolean shouldBlock(final String transport, final String forwardedFor, final String sourceHost, final String number) { - List abuseRules = abusiveHostRules.getAbusiveHostRulesFor(sourceHost); - - for (AbusiveHostRule abuseRule : abuseRules) { - if (abuseRule.blocked()) { - logger.info("Blocked host: {}, {}, {} ({}) matched rule: {}", transport, number, sourceHost, forwardedFor, abuseRule.host()); - - // did we match based on an ip block or an exact match - if (abuseRule.cidrPrefix().filter(i -> i < 32).isPresent()) { - blockedPrefixMeter.mark(); - } else { - blockedHostMeter.mark(); - } - return true; - } - - if (!abuseRule.regions().isEmpty()) { - if (abuseRule.regions().stream().noneMatch(number::startsWith)) { - logger.info("Restricted host: {}, {}, {} ({}) matched rule: {}/{}", transport, number, sourceHost, forwardedFor, abuseRule.host(), abuseRule.regions()); - filteredHostMeter.mark(); - return true; - } - } - } - return false; - } - @Timed @DELETE @Path("/me") diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/AbstractLiquibaseCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/AbstractLiquibaseCommand.java deleted file mode 100644 index e3de6627d..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/AbstractLiquibaseCommand.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.liquibase; - -import com.codahale.metrics.MetricRegistry; -import net.sourceforge.argparse4j.inf.Namespace; - -import java.sql.SQLException; - -import io.dropwizard.Configuration; -import io.dropwizard.cli.ConfiguredCommand; -import io.dropwizard.db.DatabaseConfiguration; -import io.dropwizard.db.ManagedDataSource; -import io.dropwizard.db.PooledDataSourceFactory; -import io.dropwizard.setup.Bootstrap; -import liquibase.Liquibase; -import liquibase.exception.LiquibaseException; -import liquibase.exception.ValidationFailedException; - -public abstract class AbstractLiquibaseCommand extends ConfiguredCommand { - - private final DatabaseConfiguration strategy; - private final Class configurationClass; - private final String migrations; - - protected AbstractLiquibaseCommand(String name, - String description, - String migrations, - DatabaseConfiguration strategy, - Class configurationClass) { - super(name, description); - this.migrations = migrations; - this.strategy = strategy; - this.configurationClass = configurationClass; - } - - @Override - protected Class getConfigurationClass() { - return configurationClass; - } - - @Override - @SuppressWarnings("UseOfSystemOutOrSystemErr") - protected void run(Bootstrap bootstrap, Namespace namespace, T configuration) throws Exception { - final PooledDataSourceFactory dbConfig = strategy.getDataSourceFactory(configuration); - dbConfig.asSingleConnectionPool(); - - try (final CloseableLiquibase liquibase = openLiquibase(dbConfig, namespace)) { - run(namespace, liquibase); - } catch (ValidationFailedException e) { - e.printDescriptiveError(System.err); - throw e; - } - } - - private CloseableLiquibase openLiquibase(final PooledDataSourceFactory dataSourceFactory, final Namespace namespace) - throws ClassNotFoundException, SQLException, LiquibaseException - { - final ManagedDataSource dataSource = dataSourceFactory.build(new MetricRegistry(), "liquibase"); - return new CloseableLiquibase(dataSource, migrations); - } - - protected abstract void run(Namespace namespace, Liquibase liquibase) throws Exception; - -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/CloseableLiquibase.java b/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/CloseableLiquibase.java deleted file mode 100644 index a377a6b63..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/CloseableLiquibase.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.liquibase; - -import java.sql.SQLException; - -import io.dropwizard.db.ManagedDataSource; -import liquibase.Liquibase; -import liquibase.database.jvm.JdbcConnection; -import liquibase.exception.LiquibaseException; -import liquibase.resource.ClassLoaderResourceAccessor; - - -public class CloseableLiquibase extends Liquibase implements AutoCloseable { - private final ManagedDataSource dataSource; - - public CloseableLiquibase(ManagedDataSource dataSource, String migrations) - throws LiquibaseException, ClassNotFoundException, SQLException - { - super(migrations, - new ClassLoaderResourceAccessor(), - new JdbcConnection(dataSource.getConnection())); - this.dataSource = dataSource; - } - - @Override - public void close() throws Exception { - dataSource.stop(); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/DbMigrateCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/DbMigrateCommand.java deleted file mode 100644 index 88b9a39ab..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/DbMigrateCommand.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.liquibase; - - -import com.google.common.base.Charsets; -import com.google.common.base.Joiner; -import net.sourceforge.argparse4j.impl.Arguments; -import net.sourceforge.argparse4j.inf.Namespace; -import net.sourceforge.argparse4j.inf.Subparser; - -import java.io.OutputStreamWriter; -import java.util.List; - -import io.dropwizard.Configuration; -import io.dropwizard.db.DatabaseConfiguration; -import liquibase.Liquibase; - -public class DbMigrateCommand extends AbstractLiquibaseCommand { - - public DbMigrateCommand(String migration, DatabaseConfiguration strategy, Class configurationClass) { - super("migrate", "Apply all pending change sets.", migration, strategy, configurationClass); - } - - @Override - public void configure(Subparser subparser) { - super.configure(subparser); - - subparser.addArgument("-n", "--dry-run") - .action(Arguments.storeTrue()) - .dest("dry-run") - .setDefault(Boolean.FALSE) - .help("output the DDL to stdout, don't run it"); - - subparser.addArgument("-c", "--count") - .type(Integer.class) - .dest("count") - .help("only apply the next N change sets"); - - subparser.addArgument("-i", "--include") - .action(Arguments.append()) - .dest("contexts") - .help("include change sets from the given context"); - } - - @Override - @SuppressWarnings("UseOfSystemOutOrSystemErr") - public void run(Namespace namespace, Liquibase liquibase) throws Exception { - final String context = getContext(namespace); - final Integer count = namespace.getInt("count"); - final Boolean dryRun = namespace.getBoolean("dry-run"); - if (count != null) { - if (dryRun) { - liquibase.update(count, context, new OutputStreamWriter(System.out, Charsets.UTF_8)); - } else { - liquibase.update(count, context); - } - } else { - if (dryRun) { - liquibase.update(context, new OutputStreamWriter(System.out, Charsets.UTF_8)); - } else { - liquibase.update(context); - } - } - } - - private String getContext(Namespace namespace) { - final List contexts = namespace.getList("contexts"); - if (contexts == null) { - return ""; - } - return Joiner.on(',').join(contexts); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/DbStatusCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/DbStatusCommand.java deleted file mode 100644 index 4689ef0a4..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/DbStatusCommand.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.liquibase; - -import com.google.common.base.Charsets; -import com.google.common.base.Joiner; -import net.sourceforge.argparse4j.impl.Arguments; -import net.sourceforge.argparse4j.inf.Namespace; -import net.sourceforge.argparse4j.inf.Subparser; - -import java.io.OutputStreamWriter; -import java.util.List; - -import io.dropwizard.Configuration; -import io.dropwizard.db.DatabaseConfiguration; -import liquibase.Liquibase; - -public class DbStatusCommand extends AbstractLiquibaseCommand { - - public DbStatusCommand(String migrations, DatabaseConfiguration strategy, Class configurationClass) { - super("status", "Check for pending change sets.", migrations, strategy, configurationClass); - } - - @Override - public void configure(Subparser subparser) { - super.configure(subparser); - - subparser.addArgument("-v", "--verbose") - .action(Arguments.storeTrue()) - .dest("verbose") - .help("Output verbose information"); - subparser.addArgument("-i", "--include") - .action(Arguments.append()) - .dest("contexts") - .help("include change sets from the given context"); - } - - @Override - @SuppressWarnings("UseOfSystemOutOrSystemErr") - public void run(Namespace namespace, Liquibase liquibase) throws Exception { - liquibase.reportStatus(namespace.getBoolean("verbose"), - getContext(namespace), - new OutputStreamWriter(System.out, Charsets.UTF_8)); - } - - private String getContext(Namespace namespace) { - final List contexts = namespace.getList("contexts"); - if (contexts == null) { - return ""; - } - return Joiner.on(',').join(contexts); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/NameableDbCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/NameableDbCommand.java deleted file mode 100644 index e6c91904d..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/NameableDbCommand.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.liquibase; - -import com.google.common.collect.Maps; -import net.sourceforge.argparse4j.inf.Namespace; -import net.sourceforge.argparse4j.inf.Subparser; - -import java.util.SortedMap; - -import io.dropwizard.Configuration; -import io.dropwizard.db.DatabaseConfiguration; -import liquibase.Liquibase; - -public class NameableDbCommand extends AbstractLiquibaseCommand { - private static final String COMMAND_NAME_ATTR = "subcommand"; - private final SortedMap> subcommands; - - public NameableDbCommand(String name, String migrations, DatabaseConfiguration strategy, Class configurationClass) { - super(name, "Run database migrations tasks", migrations, strategy, configurationClass); - this.subcommands = Maps.newTreeMap(); - addSubcommand(new DbMigrateCommand<>(migrations, strategy, configurationClass)); - addSubcommand(new DbStatusCommand<>(migrations, strategy, configurationClass)); - } - - private void addSubcommand(AbstractLiquibaseCommand subcommand) { - subcommands.put(subcommand.getName(), subcommand); - } - - @Override - public void configure(Subparser subparser) { - for (AbstractLiquibaseCommand subcommand : subcommands.values()) { - final Subparser cmdParser = subparser.addSubparsers() - .addParser(subcommand.getName()) - .setDefault(COMMAND_NAME_ATTR, subcommand.getName()) - .description(subcommand.getDescription()); - subcommand.configure(cmdParser); - } - } - - @Override - public void run(Namespace namespace, Liquibase liquibase) throws Exception { - final AbstractLiquibaseCommand subcommand = subcommands.get(namespace.getString(COMMAND_NAME_ATTR)); - subcommand.run(namespace, liquibase); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/NameableMigrationsBundle.java b/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/NameableMigrationsBundle.java deleted file mode 100644 index 7a96ec77d..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/liquibase/NameableMigrationsBundle.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.liquibase; - -import io.dropwizard.Bundle; -import io.dropwizard.Configuration; -import io.dropwizard.db.DatabaseConfiguration; -import io.dropwizard.setup.Bootstrap; -import io.dropwizard.setup.Environment; -import io.dropwizard.util.Generics; - -public abstract class NameableMigrationsBundle implements Bundle, DatabaseConfiguration { - - private final String name; - private final String migrations; - - public NameableMigrationsBundle(String name, String migrations) { - this.name = name; - this.migrations = migrations; - } - - public final void initialize(Bootstrap bootstrap) { - Class klass = Generics.getTypeParameter(this.getClass(), Configuration.class); - bootstrap.addCommand(new NameableDbCommand(name, migrations, this, klass)); - } - - public final void run(Environment environment) { - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRule.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRule.java deleted file mode 100644 index 4a93165bb..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRule.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage; - -import java.util.List; -import java.util.Optional; - -public record AbusiveHostRule(String host, boolean blocked, List regions) { - - public Optional cidrPrefix() { - String[] split = host.split("/"); - if (split.length != 2) { - return Optional.empty(); - } - try { - return Optional.of(Integer.parseInt(split[1])); - } catch (NumberFormatException e) { - return Optional.empty(); - } - } - -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRules.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRules.java index 4c88a169b..45b2c77ed 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRules.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AbusiveHostRules.java @@ -10,52 +10,43 @@ import static com.codahale.metrics.MetricRegistry.name; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.SharedMetricRegistries; import com.codahale.metrics.Timer; -import java.util.List; -import org.whispersystems.textsecuregcm.storage.mappers.AbusiveHostRuleRowMapper; +import java.time.Duration; +import com.google.common.annotations.VisibleForTesting; +import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; +import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster; import org.whispersystems.textsecuregcm.util.Constants; public class AbusiveHostRules { - public static final String ID = "id"; - public static final String HOST = "host"; - public static final String BLOCKED = "blocked"; - public static final String REGIONS = "regions"; - public static final String NOTES = "notes"; - + private static final String KEY_PREFIX = "abusive_hosts::"; private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME); private final Timer getTimer = metricRegistry.timer(name(AbusiveHostRules.class, "get")); private final Timer insertTimer = metricRegistry.timer(name(AbusiveHostRules.class, "setBlockedHost")); - private final FaultTolerantDatabase database; + private final FaultTolerantRedisCluster redisCluster; + private final DynamicConfigurationManager configurationManager; - public AbusiveHostRules(FaultTolerantDatabase database) { - - this.database = database; - this.database.getDatabase().registerRowMapper(new AbusiveHostRuleRowMapper()); + public AbusiveHostRules(FaultTolerantRedisCluster redisCluster, final DynamicConfigurationManager configurationManager) { + this.redisCluster = redisCluster; + this.configurationManager = configurationManager; } - public List getAbusiveHostRulesFor(String host) { - return database.with(jdbi -> jdbi.withHandle(handle -> { - try (Timer.Context timer = getTimer.time()) { - return handle.createQuery("SELECT * FROM abusive_host_rules WHERE :host::inet <<= " + HOST) - .bind("host", host) - .mapTo(AbusiveHostRule.class) - .list(); - } - })); + public boolean isBlocked(String host) { + try (Timer.Context timer = getTimer.time()) { + return this.redisCluster.withCluster(connection -> connection.sync().exists(prefix(host))) > 0; + } } - public void setBlockedHost(String host, String notes) { - database.use(jdbi -> jdbi.useHandle(handle -> { - try (Timer.Context timer = insertTimer.time()) { - handle.createUpdate( - "INSERT INTO abusive_host_rules(host, blocked, notes) VALUES(:host::inet, :blocked, :notes) ON CONFLICT DO NOTHING") - .bind("host", host) - .bind("blocked", 1) - .bind("notes", notes) - .execute(); - } - })); + public void setBlockedHost(String host) { + Duration expireTime = configurationManager.getConfiguration().getAbusiveHostRules().getExpirationTime(); + try (Timer.Context timer = insertTimer.time()) { + this.redisCluster.useCluster(connection -> connection.sync().setex(prefix(host), expireTime.toSeconds(), "1")); + } + } + + @VisibleForTesting + public static String prefix(String keyName) { + return KEY_PREFIX + keyName; } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/mappers/AbusiveHostRuleRowMapper.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/mappers/AbusiveHostRuleRowMapper.java deleted file mode 100644 index a00e04004..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/mappers/AbusiveHostRuleRowMapper.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.storage.mappers; - -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; -import org.whispersystems.textsecuregcm.storage.AbusiveHostRule; -import org.whispersystems.textsecuregcm.storage.AbusiveHostRules; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; - - -public class AbusiveHostRuleRowMapper implements RowMapper { - @Override - public AbusiveHostRule map(ResultSet resultSet, StatementContext ctx) throws SQLException { - String regionsData = resultSet.getString(AbusiveHostRules.REGIONS); - - List regions; - - if (regionsData == null) regions = new LinkedList<>(); - else regions = Arrays.asList(regionsData.split(",")); - - - return new AbusiveHostRule(resultSet.getString(AbusiveHostRules.HOST), resultSet.getInt(AbusiveHostRules.BLOCKED) == 1, regions); - } -} diff --git a/service/src/main/resources/abusedb.xml b/service/src/main/resources/abusedb.xml deleted file mode 100644 index e03840ef8..000000000 --- a/service/src/main/resources/abusedb.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AccountControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AccountControllerTest.java index ae8b078cd..2f93b6c31 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AccountControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AccountControllerTest.java @@ -87,7 +87,6 @@ import org.whispersystems.textsecuregcm.push.GcmMessage; import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient; import org.whispersystems.textsecuregcm.sms.SmsSender; import org.whispersystems.textsecuregcm.sms.TwilioVerifyExperimentEnrollmentManager; -import org.whispersystems.textsecuregcm.storage.AbusiveHostRule; import org.whispersystems.textsecuregcm.storage.AbusiveHostRules; import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.AccountsManager; @@ -113,12 +112,13 @@ class AccountControllerTest { private static final String SENDER_REG_LOCK = "+14158888888"; private static final String SENDER_HAS_STORAGE = "+14159999999"; private static final String SENDER_TRANSFER = "+14151111112"; + private static final String RESTRICTED_COUNTRY = "800"; + private static final String RESTRICTED_NUMBER = "+" + RESTRICTED_COUNTRY + "11111111"; private static final UUID SENDER_REG_LOCK_UUID = UUID.randomUUID(); private static final UUID SENDER_TRANSFER_UUID = UUID.randomUUID(); private static final String ABUSIVE_HOST = "192.168.1.1"; - private static final String RESTRICTED_HOST = "192.168.1.2"; private static final String NICE_HOST = "127.0.0.1"; private static final String RATE_LIMITED_IP_HOST = "10.0.0.1"; private static final String RATE_LIMITED_PREFIX_HOST = "10.0.0.2"; @@ -276,12 +276,12 @@ class AccountControllerTest { .thenReturn(dynamicConfiguration); DynamicCaptchaConfiguration signupCaptchaConfig = new DynamicCaptchaConfiguration(); + signupCaptchaConfig.setSignupCountryCodes(Set.of(RESTRICTED_COUNTRY)); when(dynamicConfiguration.getCaptchaConfiguration()).thenReturn(signupCaptchaConfig); } - when(abusiveHostRules.getAbusiveHostRulesFor(eq(ABUSIVE_HOST))).thenReturn(Collections.singletonList(new AbusiveHostRule(ABUSIVE_HOST, true, Collections.emptyList()))); - when(abusiveHostRules.getAbusiveHostRulesFor(eq(RESTRICTED_HOST))).thenReturn(Collections.singletonList(new AbusiveHostRule(RESTRICTED_HOST, false, Collections.singletonList("+123")))); - when(abusiveHostRules.getAbusiveHostRulesFor(eq(NICE_HOST))).thenReturn(Collections.emptyList()); + when(abusiveHostRules.isBlocked(eq(ABUSIVE_HOST))).thenReturn(true); + when(abusiveHostRules.isBlocked(eq(NICE_HOST))).thenReturn(false); when(recaptchaClient.verify(eq(INVALID_CAPTCHA_TOKEN), anyString())).thenReturn(false); when(recaptchaClient.verify(eq(VALID_CAPTCHA_TOKEN), anyString())).thenReturn(true); @@ -496,7 +496,7 @@ class AccountControllerTest { verify(smsSender).deliverSmsVerification(eq(SENDER), eq(Optional.empty()), anyString()); } verifyNoMoreInteractions(smsSender); - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST)); + verify(abusiveHostRules).isBlocked(eq(NICE_HOST)); } @Test @@ -563,7 +563,7 @@ class AccountControllerTest { } else { verify(smsSender).deliverVoxVerification(eq(SENDER), anyString(), eq(Collections.emptyList())); } - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST)); + verify(abusiveHostRules).isBlocked(eq(NICE_HOST)); } @ParameterizedTest @@ -595,7 +595,7 @@ class AccountControllerTest { } else { verify(smsSender).deliverVoxVerification(eq(SENDER), anyString(), eq(Locale.LanguageRange.parse("pt-BR"))); } - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST)); + verify(abusiveHostRules).isBlocked(eq(NICE_HOST)); } @ParameterizedTest @@ -628,7 +628,7 @@ class AccountControllerTest { verify(smsSender).deliverVoxVerification(eq(SENDER), anyString(), eq(Locale.LanguageRange .parse("en-US;q=1, ar-US;q=0.9, fa-US;q=0.8, zh-Hans-US;q=0.7, ru-RU;q=0.6, zh-Hant-US;q=0.5"))); } - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST)); + verify(abusiveHostRules).isBlocked(eq(NICE_HOST)); } @Test @@ -646,7 +646,7 @@ class AccountControllerTest { verify(smsSender, never()).deliverVoxVerification(eq(SENDER), anyString(), any()); verify(smsSender, never()).deliverVoxVerificationWithTwilioVerify(eq(SENDER), anyString(), any()); - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST)); + verify(abusiveHostRules).isBlocked(eq(NICE_HOST)); } @ParameterizedTest @@ -677,7 +677,7 @@ class AccountControllerTest { } else { verify(smsSender).deliverSmsVerification(eq(SENDER_PREAUTH), eq(Optional.empty()), anyString()); } - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST)); + verify(abusiveHostRules).isBlocked(eq(NICE_HOST)); } @Test @@ -796,7 +796,7 @@ class AccountControllerTest { assertThat(response.getStatus()).isEqualTo(402); - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(ABUSIVE_HOST)); + verify(abusiveHostRules).isBlocked(eq(ABUSIVE_HOST)); verifyNoMoreInteractions(smsSender); } @@ -880,8 +880,8 @@ class AccountControllerTest { assertThat(response.getStatus()).isEqualTo(402); - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(RATE_LIMITED_IP_HOST)); - verify(abusiveHostRules).setBlockedHost(eq(RATE_LIMITED_IP_HOST), eq("Auto-Block")); + verify(abusiveHostRules).isBlocked(eq(RATE_LIMITED_IP_HOST)); + verify(abusiveHostRules).setBlockedHost(eq(RATE_LIMITED_IP_HOST)); verifyNoMoreInteractions(abusiveHostRules); verifyNoMoreInteractions(recaptchaClient); @@ -910,8 +910,8 @@ class AccountControllerTest { assertThat(response.getStatus()).isEqualTo(402); - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(RATE_LIMITED_PREFIX_HOST)); - verify(abusiveHostRules).setBlockedHost(eq(RATE_LIMITED_PREFIX_HOST), eq("Auto-Block")); + verify(abusiveHostRules).isBlocked(eq(RATE_LIMITED_PREFIX_HOST)); + verify(abusiveHostRules).setBlockedHost(eq(RATE_LIMITED_PREFIX_HOST)); verifyNoMoreInteractions(abusiveHostRules); verifyNoMoreInteractions(recaptchaClient); @@ -940,7 +940,7 @@ class AccountControllerTest { assertThat(response.getStatus()).isEqualTo(402); - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(RATE_LIMITED_HOST2)); + verify(abusiveHostRules).isBlocked(eq(RATE_LIMITED_HOST2)); verifyNoMoreInteractions(abusiveHostRules); verifyNoMoreInteractions(recaptchaClient); @@ -965,7 +965,7 @@ class AccountControllerTest { assertThat(response.getStatus()).isEqualTo(402); - verify(abusiveHostRules, times(1)).getAbusiveHostRulesFor(eq(ABUSIVE_HOST)); + verify(abusiveHostRules, times(1)).isBlocked(eq(ABUSIVE_HOST)); verifyNoMoreInteractions(abusiveHostRules); verifyNoMoreInteractions(smsSender); @@ -979,17 +979,20 @@ class AccountControllerTest { when(verifyExperimentEnrollmentManager.isEnrolled(any(), anyString(), anyList(), anyString())) .thenReturn(enrolledInVerifyExperiment); + final String challenge = "challenge"; + when(pendingAccountsManager.getCodeForNumber(RESTRICTED_NUMBER)).thenReturn(Optional.of(new StoredVerificationCode("123456", System.currentTimeMillis(), challenge, null))); + Response response = resources.getJerseyTest() - .target(String.format("/v1/accounts/sms/code/%s", SENDER)) - .queryParam("challenge", "1234-push") + .target(String.format("/v1/accounts/sms/code/%s", RESTRICTED_NUMBER)) + .queryParam("challenge", challenge) .request() - .header("X-Forwarded-For", RESTRICTED_HOST) + .header("X-Forwarded-For", NICE_HOST) .get(); assertThat(response.getStatus()).isEqualTo(402); - verify(abusiveHostRules).getAbusiveHostRulesFor(eq(RESTRICTED_HOST)); + verify(abusiveHostRules).isBlocked(eq(NICE_HOST)); verifyNoMoreInteractions(smsSender); } @@ -1004,27 +1007,25 @@ class AccountControllerTest { when(smsSender.deliverSmsVerificationWithTwilioVerify(anyString(), any(), anyString(), anyList())) .thenReturn(CompletableFuture.completedFuture(Optional.of("VerificationSid"))); } - - final String number = "+12345678901"; final String challenge = "challenge"; - when(pendingAccountsManager.getCodeForNumber(number)).thenReturn(Optional.of(new StoredVerificationCode("123456", System.currentTimeMillis(), challenge, null))); + when(pendingAccountsManager.getCodeForNumber(SENDER)).thenReturn(Optional.of(new StoredVerificationCode("123456", System.currentTimeMillis(), challenge, null))); Response response = resources.getJerseyTest() - .target(String.format("/v1/accounts/sms/code/%s", number)) + .target(String.format("/v1/accounts/sms/code/%s", SENDER)) .queryParam("challenge", challenge) .request() - .header("X-Forwarded-For", RESTRICTED_HOST) + .header("X-Forwarded-For", NICE_HOST) .get(); assertThat(response.getStatus()).isEqualTo(200); if (enrolledInVerifyExperiment) { - verify(smsSender).deliverSmsVerificationWithTwilioVerify(eq(number), eq(Optional.empty()), anyString(), + verify(smsSender).deliverSmsVerificationWithTwilioVerify(eq(SENDER), eq(Optional.empty()), anyString(), eq(Collections.emptyList())); } else { - verify(smsSender).deliverSmsVerification(eq(number), eq(Optional.empty()), anyString()); + verify(smsSender).deliverSmsVerification(eq(SENDER), eq(Optional.empty()), anyString()); } verifyNoMoreInteractions(smsSender); @@ -1035,7 +1036,7 @@ class AccountControllerTest { void testVerifyCode(final boolean enrolledInVerifyExperiment) throws Exception { if (enrolledInVerifyExperiment) { when(pendingAccountsManager.getCodeForNumber(SENDER)).thenReturn( - Optional.of(new StoredVerificationCode("1234", System.currentTimeMillis(), "1234-push", "VerificationSid")));; + Optional.of(new StoredVerificationCode("1234", System.currentTimeMillis(), "1234-push", "VerificationSid"))); } resources.getJerseyTest() diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AbusiveHostRulesTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AbusiveHostRulesTest.java index b86558b45..0c4db6764 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AbusiveHostRulesTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/tests/storage/AbusiveHostRulesTest.java @@ -6,129 +6,83 @@ package org.whispersystems.textsecuregcm.tests.storage; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; -import io.zonky.test.db.postgres.embedded.LiquibasePreparer; -import io.zonky.test.db.postgres.junit5.EmbeddedPostgresExtension; -import io.zonky.test.db.postgres.junit5.PreparedDbExtension; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; -import org.jdbi.v3.core.Jdbi; +import com.fasterxml.jackson.core.JsonProcessingException; +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration; -import org.whispersystems.textsecuregcm.storage.AbusiveHostRule; +import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; +import org.whispersystems.textsecuregcm.redis.RedisClusterExtension; import org.whispersystems.textsecuregcm.storage.AbusiveHostRules; -import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase; +import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; class AbusiveHostRulesTest { @RegisterExtension - PreparedDbExtension db = EmbeddedPostgresExtension.preparedDatabase( - LiquibasePreparer.forClasspathLocation("abusedb.xml")); - - @RegisterExtension - PreparedDbExtension newDb = EmbeddedPostgresExtension.preparedDatabase( - LiquibasePreparer.forClasspathLocation("abusedb.xml")); - + private static final RedisClusterExtension REDIS_CLUSTER_EXTENSION = RedisClusterExtension.builder().build(); private AbusiveHostRules abusiveHostRules; + private DynamicConfigurationManager mockDynamicConfigManager; @BeforeEach - void setup() { - this.abusiveHostRules = new AbusiveHostRules( - new FaultTolerantDatabase("abusive_hosts-test", Jdbi.create(db.getTestDatabase()), - new CircuitBreakerConfiguration())); + void setup() throws JsonProcessingException { + @SuppressWarnings("unchecked") + DynamicConfigurationManager m = mock(DynamicConfigurationManager.class); + this.mockDynamicConfigManager = m; + when(mockDynamicConfigManager.getConfiguration()).thenReturn(generateConfig(Duration.ofHours(1))); + this.abusiveHostRules = new AbusiveHostRules(REDIS_CLUSTER_EXTENSION.getRedisCluster(), mockDynamicConfigManager); + } + + DynamicConfiguration generateConfig(Duration expireDuration) throws JsonProcessingException { + final String configString = String.format(""" + captcha: + scoreFloor: 1.0 + abusiveHostRules: + expirationTime: %s + """, expireDuration); + return DynamicConfigurationManager + .parseConfiguration(configString, DynamicConfiguration.class) + .orElseThrow(); } @Test - void testBlockedHost() throws SQLException { - PreparedStatement statement = db.getTestDatabase().getConnection() - .prepareStatement("INSERT INTO abusive_host_rules (host, blocked) VALUES (?::INET, ?)"); - statement.setString(1, "192.168.1.1"); - statement.setInt(2, 1); - statement.execute(); - - List rules = abusiveHostRules.getAbusiveHostRulesFor("192.168.1.1"); - assertThat(rules.size()).isEqualTo(1); - assertThat(rules.get(0).regions().isEmpty()).isTrue(); - assertThat(rules.get(0).host()).isEqualTo("192.168.1.1"); - assertThat(rules.get(0).blocked()).isTrue(); + void testBlockedHost() { + REDIS_CLUSTER_EXTENSION.getRedisCluster().useCluster(connection -> + connection.sync().set(AbusiveHostRules.prefix("192.168.1.1"), "1")); + assertThat(abusiveHostRules.isBlocked("192.168.1.1")).isTrue(); } @Test - void testBlockedCidr() throws SQLException { - PreparedStatement statement = db.getTestDatabase().getConnection() - .prepareStatement("INSERT INTO abusive_host_rules (host, blocked) VALUES (?::INET, ?)"); - statement.setString(1, "192.168.1.0/24"); - statement.setInt(2, 1); - statement.execute(); - - List rules = abusiveHostRules.getAbusiveHostRulesFor("192.168.1.1"); - assertThat(rules.size()).isEqualTo(1); - assertThat(rules.get(0).regions().isEmpty()).isTrue(); - assertThat(rules.get(0).host()).isEqualTo("192.168.1.0/24"); - assertThat(rules.get(0).blocked()).isTrue(); + void testUnblocked() { + REDIS_CLUSTER_EXTENSION.getRedisCluster().useCluster(connection -> + connection.sync().set(AbusiveHostRules.prefix("192.168.1.1"), "1")); + assertThat(abusiveHostRules.isBlocked("172.17.1.1")).isFalse(); } @Test - void testUnblocked() throws SQLException { - PreparedStatement statement = db.getTestDatabase().getConnection() - .prepareStatement("INSERT INTO abusive_host_rules (host, blocked) VALUES (?::INET, ?)"); - statement.setString(1, "192.168.1.0/24"); - statement.setInt(2, 1); - statement.execute(); - - List rules = abusiveHostRules.getAbusiveHostRulesFor("172.17.1.1"); - assertThat(rules.isEmpty()).isTrue(); + void testInsertBlocked() { + abusiveHostRules.setBlockedHost("172.17.0.1"); + assertThat(abusiveHostRules.isBlocked("172.17.0.1")).isTrue(); + abusiveHostRules.setBlockedHost("172.17.0.1"); + assertThat(abusiveHostRules.isBlocked("172.17.0.1")).isTrue(); } @Test - void testRestricted() throws SQLException { - PreparedStatement statement = db.getTestDatabase().getConnection() - .prepareStatement("INSERT INTO abusive_host_rules (host, blocked, regions) VALUES (?::INET, ?, ?)"); - statement.setString(1, "192.168.1.0/24"); - statement.setInt(2, 0); - statement.setString(3, "+1,+49"); - statement.execute(); - - List rules = abusiveHostRules.getAbusiveHostRulesFor("192.168.1.100"); - assertThat(rules.size()).isEqualTo(1); - assertThat(rules.get(0).blocked()).isFalse(); - assertThat(rules.get(0).regions()).isEqualTo(Arrays.asList("+1", "+49")); - } - - @Test - void testInsertBlocked() throws Exception { - abusiveHostRules.setBlockedHost("172.17.0.1", "Testing one two"); - - PreparedStatement statement = db.getTestDatabase().getConnection() - .prepareStatement("SELECT * from abusive_host_rules WHERE host = ?::inet"); - statement.setString(1, "172.17.0.1"); - - ResultSet resultSet = statement.executeQuery(); - - assertThat(resultSet.next()).isTrue(); - - assertThat(resultSet.getInt("blocked")).isEqualTo(1); - assertThat(resultSet.getString("regions")).isNullOrEmpty(); - assertThat(resultSet.getString("notes")).isEqualTo("Testing one two"); - - abusiveHostRules.setBlockedHost("172.17.0.1", "Different notes"); - - statement = db.getTestDatabase().getConnection() - .prepareStatement("SELECT * from abusive_host_rules WHERE host = ?::inet"); - statement.setString(1, "172.17.0.1"); - - resultSet = statement.executeQuery(); - - assertThat(resultSet.next()).isTrue(); - - assertThat(resultSet.getInt("blocked")).isEqualTo(1); - assertThat(resultSet.getString("regions")).isNullOrEmpty(); - assertThat(resultSet.getString("notes")).isEqualTo("Testing one two"); + void testExpiration() throws Exception { + when(mockDynamicConfigManager.getConfiguration()).thenReturn(generateConfig(Duration.ofSeconds(1))); + abusiveHostRules.setBlockedHost("192.168.1.1"); + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + while (true) { + if (!abusiveHostRules.isBlocked("192.168.1.1")) { + break; + } + } + }); } }