Use redis for abusive hosts autoblock

Also delete postgres dependencies that we no longer need
This commit is contained in:
Ravi Khadiwala 2022-05-25 10:37:39 -05:00 committed by ravi-signal
parent 5df24edebf
commit 5cfb133f79
20 changed files with 135 additions and 671 deletions

14
pom.xml
View File

@ -61,7 +61,6 @@
<mockito.version>4.3.1</mockito.version>
<netty.version>4.1.65.Final</netty.version>
<opentest4j.version>1.2.0</opentest4j.version>
<postgresql.version>42.3.3</postgresql.version>
<protobuf.version>3.19.4</protobuf.version>
<pushy.version>0.15.1</pushy.version>
<resilience4j.version>1.5.0</resilience4j.version>
@ -224,12 +223,6 @@
<version>${opentest4j.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@ -278,13 +271,6 @@
<artifactId>libsignal-server</artifactId>
<version>0.16.0</version>
</dependency>
<dependency>
<groupId>io.zonky.test.postgres</groupId>
<artifactId>embedded-postgres-binaries-bom</artifactId>
<version>11.13.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-bom</artifactId>

View File

@ -185,12 +185,6 @@ gcpAttachments: # GCP Storage configuration
AAAAAAAA
-----END PRIVATE KEY-----
abuseDatabase: # Postgresql database configuration
driverClass: org.postgresql.Driver
user: example
password: password
url: jdbc:postgresql://example.com:5432/abusedb
accountDatabaseCrawler:
chunkSize: 10 # accounts per run
chunkIntervalMs: 60000 # time per run

View File

@ -128,11 +128,6 @@
<artifactId>jdbi3-core</artifactId>
</dependency>
<dependency>
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
@ -305,12 +300,6 @@
<artifactId>lettuce-core</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.eatthepath</groupId>
<artifactId>pushy</artifactId>
@ -372,13 +361,6 @@
</exclusions>
</dependency>
<dependency>
<groupId>io.zonky.test</groupId>
<artifactId>embedded-postgres</artifactId>
<version>1.3.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.almworks.sqlite4java</groupId>
<artifactId>sqlite4java</artifactId>

View File

@ -143,11 +143,6 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private RedisClusterConfiguration clientPresenceCluster;
@Valid
@NotNull
@JsonProperty
private DatabaseConfiguration abuseDatabase;
@Valid
@NotNull
@JsonProperty
@ -337,10 +332,6 @@ public class WhisperServerConfiguration extends Configuration {
return rateLimitersCluster;
}
public DatabaseConfiguration getAbuseDatabaseConfiguration() {
return abuseDatabase;
}
public RateLimitsConfiguration getLimitsConfiguration() {
return limits;
}

View File

@ -114,7 +114,6 @@ import org.whispersystems.textsecuregcm.limits.PushChallengeManager;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeOptionManager;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.liquibase.NameableMigrationsBundle;
import org.whispersystems.textsecuregcm.mappers.CompletionExceptionMapper;
import org.whispersystems.textsecuregcm.mappers.DeviceLimitExceededExceptionMapper;
import org.whispersystems.textsecuregcm.mappers.IOExceptionMapper;
@ -244,13 +243,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
bootstrap.addCommand(new SetUserDiscoverabilityCommand());
bootstrap.addCommand(new ReserveUsernameCommand());
bootstrap.addCommand(new AssignUsernameCommand());
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("abusedb", "abusedb.xml") {
@Override
public PooledDataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) {
return configuration.getAbuseDatabaseConfiguration();
}
});
}
@Override
@ -308,12 +300,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
ResourceBundleLevelTranslator resourceBundleLevelTranslator = new ResourceBundleLevelTranslator(
headerControlledResourceBundleLookup);
JdbiFactory jdbiFactory = new JdbiFactory(DefaultNameStrategy.CHECK_EMPTY);
Jdbi abuseJdbi = jdbiFactory.build(environment, config.getAbuseDatabaseConfiguration(), "abusedb");
FaultTolerantDatabase abuseDatabase = new FaultTolerantDatabase("abuse_database", abuseJdbi,
config.getAbuseDatabaseConfiguration().getCircuitBreakerConfiguration());
DynamoDbAsyncClient dynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(
config.getDynamoDbClientConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@ -359,7 +345,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(dynamoDbClient,
config.getDynamoDbTables().getMessages().getTableName(),
config.getDynamoDbTables().getMessages().getExpiration());
AbusiveHostRules abusiveHostRules = new AbusiveHostRules(abuseDatabase);
RemoteConfigs remoteConfigs = new RemoteConfigs(dynamoDbClient,
config.getDynamoDbTables().getRemoteConfig().getTableName());
PushChallengeDynamoDb pushChallengeDynamoDb = new PushChallengeDynamoDb(dynamoDbClient,
@ -438,6 +423,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
ExternalServiceCredentialGenerator paymentsCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getPaymentsServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
AbusiveHostRules abusiveHostRules = new AbusiveHostRules(rateLimitersCluster, dynamicConfigurationManager);
SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor, config.getSecureBackupServiceConfiguration());
SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator, storageServiceExecutor, config.getSecureStorageServiceConfiguration());
ClientPresenceManager clientPresenceManager = new ClientPresenceManager(clientPresenceCluster, recurringJobExecutor, keyspaceNotificationDispatchExecutor);

View File

@ -0,0 +1,14 @@
package org.whispersystems.textsecuregcm.configuration.dynamic;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.Duration;
public class DynamicAbusiveHostRulesConfiguration {
@JsonProperty
private Duration expirationTime = Duration.ofDays(1);
public Duration getExpirationTime() {
return expirationTime;
}
}

View File

@ -60,6 +60,10 @@ public class DynamicConfiguration {
@Valid
private DynamicTurnConfiguration turn = new DynamicTurnConfiguration();
@JsonProperty
@Valid
DynamicAbusiveHostRulesConfiguration abusiveHostRules = new DynamicAbusiveHostRulesConfiguration();
public Optional<DynamicExperimentEnrollmentConfiguration> getExperimentEnrollmentConfiguration(
final String experimentName) {
return Optional.ofNullable(experiments.get(experimentName));
@ -117,4 +121,7 @@ public class DynamicConfiguration {
return turn;
}
public DynamicAbusiveHostRulesConfiguration getAbusiveHostRules() {
return abusiveHostRules;
}
}

View File

@ -82,7 +82,6 @@ import org.whispersystems.textsecuregcm.push.GcmMessage;
import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient;
import org.whispersystems.textsecuregcm.sms.SmsSender;
import org.whispersystems.textsecuregcm.sms.TwilioVerifyExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRule;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRules;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
@ -108,9 +107,7 @@ public class AccountController {
private final Logger logger = LoggerFactory.getLogger(AccountController.class);
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Meter blockedHostMeter = metricRegistry.meter(name(AccountController.class, "blocked_host" ));
private final Meter blockedPrefixMeter = metricRegistry.meter(name(AccountController.class, "blocked_prefix" ));
private final Meter countryFilterApplicable = metricRegistry.meter(name(AccountController.class, "country_filter_applicable"));
private final Meter filteredHostMeter = metricRegistry.meter(name(AccountController.class, "filtered_host" ));
private final Meter countryFilteredHostMeter = metricRegistry.meter(name(AccountController.class, "country_limited_host" ));
private final Meter rateLimitedHostMeter = metricRegistry.meter(name(AccountController.class, "rate_limited_host" ));
private final Meter rateLimitedPrefixMeter = metricRegistry.meter(name(AccountController.class, "rate_limited_prefix" ));
@ -246,7 +243,7 @@ public class AccountController {
if (requirement.isAutoBlock() && shouldAutoBlock(sourceHost)) {
logger.info("Auto-block: {}", sourceHost);
abusiveHostRules.setBlockedHost(sourceHost, "Auto-Block");
abusiveHostRules.setBlockedHost(sourceHost);
}
return Response.status(402).build();
@ -780,7 +777,10 @@ public class AccountController {
DynamicCaptchaConfiguration captchaConfig = dynamicConfigurationManager.getConfiguration()
.getCaptchaConfiguration();
boolean countryFiltered = captchaConfig.getSignupCountryCodes().contains(countryCode);
if (shouldBlock(transport, forwardedFor, sourceHost, number)) {
if (abusiveHostRules.isBlocked(sourceHost)) {
blockedHostMeter.mark();
logger.info("Blocked host: {}, {}, {} ({})", transport, number, sourceHost, forwardedFor);
if (countryFiltered) {
// this host was caught in the abusiveHostRules filter, but
// would be caught by country filter as well
@ -813,33 +813,6 @@ public class AccountController {
return new CaptchaRequirement(false, false);
}
private boolean shouldBlock(final String transport, final String forwardedFor, final String sourceHost, final String number) {
List<AbusiveHostRule> abuseRules = abusiveHostRules.getAbusiveHostRulesFor(sourceHost);
for (AbusiveHostRule abuseRule : abuseRules) {
if (abuseRule.blocked()) {
logger.info("Blocked host: {}, {}, {} ({}) matched rule: {}", transport, number, sourceHost, forwardedFor, abuseRule.host());
// did we match based on an ip block or an exact match
if (abuseRule.cidrPrefix().filter(i -> i < 32).isPresent()) {
blockedPrefixMeter.mark();
} else {
blockedHostMeter.mark();
}
return true;
}
if (!abuseRule.regions().isEmpty()) {
if (abuseRule.regions().stream().noneMatch(number::startsWith)) {
logger.info("Restricted host: {}, {}, {} ({}) matched rule: {}/{}", transport, number, sourceHost, forwardedFor, abuseRule.host(), abuseRule.regions());
filteredHostMeter.mark();
return true;
}
}
}
return false;
}
@Timed
@DELETE
@Path("/me")

View File

@ -1,68 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.liquibase;
import com.codahale.metrics.MetricRegistry;
import net.sourceforge.argparse4j.inf.Namespace;
import java.sql.SQLException;
import io.dropwizard.Configuration;
import io.dropwizard.cli.ConfiguredCommand;
import io.dropwizard.db.DatabaseConfiguration;
import io.dropwizard.db.ManagedDataSource;
import io.dropwizard.db.PooledDataSourceFactory;
import io.dropwizard.setup.Bootstrap;
import liquibase.Liquibase;
import liquibase.exception.LiquibaseException;
import liquibase.exception.ValidationFailedException;
public abstract class AbstractLiquibaseCommand<T extends Configuration> extends ConfiguredCommand<T> {
private final DatabaseConfiguration<T> strategy;
private final Class<T> configurationClass;
private final String migrations;
protected AbstractLiquibaseCommand(String name,
String description,
String migrations,
DatabaseConfiguration<T> strategy,
Class<T> configurationClass) {
super(name, description);
this.migrations = migrations;
this.strategy = strategy;
this.configurationClass = configurationClass;
}
@Override
protected Class<T> getConfigurationClass() {
return configurationClass;
}
@Override
@SuppressWarnings("UseOfSystemOutOrSystemErr")
protected void run(Bootstrap<T> bootstrap, Namespace namespace, T configuration) throws Exception {
final PooledDataSourceFactory dbConfig = strategy.getDataSourceFactory(configuration);
dbConfig.asSingleConnectionPool();
try (final CloseableLiquibase liquibase = openLiquibase(dbConfig, namespace)) {
run(namespace, liquibase);
} catch (ValidationFailedException e) {
e.printDescriptiveError(System.err);
throw e;
}
}
private CloseableLiquibase openLiquibase(final PooledDataSourceFactory dataSourceFactory, final Namespace namespace)
throws ClassNotFoundException, SQLException, LiquibaseException
{
final ManagedDataSource dataSource = dataSourceFactory.build(new MetricRegistry(), "liquibase");
return new CloseableLiquibase(dataSource, migrations);
}
protected abstract void run(Namespace namespace, Liquibase liquibase) throws Exception;
}

View File

@ -1,33 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.liquibase;
import java.sql.SQLException;
import io.dropwizard.db.ManagedDataSource;
import liquibase.Liquibase;
import liquibase.database.jvm.JdbcConnection;
import liquibase.exception.LiquibaseException;
import liquibase.resource.ClassLoaderResourceAccessor;
public class CloseableLiquibase extends Liquibase implements AutoCloseable {
private final ManagedDataSource dataSource;
public CloseableLiquibase(ManagedDataSource dataSource, String migrations)
throws LiquibaseException, ClassNotFoundException, SQLException
{
super(migrations,
new ClassLoaderResourceAccessor(),
new JdbcConnection(dataSource.getConnection()));
this.dataSource = dataSource;
}
@Override
public void close() throws Exception {
dataSource.stop();
}
}

View File

@ -1,77 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.liquibase;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import net.sourceforge.argparse4j.impl.Arguments;
import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser;
import java.io.OutputStreamWriter;
import java.util.List;
import io.dropwizard.Configuration;
import io.dropwizard.db.DatabaseConfiguration;
import liquibase.Liquibase;
public class DbMigrateCommand<T extends Configuration> extends AbstractLiquibaseCommand<T> {
public DbMigrateCommand(String migration, DatabaseConfiguration<T> strategy, Class<T> configurationClass) {
super("migrate", "Apply all pending change sets.", migration, strategy, configurationClass);
}
@Override
public void configure(Subparser subparser) {
super.configure(subparser);
subparser.addArgument("-n", "--dry-run")
.action(Arguments.storeTrue())
.dest("dry-run")
.setDefault(Boolean.FALSE)
.help("output the DDL to stdout, don't run it");
subparser.addArgument("-c", "--count")
.type(Integer.class)
.dest("count")
.help("only apply the next N change sets");
subparser.addArgument("-i", "--include")
.action(Arguments.append())
.dest("contexts")
.help("include change sets from the given context");
}
@Override
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public void run(Namespace namespace, Liquibase liquibase) throws Exception {
final String context = getContext(namespace);
final Integer count = namespace.getInt("count");
final Boolean dryRun = namespace.getBoolean("dry-run");
if (count != null) {
if (dryRun) {
liquibase.update(count, context, new OutputStreamWriter(System.out, Charsets.UTF_8));
} else {
liquibase.update(count, context);
}
} else {
if (dryRun) {
liquibase.update(context, new OutputStreamWriter(System.out, Charsets.UTF_8));
} else {
liquibase.update(context);
}
}
}
private String getContext(Namespace namespace) {
final List<Object> contexts = namespace.getList("contexts");
if (contexts == null) {
return "";
}
return Joiner.on(',').join(contexts);
}
}

View File

@ -1,56 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.liquibase;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import net.sourceforge.argparse4j.impl.Arguments;
import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser;
import java.io.OutputStreamWriter;
import java.util.List;
import io.dropwizard.Configuration;
import io.dropwizard.db.DatabaseConfiguration;
import liquibase.Liquibase;
public class DbStatusCommand <T extends Configuration> extends AbstractLiquibaseCommand<T> {
public DbStatusCommand(String migrations, DatabaseConfiguration<T> strategy, Class<T> configurationClass) {
super("status", "Check for pending change sets.", migrations, strategy, configurationClass);
}
@Override
public void configure(Subparser subparser) {
super.configure(subparser);
subparser.addArgument("-v", "--verbose")
.action(Arguments.storeTrue())
.dest("verbose")
.help("Output verbose information");
subparser.addArgument("-i", "--include")
.action(Arguments.append())
.dest("contexts")
.help("include change sets from the given context");
}
@Override
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public void run(Namespace namespace, Liquibase liquibase) throws Exception {
liquibase.reportStatus(namespace.getBoolean("verbose"),
getContext(namespace),
new OutputStreamWriter(System.out, Charsets.UTF_8));
}
private String getContext(Namespace namespace) {
final List<Object> contexts = namespace.getList("contexts");
if (contexts == null) {
return "";
}
return Joiner.on(',').join(contexts);
}
}

View File

@ -1,49 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.liquibase;
import com.google.common.collect.Maps;
import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser;
import java.util.SortedMap;
import io.dropwizard.Configuration;
import io.dropwizard.db.DatabaseConfiguration;
import liquibase.Liquibase;
public class NameableDbCommand<T extends Configuration> extends AbstractLiquibaseCommand<T> {
private static final String COMMAND_NAME_ATTR = "subcommand";
private final SortedMap<String, AbstractLiquibaseCommand<T>> subcommands;
public NameableDbCommand(String name, String migrations, DatabaseConfiguration<T> strategy, Class<T> configurationClass) {
super(name, "Run database migrations tasks", migrations, strategy, configurationClass);
this.subcommands = Maps.newTreeMap();
addSubcommand(new DbMigrateCommand<>(migrations, strategy, configurationClass));
addSubcommand(new DbStatusCommand<>(migrations, strategy, configurationClass));
}
private void addSubcommand(AbstractLiquibaseCommand<T> subcommand) {
subcommands.put(subcommand.getName(), subcommand);
}
@Override
public void configure(Subparser subparser) {
for (AbstractLiquibaseCommand<T> subcommand : subcommands.values()) {
final Subparser cmdParser = subparser.addSubparsers()
.addParser(subcommand.getName())
.setDefault(COMMAND_NAME_ATTR, subcommand.getName())
.description(subcommand.getDescription());
subcommand.configure(cmdParser);
}
}
@Override
public void run(Namespace namespace, Liquibase liquibase) throws Exception {
final AbstractLiquibaseCommand<T> subcommand = subcommands.get(namespace.getString(COMMAND_NAME_ATTR));
subcommand.run(namespace, liquibase);
}
}

View File

@ -1,32 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.liquibase;
import io.dropwizard.Bundle;
import io.dropwizard.Configuration;
import io.dropwizard.db.DatabaseConfiguration;
import io.dropwizard.setup.Bootstrap;
import io.dropwizard.setup.Environment;
import io.dropwizard.util.Generics;
public abstract class NameableMigrationsBundle<T extends Configuration> implements Bundle, DatabaseConfiguration<T> {
private final String name;
private final String migrations;
public NameableMigrationsBundle(String name, String migrations) {
this.name = name;
this.migrations = migrations;
}
public final void initialize(Bootstrap<?> bootstrap) {
Class klass = Generics.getTypeParameter(this.getClass(), Configuration.class);
bootstrap.addCommand(new NameableDbCommand(name, migrations, this, klass));
}
public final void run(Environment environment) {
}
}

View File

@ -1,25 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import java.util.List;
import java.util.Optional;
public record AbusiveHostRule(String host, boolean blocked, List<String> regions) {
public Optional<Integer> cidrPrefix() {
String[] split = host.split("/");
if (split.length != 2) {
return Optional.empty();
}
try {
return Optional.of(Integer.parseInt(split[1]));
} catch (NumberFormatException e) {
return Optional.empty();
}
}
}

View File

@ -10,52 +10,43 @@ import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import java.util.List;
import org.whispersystems.textsecuregcm.storage.mappers.AbusiveHostRuleRowMapper;
import java.time.Duration;
import com.google.common.annotations.VisibleForTesting;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.util.Constants;
public class AbusiveHostRules {
public static final String ID = "id";
public static final String HOST = "host";
public static final String BLOCKED = "blocked";
public static final String REGIONS = "regions";
public static final String NOTES = "notes";
private static final String KEY_PREFIX = "abusive_hosts::";
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Timer getTimer = metricRegistry.timer(name(AbusiveHostRules.class, "get"));
private final Timer insertTimer = metricRegistry.timer(name(AbusiveHostRules.class, "setBlockedHost"));
private final FaultTolerantDatabase database;
private final FaultTolerantRedisCluster redisCluster;
private final DynamicConfigurationManager<DynamicConfiguration> configurationManager;
public AbusiveHostRules(FaultTolerantDatabase database) {
this.database = database;
this.database.getDatabase().registerRowMapper(new AbusiveHostRuleRowMapper());
public AbusiveHostRules(FaultTolerantRedisCluster redisCluster, final DynamicConfigurationManager<DynamicConfiguration> configurationManager) {
this.redisCluster = redisCluster;
this.configurationManager = configurationManager;
}
public List<AbusiveHostRule> getAbusiveHostRulesFor(String host) {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context timer = getTimer.time()) {
return handle.createQuery("SELECT * FROM abusive_host_rules WHERE :host::inet <<= " + HOST)
.bind("host", host)
.mapTo(AbusiveHostRule.class)
.list();
}
}));
public boolean isBlocked(String host) {
try (Timer.Context timer = getTimer.time()) {
return this.redisCluster.withCluster(connection -> connection.sync().exists(prefix(host))) > 0;
}
}
public void setBlockedHost(String host, String notes) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context timer = insertTimer.time()) {
handle.createUpdate(
"INSERT INTO abusive_host_rules(host, blocked, notes) VALUES(:host::inet, :blocked, :notes) ON CONFLICT DO NOTHING")
.bind("host", host)
.bind("blocked", 1)
.bind("notes", notes)
.execute();
}
}));
public void setBlockedHost(String host) {
Duration expireTime = configurationManager.getConfiguration().getAbusiveHostRules().getExpirationTime();
try (Timer.Context timer = insertTimer.time()) {
this.redisCluster.useCluster(connection -> connection.sync().setex(prefix(host), expireTime.toSeconds(), "1"));
}
}
@VisibleForTesting
public static String prefix(String keyName) {
return KEY_PREFIX + keyName;
}
}

View File

@ -1,33 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage.mappers;
import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.core.statement.StatementContext;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRule;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRules;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
public class AbusiveHostRuleRowMapper implements RowMapper<AbusiveHostRule> {
@Override
public AbusiveHostRule map(ResultSet resultSet, StatementContext ctx) throws SQLException {
String regionsData = resultSet.getString(AbusiveHostRules.REGIONS);
List<String> regions;
if (regionsData == null) regions = new LinkedList<>();
else regions = Arrays.asList(regionsData.split(","));
return new AbusiveHostRule(resultSet.getString(AbusiveHostRules.HOST), resultSet.getInt(AbusiveHostRules.BLOCKED) == 1, regions);
}
}

View File

@ -1,42 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2013-2020 Signal Messenger, LLC
~ SPDX-License-Identifier: AGPL-3.0-only
-->
<databaseChangeLog
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">
<changeSet id="1" author="moxie">
<createTable tableName="abusive_host_rules">
<column name="id" type="bigint" autoIncrement="true">
<constraints primaryKey="true" nullable="false"/>
</column>
<column name="host" type="inet">
<constraints nullable="false" unique="true"/>
</column>
<column name="blocked" type="tinyint">
<constraints nullable="false"/>
</column>
<column name="regions" type="text"/>
</createTable>
<createIndex tableName="abusive_host_rules" indexName="host_index">
<column name="host"/>
</createIndex>
</changeSet>
<changeSet id="2" author="moxie">
<addColumn tableName="abusive_host_rules">
<column name="notes" type="text"/>
</addColumn>
</changeSet>
</databaseChangeLog>

View File

@ -87,7 +87,6 @@ import org.whispersystems.textsecuregcm.push.GcmMessage;
import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient;
import org.whispersystems.textsecuregcm.sms.SmsSender;
import org.whispersystems.textsecuregcm.sms.TwilioVerifyExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRule;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRules;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
@ -113,12 +112,13 @@ class AccountControllerTest {
private static final String SENDER_REG_LOCK = "+14158888888";
private static final String SENDER_HAS_STORAGE = "+14159999999";
private static final String SENDER_TRANSFER = "+14151111112";
private static final String RESTRICTED_COUNTRY = "800";
private static final String RESTRICTED_NUMBER = "+" + RESTRICTED_COUNTRY + "11111111";
private static final UUID SENDER_REG_LOCK_UUID = UUID.randomUUID();
private static final UUID SENDER_TRANSFER_UUID = UUID.randomUUID();
private static final String ABUSIVE_HOST = "192.168.1.1";
private static final String RESTRICTED_HOST = "192.168.1.2";
private static final String NICE_HOST = "127.0.0.1";
private static final String RATE_LIMITED_IP_HOST = "10.0.0.1";
private static final String RATE_LIMITED_PREFIX_HOST = "10.0.0.2";
@ -276,12 +276,12 @@ class AccountControllerTest {
.thenReturn(dynamicConfiguration);
DynamicCaptchaConfiguration signupCaptchaConfig = new DynamicCaptchaConfiguration();
signupCaptchaConfig.setSignupCountryCodes(Set.of(RESTRICTED_COUNTRY));
when(dynamicConfiguration.getCaptchaConfiguration()).thenReturn(signupCaptchaConfig);
}
when(abusiveHostRules.getAbusiveHostRulesFor(eq(ABUSIVE_HOST))).thenReturn(Collections.singletonList(new AbusiveHostRule(ABUSIVE_HOST, true, Collections.emptyList())));
when(abusiveHostRules.getAbusiveHostRulesFor(eq(RESTRICTED_HOST))).thenReturn(Collections.singletonList(new AbusiveHostRule(RESTRICTED_HOST, false, Collections.singletonList("+123"))));
when(abusiveHostRules.getAbusiveHostRulesFor(eq(NICE_HOST))).thenReturn(Collections.emptyList());
when(abusiveHostRules.isBlocked(eq(ABUSIVE_HOST))).thenReturn(true);
when(abusiveHostRules.isBlocked(eq(NICE_HOST))).thenReturn(false);
when(recaptchaClient.verify(eq(INVALID_CAPTCHA_TOKEN), anyString())).thenReturn(false);
when(recaptchaClient.verify(eq(VALID_CAPTCHA_TOKEN), anyString())).thenReturn(true);
@ -496,7 +496,7 @@ class AccountControllerTest {
verify(smsSender).deliverSmsVerification(eq(SENDER), eq(Optional.empty()), anyString());
}
verifyNoMoreInteractions(smsSender);
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST));
verify(abusiveHostRules).isBlocked(eq(NICE_HOST));
}
@Test
@ -563,7 +563,7 @@ class AccountControllerTest {
} else {
verify(smsSender).deliverVoxVerification(eq(SENDER), anyString(), eq(Collections.emptyList()));
}
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST));
verify(abusiveHostRules).isBlocked(eq(NICE_HOST));
}
@ParameterizedTest
@ -595,7 +595,7 @@ class AccountControllerTest {
} else {
verify(smsSender).deliverVoxVerification(eq(SENDER), anyString(), eq(Locale.LanguageRange.parse("pt-BR")));
}
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST));
verify(abusiveHostRules).isBlocked(eq(NICE_HOST));
}
@ParameterizedTest
@ -628,7 +628,7 @@ class AccountControllerTest {
verify(smsSender).deliverVoxVerification(eq(SENDER), anyString(), eq(Locale.LanguageRange
.parse("en-US;q=1, ar-US;q=0.9, fa-US;q=0.8, zh-Hans-US;q=0.7, ru-RU;q=0.6, zh-Hant-US;q=0.5")));
}
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST));
verify(abusiveHostRules).isBlocked(eq(NICE_HOST));
}
@Test
@ -646,7 +646,7 @@ class AccountControllerTest {
verify(smsSender, never()).deliverVoxVerification(eq(SENDER), anyString(), any());
verify(smsSender, never()).deliverVoxVerificationWithTwilioVerify(eq(SENDER), anyString(), any());
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST));
verify(abusiveHostRules).isBlocked(eq(NICE_HOST));
}
@ParameterizedTest
@ -677,7 +677,7 @@ class AccountControllerTest {
} else {
verify(smsSender).deliverSmsVerification(eq(SENDER_PREAUTH), eq(Optional.empty()), anyString());
}
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(NICE_HOST));
verify(abusiveHostRules).isBlocked(eq(NICE_HOST));
}
@Test
@ -796,7 +796,7 @@ class AccountControllerTest {
assertThat(response.getStatus()).isEqualTo(402);
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(ABUSIVE_HOST));
verify(abusiveHostRules).isBlocked(eq(ABUSIVE_HOST));
verifyNoMoreInteractions(smsSender);
}
@ -880,8 +880,8 @@ class AccountControllerTest {
assertThat(response.getStatus()).isEqualTo(402);
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(RATE_LIMITED_IP_HOST));
verify(abusiveHostRules).setBlockedHost(eq(RATE_LIMITED_IP_HOST), eq("Auto-Block"));
verify(abusiveHostRules).isBlocked(eq(RATE_LIMITED_IP_HOST));
verify(abusiveHostRules).setBlockedHost(eq(RATE_LIMITED_IP_HOST));
verifyNoMoreInteractions(abusiveHostRules);
verifyNoMoreInteractions(recaptchaClient);
@ -910,8 +910,8 @@ class AccountControllerTest {
assertThat(response.getStatus()).isEqualTo(402);
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(RATE_LIMITED_PREFIX_HOST));
verify(abusiveHostRules).setBlockedHost(eq(RATE_LIMITED_PREFIX_HOST), eq("Auto-Block"));
verify(abusiveHostRules).isBlocked(eq(RATE_LIMITED_PREFIX_HOST));
verify(abusiveHostRules).setBlockedHost(eq(RATE_LIMITED_PREFIX_HOST));
verifyNoMoreInteractions(abusiveHostRules);
verifyNoMoreInteractions(recaptchaClient);
@ -940,7 +940,7 @@ class AccountControllerTest {
assertThat(response.getStatus()).isEqualTo(402);
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(RATE_LIMITED_HOST2));
verify(abusiveHostRules).isBlocked(eq(RATE_LIMITED_HOST2));
verifyNoMoreInteractions(abusiveHostRules);
verifyNoMoreInteractions(recaptchaClient);
@ -965,7 +965,7 @@ class AccountControllerTest {
assertThat(response.getStatus()).isEqualTo(402);
verify(abusiveHostRules, times(1)).getAbusiveHostRulesFor(eq(ABUSIVE_HOST));
verify(abusiveHostRules, times(1)).isBlocked(eq(ABUSIVE_HOST));
verifyNoMoreInteractions(abusiveHostRules);
verifyNoMoreInteractions(smsSender);
@ -979,17 +979,20 @@ class AccountControllerTest {
when(verifyExperimentEnrollmentManager.isEnrolled(any(), anyString(), anyList(), anyString()))
.thenReturn(enrolledInVerifyExperiment);
final String challenge = "challenge";
when(pendingAccountsManager.getCodeForNumber(RESTRICTED_NUMBER)).thenReturn(Optional.of(new StoredVerificationCode("123456", System.currentTimeMillis(), challenge, null)));
Response response =
resources.getJerseyTest()
.target(String.format("/v1/accounts/sms/code/%s", SENDER))
.queryParam("challenge", "1234-push")
.target(String.format("/v1/accounts/sms/code/%s", RESTRICTED_NUMBER))
.queryParam("challenge", challenge)
.request()
.header("X-Forwarded-For", RESTRICTED_HOST)
.header("X-Forwarded-For", NICE_HOST)
.get();
assertThat(response.getStatus()).isEqualTo(402);
verify(abusiveHostRules).getAbusiveHostRulesFor(eq(RESTRICTED_HOST));
verify(abusiveHostRules).isBlocked(eq(NICE_HOST));
verifyNoMoreInteractions(smsSender);
}
@ -1004,27 +1007,25 @@ class AccountControllerTest {
when(smsSender.deliverSmsVerificationWithTwilioVerify(anyString(), any(), anyString(), anyList()))
.thenReturn(CompletableFuture.completedFuture(Optional.of("VerificationSid")));
}
final String number = "+12345678901";
final String challenge = "challenge";
when(pendingAccountsManager.getCodeForNumber(number)).thenReturn(Optional.of(new StoredVerificationCode("123456", System.currentTimeMillis(), challenge, null)));
when(pendingAccountsManager.getCodeForNumber(SENDER)).thenReturn(Optional.of(new StoredVerificationCode("123456", System.currentTimeMillis(), challenge, null)));
Response response =
resources.getJerseyTest()
.target(String.format("/v1/accounts/sms/code/%s", number))
.target(String.format("/v1/accounts/sms/code/%s", SENDER))
.queryParam("challenge", challenge)
.request()
.header("X-Forwarded-For", RESTRICTED_HOST)
.header("X-Forwarded-For", NICE_HOST)
.get();
assertThat(response.getStatus()).isEqualTo(200);
if (enrolledInVerifyExperiment) {
verify(smsSender).deliverSmsVerificationWithTwilioVerify(eq(number), eq(Optional.empty()), anyString(),
verify(smsSender).deliverSmsVerificationWithTwilioVerify(eq(SENDER), eq(Optional.empty()), anyString(),
eq(Collections.emptyList()));
} else {
verify(smsSender).deliverSmsVerification(eq(number), eq(Optional.empty()), anyString());
verify(smsSender).deliverSmsVerification(eq(SENDER), eq(Optional.empty()), anyString());
}
verifyNoMoreInteractions(smsSender);
@ -1035,7 +1036,7 @@ class AccountControllerTest {
void testVerifyCode(final boolean enrolledInVerifyExperiment) throws Exception {
if (enrolledInVerifyExperiment) {
when(pendingAccountsManager.getCodeForNumber(SENDER)).thenReturn(
Optional.of(new StoredVerificationCode("1234", System.currentTimeMillis(), "1234-push", "VerificationSid")));;
Optional.of(new StoredVerificationCode("1234", System.currentTimeMillis(), "1234-push", "VerificationSid")));
}
resources.getJerseyTest()

View File

@ -6,129 +6,83 @@
package org.whispersystems.textsecuregcm.tests.storage;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import io.zonky.test.db.postgres.embedded.LiquibasePreparer;
import io.zonky.test.db.postgres.junit5.EmbeddedPostgresExtension;
import io.zonky.test.db.postgres.junit5.PreparedDbExtension;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
import org.jdbi.v3.core.Jdbi;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.time.Duration;
import java.time.Instant;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.whispersystems.textsecuregcm.configuration.CircuitBreakerConfiguration;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRule;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.redis.RedisClusterExtension;
import org.whispersystems.textsecuregcm.storage.AbusiveHostRules;
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
class AbusiveHostRulesTest {
@RegisterExtension
PreparedDbExtension db = EmbeddedPostgresExtension.preparedDatabase(
LiquibasePreparer.forClasspathLocation("abusedb.xml"));
@RegisterExtension
PreparedDbExtension newDb = EmbeddedPostgresExtension.preparedDatabase(
LiquibasePreparer.forClasspathLocation("abusedb.xml"));
private static final RedisClusterExtension REDIS_CLUSTER_EXTENSION = RedisClusterExtension.builder().build();
private AbusiveHostRules abusiveHostRules;
private DynamicConfigurationManager<DynamicConfiguration> mockDynamicConfigManager;
@BeforeEach
void setup() {
this.abusiveHostRules = new AbusiveHostRules(
new FaultTolerantDatabase("abusive_hosts-test", Jdbi.create(db.getTestDatabase()),
new CircuitBreakerConfiguration()));
void setup() throws JsonProcessingException {
@SuppressWarnings("unchecked")
DynamicConfigurationManager<DynamicConfiguration> m = mock(DynamicConfigurationManager.class);
this.mockDynamicConfigManager = m;
when(mockDynamicConfigManager.getConfiguration()).thenReturn(generateConfig(Duration.ofHours(1)));
this.abusiveHostRules = new AbusiveHostRules(REDIS_CLUSTER_EXTENSION.getRedisCluster(), mockDynamicConfigManager);
}
DynamicConfiguration generateConfig(Duration expireDuration) throws JsonProcessingException {
final String configString = String.format("""
captcha:
scoreFloor: 1.0
abusiveHostRules:
expirationTime: %s
""", expireDuration);
return DynamicConfigurationManager
.parseConfiguration(configString, DynamicConfiguration.class)
.orElseThrow();
}
@Test
void testBlockedHost() throws SQLException {
PreparedStatement statement = db.getTestDatabase().getConnection()
.prepareStatement("INSERT INTO abusive_host_rules (host, blocked) VALUES (?::INET, ?)");
statement.setString(1, "192.168.1.1");
statement.setInt(2, 1);
statement.execute();
List<AbusiveHostRule> rules = abusiveHostRules.getAbusiveHostRulesFor("192.168.1.1");
assertThat(rules.size()).isEqualTo(1);
assertThat(rules.get(0).regions().isEmpty()).isTrue();
assertThat(rules.get(0).host()).isEqualTo("192.168.1.1");
assertThat(rules.get(0).blocked()).isTrue();
void testBlockedHost() {
REDIS_CLUSTER_EXTENSION.getRedisCluster().useCluster(connection ->
connection.sync().set(AbusiveHostRules.prefix("192.168.1.1"), "1"));
assertThat(abusiveHostRules.isBlocked("192.168.1.1")).isTrue();
}
@Test
void testBlockedCidr() throws SQLException {
PreparedStatement statement = db.getTestDatabase().getConnection()
.prepareStatement("INSERT INTO abusive_host_rules (host, blocked) VALUES (?::INET, ?)");
statement.setString(1, "192.168.1.0/24");
statement.setInt(2, 1);
statement.execute();
List<AbusiveHostRule> rules = abusiveHostRules.getAbusiveHostRulesFor("192.168.1.1");
assertThat(rules.size()).isEqualTo(1);
assertThat(rules.get(0).regions().isEmpty()).isTrue();
assertThat(rules.get(0).host()).isEqualTo("192.168.1.0/24");
assertThat(rules.get(0).blocked()).isTrue();
void testUnblocked() {
REDIS_CLUSTER_EXTENSION.getRedisCluster().useCluster(connection ->
connection.sync().set(AbusiveHostRules.prefix("192.168.1.1"), "1"));
assertThat(abusiveHostRules.isBlocked("172.17.1.1")).isFalse();
}
@Test
void testUnblocked() throws SQLException {
PreparedStatement statement = db.getTestDatabase().getConnection()
.prepareStatement("INSERT INTO abusive_host_rules (host, blocked) VALUES (?::INET, ?)");
statement.setString(1, "192.168.1.0/24");
statement.setInt(2, 1);
statement.execute();
List<AbusiveHostRule> rules = abusiveHostRules.getAbusiveHostRulesFor("172.17.1.1");
assertThat(rules.isEmpty()).isTrue();
void testInsertBlocked() {
abusiveHostRules.setBlockedHost("172.17.0.1");
assertThat(abusiveHostRules.isBlocked("172.17.0.1")).isTrue();
abusiveHostRules.setBlockedHost("172.17.0.1");
assertThat(abusiveHostRules.isBlocked("172.17.0.1")).isTrue();
}
@Test
void testRestricted() throws SQLException {
PreparedStatement statement = db.getTestDatabase().getConnection()
.prepareStatement("INSERT INTO abusive_host_rules (host, blocked, regions) VALUES (?::INET, ?, ?)");
statement.setString(1, "192.168.1.0/24");
statement.setInt(2, 0);
statement.setString(3, "+1,+49");
statement.execute();
List<AbusiveHostRule> rules = abusiveHostRules.getAbusiveHostRulesFor("192.168.1.100");
assertThat(rules.size()).isEqualTo(1);
assertThat(rules.get(0).blocked()).isFalse();
assertThat(rules.get(0).regions()).isEqualTo(Arrays.asList("+1", "+49"));
}
@Test
void testInsertBlocked() throws Exception {
abusiveHostRules.setBlockedHost("172.17.0.1", "Testing one two");
PreparedStatement statement = db.getTestDatabase().getConnection()
.prepareStatement("SELECT * from abusive_host_rules WHERE host = ?::inet");
statement.setString(1, "172.17.0.1");
ResultSet resultSet = statement.executeQuery();
assertThat(resultSet.next()).isTrue();
assertThat(resultSet.getInt("blocked")).isEqualTo(1);
assertThat(resultSet.getString("regions")).isNullOrEmpty();
assertThat(resultSet.getString("notes")).isEqualTo("Testing one two");
abusiveHostRules.setBlockedHost("172.17.0.1", "Different notes");
statement = db.getTestDatabase().getConnection()
.prepareStatement("SELECT * from abusive_host_rules WHERE host = ?::inet");
statement.setString(1, "172.17.0.1");
resultSet = statement.executeQuery();
assertThat(resultSet.next()).isTrue();
assertThat(resultSet.getInt("blocked")).isEqualTo(1);
assertThat(resultSet.getString("regions")).isNullOrEmpty();
assertThat(resultSet.getString("notes")).isEqualTo("Testing one two");
void testExpiration() throws Exception {
when(mockDynamicConfigManager.getConfiguration()).thenReturn(generateConfig(Duration.ofSeconds(1)));
abusiveHostRules.setBlockedHost("192.168.1.1");
assertTimeoutPreemptively(Duration.ofSeconds(5), () -> {
while (true) {
if (!abusiveHostRules.isBlocked("192.168.1.1")) {
break;
}
}
});
}
}