From a297d03db56273be2a5199d212d87073f873e049 Mon Sep 17 00:00:00 2001 From: Moxie Marlinspike Date: Thu, 17 Mar 2016 15:24:49 -0700 Subject: [PATCH] Add periodic stats command // FREEBIE --- .../WhisperServerConfiguration.java | 15 +- .../textsecuregcm/WhisperServerService.java | 11 +- .../metrics/JsonMetricsReporter.java | 2 +- .../textsecuregcm/storage/Accounts.java | 6 + .../workers/PeriodicStatsCommand.java | 128 ++++++++++++++++++ 5 files changed, 144 insertions(+), 18 deletions(-) create mode 100644 src/main/java/org/whispersystems/textsecuregcm/workers/PeriodicStatsCommand.java diff --git a/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java index 89f5b62d0..d16e92398 100644 --- a/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java +++ b/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java @@ -85,15 +85,14 @@ public class WhisperServerConfiguration extends Configuration { @JsonProperty private DataSourceFactory database = new DataSourceFactory(); + @JsonProperty + private DataSourceFactory read_database; + @Valid @NotNull @JsonProperty private RateLimitsConfiguration limits = new RateLimitsConfiguration(); - @Valid - @JsonProperty - private GraphiteConfiguration graphite = new GraphiteConfiguration(); - @Valid @JsonProperty private WebsocketConfiguration websocket = new WebsocketConfiguration(); @@ -143,6 +142,10 @@ public class WhisperServerConfiguration extends Configuration { return database; } + public DataSourceFactory getReadDataSourceFactory() { + return read_database; + } + public RateLimitsConfiguration getLimitsConfiguration() { return limits; } @@ -151,10 +154,6 @@ public class WhisperServerConfiguration extends Configuration { return federation; } - public GraphiteConfiguration getGraphiteConfiguration() { - return graphite; - } - public RedPhoneConfiguration getRedphoneConfiguration() { return redphone; } diff --git a/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 9ef3f2f86..66b8fdeb0 100644 --- a/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -88,6 +88,7 @@ import org.whispersystems.textsecuregcm.websocket.DeadLetterHandler; import org.whispersystems.textsecuregcm.websocket.ProvisioningConnectListener; import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator; import org.whispersystems.textsecuregcm.workers.DirectoryCommand; +import org.whispersystems.textsecuregcm.workers.PeriodicStatsCommand; import org.whispersystems.textsecuregcm.workers.TrimMessagesCommand; import org.whispersystems.textsecuregcm.workers.VacuumCommand; import org.whispersystems.websocket.WebSocketResourceProviderFactory; @@ -122,6 +123,7 @@ public class WhisperServerService extends Application("accountdb", "accountsdb.xml") { @Override public DataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) { @@ -269,15 +271,6 @@ public class WhisperServerService extends Application stringTimerSortedMap) { try { - logger.debug("Reporting metrics..."); + logger.info("Reporting metrics..."); URL url = new URL("https", hostname, 443, String.format("/report/metrics?t=%s&h=%s", token, host)); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); diff --git a/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java b/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java index 13f260c47..704d845fa 100644 --- a/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java +++ b/src/main/java/org/whispersystems/textsecuregcm/storage/Accounts.java @@ -77,6 +77,12 @@ public abstract class Accounts { @SqlQuery("SELECT * FROM accounts") public abstract Iterator getAll(); + @SqlQuery("SELECT COUNT(*) FROM accounts a, json_array_elements(a.data->'devices') devices WHERE devices->>'id' = '1' AND (devices->>'lastSeen')\\:\\:bigint >= :since") + public abstract int getActiveSinceCount(@Bind("since") long since); + + @SqlQuery("SELECT count(*) FROM accounts a, json_array_elements(a.data->'devices') devices WHERE devices->>'id' = '1' AND (devices->>'lastSeen')\\:\\:bigint >= :since AND (devices->>'signedPreKey') is null AND (devices->>'gcmId') is not null") + public abstract int getUnsignedKeysCount(@Bind("since") long since); + @Transaction(TransactionIsolationLevel.SERIALIZABLE) public long create(Account account) { removeAccount(account.getNumber()); diff --git a/src/main/java/org/whispersystems/textsecuregcm/workers/PeriodicStatsCommand.java b/src/main/java/org/whispersystems/textsecuregcm/workers/PeriodicStatsCommand.java new file mode 100644 index 000000000..260ea7138 --- /dev/null +++ b/src/main/java/org/whispersystems/textsecuregcm/workers/PeriodicStatsCommand.java @@ -0,0 +1,128 @@ +package org.whispersystems.textsecuregcm.workers; + +import com.codahale.metrics.Gauge; +import com.codahale.metrics.ScheduledReporter; +import com.codahale.metrics.SharedMetricRegistries; +import com.fasterxml.jackson.databind.DeserializationFeature; +import net.sourceforge.argparse4j.inf.Namespace; +import org.skife.jdbi.v2.DBI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.whispersystems.textsecuregcm.WhisperServerConfiguration; +import org.whispersystems.textsecuregcm.metrics.JsonMetricsReporter; +import org.whispersystems.textsecuregcm.metrics.JsonMetricsReporterFactory; +import org.whispersystems.textsecuregcm.storage.Accounts; +import org.whispersystems.textsecuregcm.util.Constants; + +import java.util.concurrent.TimeUnit; + +import static com.codahale.metrics.MetricRegistry.name; +import io.dropwizard.Application; +import io.dropwizard.cli.EnvironmentCommand; +import io.dropwizard.db.DataSourceFactory; +import io.dropwizard.jdbi.ImmutableListContainerFactory; +import io.dropwizard.jdbi.ImmutableSetContainerFactory; +import io.dropwizard.jdbi.OptionalContainerFactory; +import io.dropwizard.jdbi.args.OptionalArgumentFactory; +import io.dropwizard.metrics.ReporterFactory; +import io.dropwizard.setup.Environment; + +public class PeriodicStatsCommand extends EnvironmentCommand { + + private final Logger logger = LoggerFactory.getLogger(PeriodicStatsCommand.class); + + public PeriodicStatsCommand() { + super(new Application() { + @Override + public void run(WhisperServerConfiguration configuration, Environment environment) + throws Exception + { + + } + }, "stats", "Update periodic stats."); + } + + @Override + protected void run(Environment environment, Namespace namespace, + WhisperServerConfiguration configuration) + throws Exception + { + try { + environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + DataSourceFactory dbConfig = configuration.getReadDataSourceFactory(); + + if (dbConfig == null) { + logger.warn("No slave database configuration found!"); + return; + } + + DBI dbi = new DBI(dbConfig.getUrl(), dbConfig.getUser(), dbConfig.getPassword()); + dbi.registerArgumentFactory(new OptionalArgumentFactory(dbConfig.getDriverClass())); + dbi.registerContainerFactory(new ImmutableListContainerFactory()); + dbi.registerContainerFactory(new ImmutableSetContainerFactory()); + dbi.registerContainerFactory(new OptionalContainerFactory()); + + Accounts accounts = dbi.onDemand(Accounts.class); + long yesterday = TimeUnit.MILLISECONDS.toDays(System.currentTimeMillis()) - 1; + long monthAgo = yesterday - 30; + + logger.info("Calculating daily active"); + final int dailyActive = accounts.getActiveSinceCount(TimeUnit.DAYS.toMillis(yesterday)); + logger.info("Calculating monthly active"); + final int monthlyActive = accounts.getActiveSinceCount(TimeUnit.DAYS.toMillis(monthAgo)); + + logger.info("Calculating daily signed keys"); + final int dailyActiveNoSignedKeys = accounts.getUnsignedKeysCount(TimeUnit.DAYS.toMillis(yesterday)); + logger.info("Calculating monthly signed keys"); + final int monthlyActiveNoSignedKeys = accounts.getUnsignedKeysCount(TimeUnit.DAYS.toMillis(monthAgo )); + + environment.metrics().register(name(PeriodicStatsCommand.class, "daily_active"), + new Gauge() { + @Override + public Integer getValue() { + return dailyActive; + } + }); + + environment.metrics().register(name(PeriodicStatsCommand.class, "monthly_active"), + new Gauge() { + @Override + public Integer getValue() { + return monthlyActive; + } + }); + + environment.metrics().register(name(PeriodicStatsCommand.class, "daily_no_signed_keys"), + new Gauge() { + @Override + public Integer getValue() { + return dailyActiveNoSignedKeys; + } + }); + + environment.metrics().register(name(PeriodicStatsCommand.class, "monthly_no_signed_keys"), + new Gauge() { + @Override + public Integer getValue() { + return monthlyActiveNoSignedKeys; + } + }); + + + for (ReporterFactory reporterFactory : configuration.getMetricsFactory().getReporters()) { + ScheduledReporter reporter = reporterFactory.build(environment.metrics()); + logger.info("Reporting via: " + reporter); + reporter.report(); + logger.info("Reporting finished..."); + } + + } catch (Exception ex) { + logger.warn("Directory Exception", ex); + throw new RuntimeException(ex); + } finally { + Thread.sleep(3000); + System.exit(0); + } + } +}