MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Report system resource metrics from background tasks

This commit is contained in:
Chris Eager 2023-06-14 16:48:23 -05:00 committed by GitHub
parent 9cfd88a23f
commit 13afdbda97
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 94 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
View File

@ -46,7 +46,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import javax.servlet.DispatcherType;
import javax.servlet.FilterRegistration;
import javax.servlet.ServletRegistration;
@ -127,18 +126,8 @@ import org.whispersystems.textsecuregcm.mappers.NonNormalizedPhoneNumberExceptio
import org.whispersystems.textsecuregcm.mappers.RateLimitExceededExceptionMapper;
import org.whispersystems.textsecuregcm.mappers.RegistrationServiceSenderExceptionMapper;
import org.whispersystems.textsecuregcm.mappers.ServerRejectedExceptionMapper;
import org.whispersystems.textsecuregcm.metrics.ApplicationShutdownMonitor;
import org.whispersystems.textsecuregcm.metrics.BufferPoolGauges;
import org.whispersystems.textsecuregcm.metrics.CpuUsageGauge;
import org.whispersystems.textsecuregcm.metrics.FileDescriptorGauge;
import org.whispersystems.textsecuregcm.metrics.FreeMemoryGauge;
import org.whispersystems.textsecuregcm.metrics.GarbageCollectionGauges;
import org.whispersystems.textsecuregcm.metrics.MaxFileDescriptorGauge;
import org.whispersystems.textsecuregcm.metrics.MetricsApplicationEventListener;
import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
import org.whispersystems.textsecuregcm.metrics.NetworkReceivedGauge;
import org.whispersystems.textsecuregcm.metrics.NetworkSentGauge;
import org.whispersystems.textsecuregcm.metrics.OperatingSystemMemoryGauge;
import org.whispersystems.textsecuregcm.metrics.ReportedMessageMetricsListener;
import org.whispersystems.textsecuregcm.metrics.TrafficSource;
import org.whispersystems.textsecuregcm.providers.MultiRecipientMessageProvider;
@ -783,7 +772,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
webSocketEnvironment.jersey().register(controller);
}
WebSocketEnvironment<AuthenticatedAccount> provisioningEnvironment = new WebSocketEnvironment<>(environment,
webSocketEnvironment.getRequestLog(), 60000);
provisioningEnvironment.jersey().register(new WebsocketRefreshApplicationEventListener(accountsManager, clientPresenceManager));
@ -817,21 +805,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
environment.healthChecks().register("cacheCluster", new RedisClusterHealthCheck(cacheCluster));
environment.lifecycle().manage(new ApplicationShutdownMonitor(Metrics.globalRegistry));
environment.metrics().register(name(CpuUsageGauge.class, "cpu"), new CpuUsageGauge(3, TimeUnit.SECONDS));
environment.metrics().register(name(FreeMemoryGauge.class, "free_memory"), new FreeMemoryGauge());
environment.metrics().register(name(NetworkSentGauge.class, "bytes_sent"), new NetworkSentGauge());
environment.metrics().register(name(NetworkReceivedGauge.class, "bytes_received"), new NetworkReceivedGauge());
environment.metrics().register(name(FileDescriptorGauge.class, "fd_count"), new FileDescriptorGauge());
environment.metrics().register(name(MaxFileDescriptorGauge.class, "max_fd_count"), new MaxFileDescriptorGauge());
environment.metrics()
.register(name(OperatingSystemMemoryGauge.class, "buffers"), new OperatingSystemMemoryGauge("Buffers"));
environment.metrics()
.register(name(OperatingSystemMemoryGauge.class, "cached"), new OperatingSystemMemoryGauge("Cached"));
BufferPoolGauges.registerMetrics();
GarbageCollectionGauges.registerMetrics();
MetricsUtil.registerSystemResourceMetrics(environment);
}

60
service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java
View File

@ -5,7 +5,11 @@
package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import io.dropwizard.lifecycle.JettyManaged;
import io.dropwizard.lifecycle.setup.LifecycleEnvironment;
import io.dropwizard.metrics.ScheduledReporterManager;
import io.dropwizard.setup.Environment;
import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.Metrics;
@ -13,6 +17,7 @@ import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.config.MeterFilter;
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
import io.micrometer.datadog.DatadogMeterRegistry;
import java.util.concurrent.TimeUnit;
import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
import org.whispersystems.textsecuregcm.WhisperServerVersion;
import org.whispersystems.textsecuregcm.util.Constants;
@ -66,5 +71,60 @@ public class MetricsUtil {
}
environment.lifecycle().manage(new MicrometerRegistryManager(Metrics.globalRegistry));
environment.lifecycle().manage(new ApplicationShutdownMonitor(Metrics.globalRegistry));
}
public static void registerSystemResourceMetrics(final Environment environment) {
environment.metrics().register(name(CpuUsageGauge.class, "cpu"), new CpuUsageGauge(3, TimeUnit.SECONDS));
environment.metrics().register(name(FreeMemoryGauge.class, "free_memory"), new FreeMemoryGauge());
environment.metrics().register(name(NetworkSentGauge.class, "bytes_sent"), new NetworkSentGauge());
environment.metrics().register(name(NetworkReceivedGauge.class, "bytes_received"), new NetworkReceivedGauge());
environment.metrics().register(name(FileDescriptorGauge.class, "fd_count"), new FileDescriptorGauge());
environment.metrics().register(name(MaxFileDescriptorGauge.class, "max_fd_count"), new MaxFileDescriptorGauge());
environment.metrics()
.register(name(OperatingSystemMemoryGauge.class, "buffers"), new OperatingSystemMemoryGauge("Buffers"));
environment.metrics()
.register(name(OperatingSystemMemoryGauge.class, "cached"), new OperatingSystemMemoryGauge("Cached"));
BufferPoolGauges.registerMetrics();
GarbageCollectionGauges.registerMetrics();
}
/**
* For use in commands where {@link JettyManaged} doesn't apply
*
* @see io.dropwizard.metrics.MetricsFactory#configure(LifecycleEnvironment, MetricRegistry)
*/
public static void startManagedReporters(Environment environment) {
environment.lifecycle().getManagedObjects().forEach(managedObject -> {
if (managedObject instanceof JettyManaged jettyManaged) {
if (jettyManaged.getManaged() instanceof ScheduledReporterManager scheduledReporterManager) {
try {
scheduledReporterManager.start();
} catch (final Exception e) {
throw new RuntimeException(e);
}
}
}
});
}
/**
* For use in commands where {@link JettyManaged} doesn't apply
*
* @see io.dropwizard.metrics.MetricsFactory#configure(LifecycleEnvironment, MetricRegistry)
*/
public static void stopManagedReporters(final Environment environment) {
environment.lifecycle().getManagedObjects().forEach(lifeCycle -> {
if (lifeCycle instanceof JettyManaged jettyManaged) {
if (jettyManaged.getManaged() instanceof ScheduledReporterManager scheduledReporterManager) {
try {
scheduledReporterManager.stop();
} catch (final Exception e) {
throw new RuntimeException(e);
}
}
}
});
}
}

56
service/src/main/java/org/whispersystems/textsecuregcm/workers/CrawlAccountsCommand.java
View File

@ -97,38 +97,39 @@ public class CrawlAccountsCommand extends EnvironmentCommand<WhisperServerConfig
DynamicConfiguration.class);
dynamicConfigurationManager.start();
MetricsUtil.registerSystemResourceMetrics(environment);
final AccountDatabaseCrawler crawler =
final AccountDatabaseCrawler crawler = switch ((CrawlType) namespace.get(CRAWL_TYPE)) {
case GENERAL_PURPOSE -> {
final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
cacheCluster,
AccountDatabaseCrawlerCache.GENERAL_PURPOSE_PREFIX);
switch ((CrawlType) namespace.get(CRAWL_TYPE)) {
case GENERAL_PURPOSE -> {
final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
cacheCluster,
AccountDatabaseCrawlerCache.GENERAL_PURPOSE_PREFIX);
yield new AccountDatabaseCrawler("General-purpose account crawler",
accountsManager,
accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
dynamicConfigurationManager
);
}
case ACCOUNT_CLEANER -> {
final ExecutorService accountDeletionExecutor = environment.lifecycle()
.executorService(name(getClass(), "accountCleaner-%d")).maxThreads(16).minThreads(16).build();
yield new AccountDatabaseCrawler("General-purpose account crawler",
accountsManager,
accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
dynamicConfigurationManager
);
}
case ACCOUNT_CLEANER -> {
final ExecutorService accountDeletionExecutor = environment.lifecycle()
.executorService(name(getClass(), "accountCleaner-%d")).maxThreads(16).minThreads(16).build();
final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
cacheCluster, AccountDatabaseCrawlerCache.ACCOUNT_CLEANER_PREFIX);
final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
cacheCluster, AccountDatabaseCrawlerCache.ACCOUNT_CLEANER_PREFIX);
yield new AccountDatabaseCrawler("Account cleaner crawler",
accountsManager,
accountDatabaseCrawlerCache,
List.of(new AccountCleaner(accountsManager, accountDeletionExecutor)),
configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
dynamicConfigurationManager
);
}
};
yield new AccountDatabaseCrawler("Account cleaner crawler",
accountsManager,
accountDatabaseCrawlerCache,
List.of(new AccountCleaner(accountsManager, accountDeletionExecutor)),
configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
dynamicConfigurationManager
);
}
};
MetricsUtil.startManagedReporters(environment);
try {
crawler.crawlAllAccounts();
@ -136,5 +137,6 @@ public class CrawlAccountsCommand extends EnvironmentCommand<WhisperServerConfig
LoggerFactory.getLogger(CrawlAccountsCommand.class).error("Error crawling accounts", e);
}
MetricsUtil.stopManagedReporters(environment);
}
}

2
service/src/main/java/org/whispersystems/textsecuregcm/workers/MessagePersisterServiceCommand.java
View File

@ -70,6 +70,8 @@ public class MessagePersisterServiceCommand extends ServerCommand<WhisperServerC
environment.lifecycle().manage(deps.messagesCache());
environment.lifecycle().manage(messagePersister);
MetricsUtil.registerSystemResourceMetrics(environment);
super.run(environment, namespace, configuration);
}

2
service/src/main/java/org/whispersystems/textsecuregcm/workers/ScheduledApnPushNotificationSenderServiceCommand.java
View File

@ -79,6 +79,8 @@ public class ScheduledApnPushNotificationSenderServiceCommand extends ServerComm
environment.lifecycle().manage(apnSender);
environment.lifecycle().manage(apnPushNotificationScheduler);
MetricsUtil.registerSystemResourceMetrics(environment);
super.run(environment, namespace, configuration);
}