Introduce a set of gauges for various network statistics as reported by `nstat`.

pom.xml

@@ -138,7 +138,7 @@
         </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-api</artifactId>
+            <artifactId>junit-jupiter</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>

service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java

@@ -110,6 +110,7 @@ import org.whispersystems.textsecuregcm.metrics.MaxFileDescriptorGauge;
 import org.whispersystems.textsecuregcm.metrics.MetricsApplicationEventListener;
 import org.whispersystems.textsecuregcm.metrics.NetworkReceivedGauge;
 import org.whispersystems.textsecuregcm.metrics.NetworkSentGauge;
+import org.whispersystems.textsecuregcm.metrics.NstatCounters;
 import org.whispersystems.textsecuregcm.metrics.OperatingSystemMemoryGauge;
 import org.whispersystems.textsecuregcm.metrics.PushLatencyManager;
 import org.whispersystems.textsecuregcm.metrics.TrafficSource;
@@ -221,7 +222,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
   {
     SharedMetricRegistries.add(Constants.METRICS_NAME, environment.metrics());
 
-    Metrics.addRegistry(new WavefrontMeterRegistry(new WavefrontConfig() {
+    final WavefrontConfig wavefrontConfig = new WavefrontConfig() {
       @Override
       public String get(final String key) {
         return null;
@@ -236,7 +237,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
       public int batchSize() {
         return config.getMicrometerConfiguration().getBatchSize();
       }
-    }, Clock.SYSTEM) {
+    };
+
+    Metrics.addRegistry(new WavefrontMeterRegistry(wavefrontConfig, Clock.SYSTEM) {
       @Override
       protected DistributionStatisticConfig defaultHistogramConfig() {
         return DistributionStatisticConfig.builder()
@@ -492,6 +495,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
 
     BufferPoolGauges.registerMetrics();
     GarbageCollectionGauges.registerMetrics();
+
+    new NstatCounters().registerMetrics(recurringJobExecutor, wavefrontConfig.step());
   }
 
   private void registerExceptionMappers(Environment environment, WebSocketEnvironment<Account> webSocketEnvironment, WebSocketEnvironment<Account> provisioningEnvironment) {

service/src/main/java/org/whispersystems/textsecuregcm/metrics/NstatCounters.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2021 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.metrics;
+
+import static com.codahale.metrics.MetricRegistry.name;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import io.micrometer.core.instrument.Metrics;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NstatCounters {
+
+  private final Map<String, Long> networkStatistics = new ConcurrentHashMap<>();
+
+  private static final String[] NSTAT_COMMAND_LINE = new String[] { "nstat", "--zero", "--json", "--noupdate", "--ignore" };
+  private static final String[] EXCLUDE_METRIC_NAME_PREFIXES = new String[] { "Icmp", "Udp", "Ip6" };
+
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
+      .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+  private static final Logger log = LoggerFactory.getLogger(NstatCounters.class);
+
+  @VisibleForTesting
+  static class NetworkStatistics {
+    private final Map<String, Long> kernelStatistics;
+
+    @JsonCreator
+    private NetworkStatistics(@JsonProperty("kernel") final Map<String, Long> kernelStatistics) {
+      this.kernelStatistics = kernelStatistics;
+    }
+
+    public Map<String, Long> getKernelStatistics() {
+      return kernelStatistics;
+    }
+  }
+
+  public void registerMetrics(final ScheduledExecutorService refreshService, final Duration refreshInterval) {
+    refreshNetworkStatistics();
+
+    networkStatistics.keySet().stream()
+        .filter(NstatCounters::shouldIncludeMetric)
+        .forEach(metricName -> Metrics.globalRegistry.more().counter(name(getClass(), "kernel", metricName),
+            Collections.emptyList(), networkStatistics, statistics -> statistics.get(metricName)));
+
+    refreshService.scheduleAtFixedRate(this::refreshNetworkStatistics,
+        refreshInterval.toMillis(), refreshInterval.toMillis(), TimeUnit.MILLISECONDS);
+  }
+
+  private void refreshNetworkStatistics() {
+    try {
+      networkStatistics.putAll(loadNetworkStatistics().getKernelStatistics());
+    } catch (final InterruptedException | IOException e) {
+      log.warn("Failed to refresh network statistics", e);
+    }
+  }
+
+  @VisibleForTesting
+  static boolean shouldIncludeMetric(final String metricName) {
+    for (final String prefix : EXCLUDE_METRIC_NAME_PREFIXES) {
+      if (metricName.startsWith(prefix)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  @VisibleForTesting
+  static NetworkStatistics loadNetworkStatistics() throws IOException, InterruptedException {
+    final Process nstatProcess = Runtime.getRuntime().exec(NSTAT_COMMAND_LINE);
+
+    if (nstatProcess.waitFor() == 0) {
+      return OBJECT_MAPPER.readValue(nstatProcess.getInputStream(), NetworkStatistics.class);
+    } else {
+      throw new IOException("nstat process did not exit normally");
+    }
+  }
+}

service/src/test/java/org/whispersystems/textsecuregcm/metrics/NstatCountersTest.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.metrics;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.whispersystems.textsecuregcm.metrics.NstatCounters.NetworkStatistics;
+
+import java.io.IOException;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+class NstatCountersTest {
+
+  @Test
+  @EnabledOnOs(OS.LINUX)
+  void loadNetworkStatistics() throws IOException, InterruptedException {
+    final NetworkStatistics networkStatistics = NstatCounters.loadNetworkStatistics();
+
+    assertNotNull(networkStatistics.getKernelStatistics());
+    assertFalse(networkStatistics.getKernelStatistics().isEmpty());
+  }
+
+  @ParameterizedTest
+  @MethodSource("shouldIncludeMetricNameProvider")
+  void shouldIncludeMetric(final String metricName, final boolean expectInclude) {
+    assertEquals(expectInclude, NstatCounters.shouldIncludeMetric(metricName));
+  }
+
+  static Stream<Arguments> shouldIncludeMetricNameProvider() {
+    return Stream.of(Arguments.of("IpInReceives", true),
+        Arguments.of("TcpActiveOpens", true),
+        Arguments.of("UdpInDatagrams", false),
+        Arguments.of("Ip6InReceives", false),
+        Arguments.of("Udp6InDatagrams", false),
+        Arguments.of("TcpExtSyncookiesSent", true),
+        Arguments.of("IpExtInNoRoutes", true));
+  }
+}