From 635f669a3293afa59a658be399d1f9ab0c3fb1ce Mon Sep 17 00:00:00 2001 From: Jon Chambers Date: Fri, 12 Feb 2021 15:15:03 -0500 Subject: [PATCH] Count slow queue drain events by platform. --- .../websocket/WebSocketConnection.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnection.java b/service/src/main/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnection.java index 912dcf880..fbaedd528 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnection.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/websocket/WebSocketConnection.java @@ -68,11 +68,14 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac private static final String INITIAL_QUEUE_LENGTH_DISTRIBUTION_NAME = name(WebSocketConnection.class, "initialQueueLength"); private static final String INITIAL_QUEUE_DRAIN_TIMER_NAME = name(WebSocketConnection.class, "drainInitialQueue"); + private static final String SLOW_QUEUE_DRAIN_COUNTER_NAME = name(WebSocketConnection.class, "slowQueueDrain"); private static final String DISPLACEMENT_COUNTER_NAME = name(WebSocketConnection.class, "displacement"); private static final String NON_SUCCESS_RESPONSE_COUNTER_NAME = name(WebSocketConnection.class, "clientNonSuccessResponse"); private static final String STATUS_CODE_TAG = "status"; private static final String STATUS_MESSAGE_TAG = "message"; + private static final long SLOW_DRAIN_THRESHOLD = 10_000; + @VisibleForTesting static final int MAX_DESKTOP_MESSAGE_SIZE = 1024 * 1024; @@ -201,8 +204,15 @@ public class WebSocketConnection implements MessageAvailabilityListener, Displac queueClearedFuture.whenComplete((v, cause) -> { if (cause == null && sentInitialQueueEmptyMessage.compareAndSet(false, true)) { - Metrics.summary(INITIAL_QUEUE_LENGTH_DISTRIBUTION_NAME, List.of(UserAgentTagUtil.getPlatformTag(client.getUserAgent()))).record(sentMessageCounter.sum()); - Metrics.timer(INITIAL_QUEUE_DRAIN_TIMER_NAME, List.of(UserAgentTagUtil.getPlatformTag(client.getUserAgent()))).record(System.currentTimeMillis() - queueDrainStartTime.get(), TimeUnit.MILLISECONDS); + final List tags = List.of(UserAgentTagUtil.getPlatformTag(client.getUserAgent())); + final long drainDuration = System.currentTimeMillis() - queueDrainStartTime.get(); + + Metrics.summary(INITIAL_QUEUE_LENGTH_DISTRIBUTION_NAME, tags).record(sentMessageCounter.sum()); + Metrics.timer(INITIAL_QUEUE_DRAIN_TIMER_NAME, tags).record(drainDuration, TimeUnit.MILLISECONDS); + + if (drainDuration > SLOW_DRAIN_THRESHOLD) { + Metrics.counter(SLOW_QUEUE_DRAIN_COUNTER_NAME, tags).increment(); + } client.sendRequest("PUT", "/api/v1/queue/empty", Collections.singletonList(TimestampHeaderUtil.getTimestampHeader()), Optional.empty()); }