From 2b07a2147732052fa38b49a1f880052b44126c71 Mon Sep 17 00:00:00 2001 From: Ravi Khadiwala Date: Wed, 26 Mar 2025 11:59:08 -0500 Subject: [PATCH] Add some additional backup metrics --- .../textsecuregcm/WhisperServerService.java | 4 +- .../textsecuregcm/backup/BackupManager.java | 2 +- .../controllers/ArchiveController.java | 37 +++++++++++--- .../grpc/BackupsAnonymousGrpcService.java | 15 +++++- .../grpc/BackupsGrpcService.java | 23 +++++++-- .../textsecuregcm/metrics/BackupMetrics.java | 51 +++++++++++++++++++ .../metrics/UserAgentTagUtil.java | 15 ++++-- .../workers/BackupMetricsCommand.java | 9 +++- .../controllers/ArchiveControllerTest.java | 3 +- .../grpc/BackupsAnonymousGrpcServiceTest.java | 3 +- .../grpc/BackupsGrpcServiceTest.java | 3 +- 11 files changed, 142 insertions(+), 23 deletions(-) create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/metrics/BackupMetrics.java diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 20ec0b24a..6c6aadb3e 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -178,6 +178,7 @@ import org.whispersystems.textsecuregcm.mappers.RateLimitExceededExceptionMapper import org.whispersystems.textsecuregcm.mappers.RegistrationServiceSenderExceptionMapper; import org.whispersystems.textsecuregcm.mappers.ServerRejectedExceptionMapper; import org.whispersystems.textsecuregcm.mappers.SubscriptionExceptionMapper; +import org.whispersystems.textsecuregcm.metrics.BackupMetrics; import org.whispersystems.textsecuregcm.metrics.MessageMetrics; import org.whispersystems.textsecuregcm.metrics.MetricsApplicationEventListener; import org.whispersystems.textsecuregcm.metrics.MetricsHttpChannelListener; @@ -955,6 +956,7 @@ public class WhisperServerService extends Application getBackupZKCredentials( @Mutable @Auth AuthenticatedDevice auth, + @HeaderParam(HttpHeaders.USER_AGENT) final String userAgent, @NotNull @QueryParam("redemptionStartSeconds") Long startSeconds, @NotNull @QueryParam("redemptionEndSeconds") Long endSeconds) { @@ -258,11 +269,17 @@ public class ArchiveController { auth.getAccount(), credentialType, Instant.ofEpochSecond(startSeconds), Instant.ofEpochSecond(endSeconds)) - .thenAccept(credentials -> credentialsByType.put(credentialType, credentials.stream() - .map(credential -> new BackupAuthCredentialsResponse.BackupAuthCredential( - credential.credential().serialize(), - credential.redemptionTime().getEpochSecond())) - .toList()))) + .thenAccept(credentials -> { + backupMetrics.updateGetCredentialCounter( + UserAgentTagUtil.getPlatformTag(userAgent), + credentialType, + credentials.size()); + credentialsByType.put(credentialType, credentials.stream() + .map(credential -> new BackupAuthCredentialsResponse.BackupAuthCredential( + credential.credential().serialize(), + credential.redemptionTime().getEpochSecond())) + .toList()); + })) .toArray(CompletableFuture[]::new)) .thenApply(ignored -> new BackupAuthCredentialsResponse(credentialsByType.entrySet().stream() .collect(Collectors.toMap( @@ -586,6 +603,7 @@ public class ArchiveController { @ApiResponseZkAuth public CompletionStage copyMedia( @ReadOnly @Auth final Optional account, + @HeaderParam(HttpHeaders.USER_AGENT) final String userAgent, @Parameter(description = BackupAuthCredentialPresentationHeader.DESCRIPTION, schema = @Schema(implementation = String.class)) @NotNull @@ -605,6 +623,7 @@ public class ArchiveController { .fromFuture(backupManager.authenticateBackupUser(presentation.presentation, signature.signature)) .flatMap(backupUser -> backupManager.copyToBackup(backupUser, List.of(copyMediaRequest.toCopyParameters())) .next() + .doOnNext(result -> backupMetrics.updateCopyCounter(result, UserAgentTagUtil.getPlatformTag(userAgent))) .map(copyResult -> switch (copyResult.outcome()) { case SUCCESS -> new CopyMediaResponse(copyResult.cdn()); case SOURCE_WRONG_LENGTH -> throw new BadRequestException("Invalid length"); @@ -683,6 +702,7 @@ public class ArchiveController { @ApiResponseZkAuth public CompletionStage copyMedia( @ReadOnly @Auth final Optional account, + @HeaderParam(HttpHeaders.USER_AGENT) final String userAgent, @Parameter(description = BackupAuthCredentialPresentationHeader.DESCRIPTION, schema = @Schema(implementation = String.class)) @NotNull @@ -701,6 +721,7 @@ public class ArchiveController { final Stream copyParams = copyMediaRequest.items().stream().map(CopyMediaRequest::toCopyParameters); return Mono.fromFuture(backupManager.authenticateBackupUser(presentation.presentation, signature.signature)) .flatMapMany(backupUser -> backupManager.copyToBackup(backupUser, copyParams.toList())) + .doOnNext(result -> backupMetrics.updateCopyCounter(result, UserAgentTagUtil.getPlatformTag(userAgent))) .map(CopyMediaBatchResponse.Entry::fromCopyResult) .collectList() .map(list -> Response.status(207).entity(new CopyMediaBatchResponse(list)).build()) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java index c785493df..737aaf1c1 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java @@ -8,6 +8,9 @@ import com.google.protobuf.ByteString; import io.grpc.Status; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; import org.signal.chat.backup.CopyMediaRequest; import org.signal.chat.backup.CopyMediaResponse; import org.signal.chat.backup.DeleteAllRequest; @@ -36,15 +39,22 @@ import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser; import org.whispersystems.textsecuregcm.backup.BackupManager; import org.whispersystems.textsecuregcm.backup.CopyParameters; import org.whispersystems.textsecuregcm.backup.MediaEncryptionParameters; +import org.whispersystems.textsecuregcm.controllers.ArchiveController; +import org.whispersystems.textsecuregcm.metrics.BackupMetrics; +import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name; + public class BackupsAnonymousGrpcService extends ReactorBackupsAnonymousGrpc.BackupsAnonymousImplBase { private final BackupManager backupManager; + private final BackupMetrics backupMetrics; - public BackupsAnonymousGrpcService(final BackupManager backupManager) { + public BackupsAnonymousGrpcService(final BackupManager backupManager, final BackupMetrics backupMetrics) { this.backupManager = backupManager; + this.backupMetrics = backupMetrics; } @Override @@ -115,6 +125,9 @@ public class BackupsAnonymousGrpcService extends ReactorBackupsAnonymousGrpc.Bac fromUnsignedExact(item.getObjectLength()), new MediaEncryptionParameters(item.getEncryptionKey().toByteArray(), item.getHmacKey().toByteArray()), item.getMediaId().toByteArray())).toList())) + .doOnNext(result -> backupMetrics.updateCopyCounter( + result, + UserAgentTagUtil.getPlatformTag(RequestAttributesUtil.getUserAgent().orElse(null)))) .map(copyResult -> { CopyMediaResponse.Builder builder = CopyMediaResponse .newBuilder() diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcService.java index 605da9fba..eec156d7d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcService.java @@ -9,6 +9,9 @@ import io.grpc.Status; import java.time.Instant; import java.util.List; import java.util.stream.Collectors; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; import org.signal.chat.backup.GetBackupAuthCredentialsRequest; import org.signal.chat.backup.GetBackupAuthCredentialsResponse; import org.signal.chat.backup.ReactorBackupsGrpc; @@ -24,18 +27,25 @@ import org.signal.libsignal.zkgroup.receipts.ReceiptCredentialPresentation; import org.whispersystems.textsecuregcm.auth.grpc.AuthenticatedDevice; import org.whispersystems.textsecuregcm.auth.grpc.AuthenticationUtil; import org.whispersystems.textsecuregcm.backup.BackupAuthManager; +import org.whispersystems.textsecuregcm.controllers.ArchiveController; +import org.whispersystems.textsecuregcm.metrics.BackupMetrics; +import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil; import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.AccountsManager; import reactor.core.publisher.Mono; +import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name; + public class BackupsGrpcService extends ReactorBackupsGrpc.BackupsImplBase { private final AccountsManager accountManager; private final BackupAuthManager backupAuthManager; + private final BackupMetrics backupMetrics; - public BackupsGrpcService(final AccountsManager accountManager, final BackupAuthManager backupAuthManager) { + public BackupsGrpcService(final AccountsManager accountManager, final BackupAuthManager backupAuthManager, final BackupMetrics backupMetrics) { this.accountManager = accountManager; this.backupAuthManager = backupAuthManager; + this.backupMetrics = backupMetrics; } @@ -67,19 +77,26 @@ public class BackupsGrpcService extends ReactorBackupsGrpc.BackupsImplBase { @Override public Mono getBackupAuthCredentials(GetBackupAuthCredentialsRequest request) { + final Tag platformTag = UserAgentTagUtil.getPlatformTag(RequestAttributesUtil.getUserAgent().orElse(null)); return authenticatedAccount().flatMap(account -> { + final Mono> messageCredentials = Mono.fromCompletionStage(() -> backupAuthManager.getBackupAuthCredentials( account, BackupCredentialType.MESSAGES, Instant.ofEpochSecond(request.getRedemptionStart()), - Instant.ofEpochSecond(request.getRedemptionStop()))); + Instant.ofEpochSecond(request.getRedemptionStop()))) + .doOnSuccess(credentials -> + backupMetrics.updateGetCredentialCounter(platformTag, BackupCredentialType.MESSAGES, credentials.size())); + final Mono> mediaCredentials = Mono.fromCompletionStage(() -> backupAuthManager.getBackupAuthCredentials( account, BackupCredentialType.MEDIA, Instant.ofEpochSecond(request.getRedemptionStart()), - Instant.ofEpochSecond(request.getRedemptionStop()))); + Instant.ofEpochSecond(request.getRedemptionStop()))) + .doOnSuccess(credentials -> + backupMetrics.updateGetCredentialCounter(platformTag, BackupCredentialType.MEDIA, credentials.size())); return messageCredentials.zipWith(mediaCredentials, (messageCreds, mediaCreds) -> GetBackupAuthCredentialsResponse.newBuilder() diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/BackupMetrics.java b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/BackupMetrics.java new file mode 100644 index 000000000..5757b339a --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/BackupMetrics.java @@ -0,0 +1,51 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.textsecuregcm.metrics; + +import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name; + +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import org.signal.libsignal.zkgroup.backups.BackupCredentialType; +import org.whispersystems.textsecuregcm.backup.CopyResult; + +public class BackupMetrics { + + private final static String COPY_MEDIA_COUNTER_NAME = name(BackupMetrics.class, "copyMedia"); + private final static String GET_BACKUP_CREDENTIALS_NAME = name(BackupMetrics.class, "getBackupCredentials"); + + + private MeterRegistry registry; + + public BackupMetrics() { + this(Metrics.globalRegistry); + } + + @VisibleForTesting + BackupMetrics(MeterRegistry registry) { + this.registry = registry; + } + + public void updateCopyCounter(final CopyResult copyResult, final Tag platformTag) { + registry.counter(COPY_MEDIA_COUNTER_NAME, Tags.of( + platformTag, + Tag.of("outcome", copyResult.outcome().name().toLowerCase()))) + .increment(); + } + + public void updateGetCredentialCounter(final Tag platformTag, BackupCredentialType credentialType, + final int numCredentials) { + Metrics.counter(GET_BACKUP_CREDENTIALS_NAME, Tags.of( + platformTag, + Tag.of("num", Integer.toString(numCredentials)), + Tag.of("type", credentialType.name().toLowerCase()))) + .increment(); + } + +} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/UserAgentTagUtil.java b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/UserAgentTagUtil.java index 6a4d09ddd..90f0f5d5c 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/UserAgentTagUtil.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/UserAgentTagUtil.java @@ -12,6 +12,7 @@ import org.whispersystems.textsecuregcm.storage.ClientReleaseManager; import org.whispersystems.textsecuregcm.util.ua.UnrecognizedUserAgentException; import org.whispersystems.textsecuregcm.util.ua.UserAgent; import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil; +import javax.annotation.Nullable; /** * Utility class for extracting platform/version metrics tags from User-Agent strings. @@ -26,15 +27,19 @@ public class UserAgentTagUtil { } public static Tag getPlatformTag(final String userAgentString) { - String platform; + + UserAgent userAgent = null; try { - platform = UserAgentUtil.parseUserAgentString(userAgentString).getPlatform().name().toLowerCase(); - } catch (final UnrecognizedUserAgentException e) { - platform = "unrecognized"; + userAgent = UserAgentUtil.parseUserAgentString(userAgentString); + } catch (final UnrecognizedUserAgentException ignored) { } - return Tag.of(PLATFORM_TAG, platform); + return getPlatformTag(userAgent); + } + + public static Tag getPlatformTag(@Nullable final UserAgent userAgent) { + return Tag.of(PLATFORM_TAG, userAgent != null ? userAgent.getPlatform().name().toLowerCase() : "unrecognized"); } public static Optional getClientVersionTag(final String userAgentString, final ClientReleaseManager clientReleaseManager) { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/BackupMetricsCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/BackupMetricsCommand.java index c5ef322be..3a989be86 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/BackupMetricsCommand.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/BackupMetricsCommand.java @@ -93,7 +93,14 @@ public class BackupMetricsCommand extends AbstractCommandWithDependencies { } timeSinceLastRefresh.record(timeSince(backupMetadata.lastRefresh()).getSeconds()); timeSinceLastMediaRefresh.record(timeSince(backupMetadata.lastMediaRefresh()).getSeconds()); - Metrics.counter(backupsCounterName, "subscribed", String.valueOf(subscribed)).increment(); + + // Report that the backup is out of quota if it cannot store a max size media object + final boolean quotaExhausted = backupMetadata.bytesUsed() >= + (BackupManager.MAX_TOTAL_BACKUP_MEDIA_BYTES - BackupManager.MAX_MEDIA_OBJECT_SIZE); + + Metrics.counter(backupsCounterName, + "subscribed", String.valueOf(subscribed), + "quotaExhausted", String.valueOf(quotaExhausted)).increment(); }) .count() .block(); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java index 5a45c020f..660b7ec5c 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ArchiveControllerTest.java @@ -72,6 +72,7 @@ import org.whispersystems.textsecuregcm.entities.RemoteAttachment; import org.whispersystems.textsecuregcm.mappers.CompletionExceptionMapper; import org.whispersystems.textsecuregcm.mappers.GrpcStatusRuntimeExceptionMapper; import org.whispersystems.textsecuregcm.mappers.RateLimitExceededExceptionMapper; +import org.whispersystems.textsecuregcm.metrics.BackupMetrics; import org.whispersystems.textsecuregcm.tests.util.AuthHelper; import org.whispersystems.textsecuregcm.util.EnumMapUtil; import org.whispersystems.textsecuregcm.util.SystemMapper; @@ -94,7 +95,7 @@ public class ArchiveControllerTest { .addProvider(new RateLimitExceededExceptionMapper()) .setMapper(SystemMapper.jsonMapper()) .setTestContainerFactory(new GrizzlyWebTestContainerFactory()) - .addResource(new ArchiveController(backupAuthManager, backupManager)) + .addResource(new ArchiveController(backupAuthManager, backupManager, new BackupMetrics())) .build(); private final UUID aci = UUID.randomUUID(); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java index 5b1e3681c..4297201cb 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java @@ -57,6 +57,7 @@ import org.whispersystems.textsecuregcm.backup.BackupManager; import org.whispersystems.textsecuregcm.backup.BackupUploadDescriptor; import org.whispersystems.textsecuregcm.backup.CopyResult; import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; +import org.whispersystems.textsecuregcm.metrics.BackupMetrics; import org.whispersystems.textsecuregcm.util.TestRandomUtil; import reactor.core.publisher.Flux; @@ -74,7 +75,7 @@ class BackupsAnonymousGrpcServiceTest extends @Override protected BackupsAnonymousGrpcService createServiceBeforeEachTest() { - return new BackupsAnonymousGrpcService(backupManager); + return new BackupsAnonymousGrpcService(backupManager, new BackupMetrics()); } @BeforeEach diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcServiceTest.java index 7a6e5c42e..5fee5ee00 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcServiceTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcServiceTest.java @@ -54,6 +54,7 @@ import org.signal.libsignal.zkgroup.receipts.ServerZkReceiptOperations; import org.whispersystems.textsecuregcm.backup.BackupAuthManager; import org.whispersystems.textsecuregcm.backup.BackupAuthTestUtil; import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; +import org.whispersystems.textsecuregcm.metrics.BackupMetrics; import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.AccountsManager; import org.whispersystems.textsecuregcm.util.EnumMapUtil; @@ -77,7 +78,7 @@ class BackupsGrpcServiceTest extends SimpleBaseGrpcTest