From 3c1705994d94e36147a0456b0d036bcbac913b5f Mon Sep 17 00:00:00 2001
From: Jon Chambers <jon@signal.org>
Date: Tue, 5 Oct 2021 15:58:55 -0400
Subject: [PATCH] Count accounts with non-normalized phone numbers

---
 .../textsecuregcm/WhisperServerService.java   |  2 +
 .../NonNormalizedAccountCrawlerListener.java  | 91 +++++++++++++++++++
 ...nNormalizedAccountCrawlerListenerTest.java | 39 ++++++++
 3 files changed, 132 insertions(+)
 create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/storage/NonNormalizedAccountCrawlerListener.java
 create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/storage/NonNormalizedAccountCrawlerListenerTest.java

diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
index e090b356f..77014ed16 100644
--- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
@@ -172,6 +172,7 @@ import org.whispersystems.textsecuregcm.storage.MessagePersister;
 import org.whispersystems.textsecuregcm.storage.MessagesCache;
 import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
 import org.whispersystems.textsecuregcm.storage.MessagesManager;
+import org.whispersystems.textsecuregcm.storage.NonNormalizedAccountCrawlerListener;
 import org.whispersystems.textsecuregcm.storage.Profiles;
 import org.whispersystems.textsecuregcm.storage.ProfilesManager;
 import org.whispersystems.textsecuregcm.storage.PubSubManager;
@@ -496,6 +497,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
           directoryServerConfiguration.getReplicationName(), directoryReconciliationClient);
       deletedAccountsDirectoryReconcilers.add(deletedAccountsDirectoryReconciler);
     }
+    accountDatabaseCrawlerListeners.add(new NonNormalizedAccountCrawlerListener(metricsCluster));
     accountDatabaseCrawlerListeners.add(new ContactDiscoveryWriter(accountsManager));
     // PushFeedbackProcessor may update device properties
     accountDatabaseCrawlerListeners.add(new PushFeedbackProcessor(accountsManager));
diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/NonNormalizedAccountCrawlerListener.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/NonNormalizedAccountCrawlerListener.java
new file mode 100644
index 000000000..da255c25e
--- /dev/null
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/NonNormalizedAccountCrawlerListener.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2013-2021 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.i18n.phonenumbers.NumberParseException;
+import com.google.i18n.phonenumbers.PhoneNumberUtil;
+import com.google.i18n.phonenumbers.PhoneNumberUtil.PhoneNumberFormat;
+import com.google.i18n.phonenumbers.Phonenumber.PhoneNumber;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+public class NonNormalizedAccountCrawlerListener extends AccountDatabaseCrawlerListener {
+
+  private final FaultTolerantRedisCluster metricsCluster;
+
+  private static final String NORMALIZED_NUMBER_COUNT_KEY = "NonNormalizedAccountCrawlerListener::normalized";
+  private static final String NON_NORMALIZED_NUMBER_COUNT_KEY = "NonNormalizedAccountCrawlerListener::nonNormalized";
+
+  private static final PhoneNumberUtil PHONE_NUMBER_UTIL = PhoneNumberUtil.getInstance();
+
+  private static final Logger log = LoggerFactory.getLogger(NonNormalizedAccountCrawlerListener.class);
+
+  public NonNormalizedAccountCrawlerListener(final FaultTolerantRedisCluster metricsCluster) {
+    this.metricsCluster = metricsCluster;
+  }
+
+  @Override
+  public void onCrawlStart() {
+    metricsCluster.useCluster(connection -> {
+      connection.sync().del(NORMALIZED_NUMBER_COUNT_KEY, NON_NORMALIZED_NUMBER_COUNT_KEY);
+    });
+  }
+
+  @Override
+  protected void onCrawlChunk(final Optional<UUID> fromUuid, final List<Account> chunkAccounts) {
+
+    final int normalizedNumbers;
+    final int nonNormalizedNumbers;
+    {
+      int workingNormalizedNumbers = 0;
+      int workingNonNormalizedNumbers = 0;
+
+      for (final Account account : chunkAccounts) {
+        if (hasNumberNormalized(account)) {
+          workingNormalizedNumbers++;
+        } else {
+          workingNonNormalizedNumbers++;
+        }
+      }
+
+      normalizedNumbers = workingNormalizedNumbers;
+      nonNormalizedNumbers = workingNonNormalizedNumbers;
+    }
+
+    metricsCluster.useCluster(connection -> {
+      connection.sync().incrby(NORMALIZED_NUMBER_COUNT_KEY, normalizedNumbers);
+      connection.sync().incrby(NON_NORMALIZED_NUMBER_COUNT_KEY, nonNormalizedNumbers);
+    });
+  }
+
+  @Override
+  public void onCrawlEnd(final Optional<UUID> fromUuid) {
+    final int normalizedNumbers = metricsCluster.withCluster(connection ->
+        Integer.parseInt(connection.sync().get(NORMALIZED_NUMBER_COUNT_KEY)));
+
+    final int nonNormalizedNumbers = metricsCluster.withCluster(connection ->
+        Integer.parseInt(connection.sync().get(NON_NORMALIZED_NUMBER_COUNT_KEY)));
+
+    log.info("Crawl completed. Normalized numbers: {}; non-normalized numbers: {}",
+        normalizedNumbers, nonNormalizedNumbers);
+  }
+
+  @VisibleForTesting
+  static boolean hasNumberNormalized(final Account account) {
+    try {
+      final PhoneNumber phoneNumber = PHONE_NUMBER_UTIL.parse(account.getNumber(), null);
+      return account.getNumber().equals(PHONE_NUMBER_UTIL.format(phoneNumber, PhoneNumberFormat.E164));
+    } catch (final NumberParseException e) {
+      log.warn("Failed to parse phone number for account {}", account.getUuid(), e);
+      return false;
+    }
+  }
+}
diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/NonNormalizedAccountCrawlerListenerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/NonNormalizedAccountCrawlerListenerTest.java
new file mode 100644
index 000000000..2f2770026
--- /dev/null
+++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/NonNormalizedAccountCrawlerListenerTest.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2013-2021 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.storage;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.UUID;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class NonNormalizedAccountCrawlerListenerTest {
+
+  @ParameterizedTest
+  @MethodSource
+  void hasNumberNormalized(final String number, final boolean expectNormalized) {
+    final Account account = mock(Account.class);
+    when(account.getUuid()).thenReturn(UUID.randomUUID());
+    when(account.getNumber()).thenReturn(number);
+
+    assertEquals(expectNormalized, NonNormalizedAccountCrawlerListener.hasNumberNormalized(account));
+  }
+
+  private static Stream<Arguments> hasNumberNormalized() {
+    return Stream.of(
+        Arguments.of("+447700900111", true),
+        Arguments.of("+4407700900111", false),
+        Arguments.of("Not a real phone number", false),
+        Arguments.of(null, false)
+    );
+  }
+}