diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/experiment/Experiment.java b/service/src/main/java/org/whispersystems/textsecuregcm/experiment/Experiment.java new file mode 100644 index 000000000..9a9306fc3 --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/experiment/Experiment.java @@ -0,0 +1,59 @@ +package org.whispersystems.textsecuregcm.experiment; + +import com.codahale.metrics.MetricRegistry; +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tag; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletionStage; + +/** + * An experiment compares the results of two operations and records metrics to assess how frequently they match. + */ +public class Experiment { + + private final String counterName; + private final MeterRegistry meterRegistry; + + static final String OUTCOME_TAG = "outcome"; + static final String CAUSE_TAG = "cause"; + + static final String MATCH_OUTCOME = "match"; + static final String MISMATCH_OUTCOME = "mismatch"; + static final String ERROR_OUTCOME = "error"; + + public Experiment(final String... names) { + this(Metrics.globalRegistry, names); + } + + @VisibleForTesting + Experiment(final MeterRegistry meterRegistry, final String... names) { + if (names == null || names.length == 0) { + throw new IllegalArgumentException("Experiments must have a name"); + } + + this.counterName = MetricRegistry.name(getClass(), names); + this.meterRegistry = meterRegistry; + } + + public void compareResult(final T expected, final CompletionStage experimentStage) { + experimentStage.whenComplete((actual, cause) -> { + if (cause != null) { + meterRegistry.counter(counterName, + List.of(Tag.of(OUTCOME_TAG, ERROR_OUTCOME), Tag.of(CAUSE_TAG, cause.getClass().getSimpleName()))) + .increment(); + } else { + final boolean shouldIgnore = actual == null && expected != null; + + if (!shouldIgnore) { + meterRegistry.counter(counterName, + List.of(Tag.of(OUTCOME_TAG, Objects.equals(expected, actual) ? MATCH_OUTCOME : MISMATCH_OUTCOME))) + .increment(); + } + } + }); + } +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/experiment/ExperimentTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/experiment/ExperimentTest.java new file mode 100644 index 000000000..f22a4e2c6 --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/experiment/ExperimentTest.java @@ -0,0 +1,105 @@ +package org.whispersystems.textsecuregcm.experiment; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.anyVararg; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ExperimentTest { + + private MeterRegistry meterRegistry; + + @Before + public void setUp() { + meterRegistry = mock(MeterRegistry.class); + } + + @Test + public void compareResultMatch() { + final Counter counter = mock(Counter.class); + when(meterRegistry.counter(anyString(), ArgumentMatchers.>any())).thenReturn(counter); + + new Experiment(meterRegistry, "test").compareResult(12, CompletableFuture.completedFuture(12)); + + @SuppressWarnings("unchecked") final ArgumentCaptor> tagCaptor = ArgumentCaptor.forClass(Iterable.class); + + verify(meterRegistry).counter(anyString(), tagCaptor.capture()); + + final Set tags = getTagSet(tagCaptor.getValue()); + assertEquals(tags, Set.of(Tag.of(Experiment.OUTCOME_TAG, Experiment.MATCH_OUTCOME))); + + verify(counter).increment(); + } + + @Test + public void compareResultMismatch() { + final Counter counter = mock(Counter.class); + when(meterRegistry.counter(anyString(), ArgumentMatchers.>any())).thenReturn(counter); + + new Experiment(meterRegistry, "test").compareResult(12, CompletableFuture.completedFuture(77)); + + @SuppressWarnings("unchecked") final ArgumentCaptor> tagCaptor = ArgumentCaptor.forClass(Iterable.class); + + verify(meterRegistry).counter(anyString(), tagCaptor.capture()); + + final Set tags = getTagSet(tagCaptor.getValue()); + assertEquals(tags, Set.of(Tag.of(Experiment.OUTCOME_TAG, Experiment.MISMATCH_OUTCOME))); + + verify(counter).increment(); + } + + @Test + public void compareResultError() { + final Counter counter = mock(Counter.class); + when(meterRegistry.counter(anyString(), ArgumentMatchers.>any())).thenReturn(counter); + + new Experiment(meterRegistry, "test").compareResult(12, CompletableFuture.failedFuture(new RuntimeException("OH NO"))); + + @SuppressWarnings("unchecked") final ArgumentCaptor> tagCaptor = ArgumentCaptor.forClass(Iterable.class); + + verify(meterRegistry).counter(anyString(), tagCaptor.capture()); + + final Set tags = getTagSet(tagCaptor.getValue()); + assertEquals(tags, Set.of(Tag.of(Experiment.OUTCOME_TAG, Experiment.ERROR_OUTCOME), Tag.of(Experiment.CAUSE_TAG, "RuntimeException"))); + + verify(counter).increment(); + } + + @Test + public void compareResultNoExperimentData() { + final Counter counter = mock(Counter.class); + when(meterRegistry.counter(anyString(), ArgumentMatchers.>any())).thenReturn(counter); + + new Experiment(meterRegistry, "test").compareResult(12, CompletableFuture.completedFuture(null)); + + verify(counter, never()).increment(); + } + + private static Set getTagSet(final Iterable tags) { + final Set tagSet = new HashSet<>(); + + for (final Tag tag : tags) { + tagSet.add(tag); + } + + return tagSet; + } +}