MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce the Experiment class to compare results from parallel systems.

This commit is contained in:
Jon Chambers 2020-06-07 13:53:04 -04:00 committed by Jon Chambers
parent 1db5977e80
commit c2a4a2778e
2 changed files with 164 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
service/src/main/java/org/whispersystems/textsecuregcm/experiment/Experiment.java Normal file
View File

@ -0,0 +1,59 @@
package org.whispersystems.textsecuregcm.experiment;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletionStage;
/**
* An experiment compares the results of two operations and records metrics to assess how frequently they match.
*/
public class Experiment {
private final String counterName;
private final MeterRegistry meterRegistry;
static final String OUTCOME_TAG = "outcome";
static final String CAUSE_TAG = "cause";
static final String MATCH_OUTCOME = "match";
static final String MISMATCH_OUTCOME = "mismatch";
static final String ERROR_OUTCOME = "error";
public Experiment(final String... names) {
this(Metrics.globalRegistry, names);
}
@VisibleForTesting
Experiment(final MeterRegistry meterRegistry, final String... names) {
if (names == null || names.length == 0) {
throw new IllegalArgumentException("Experiments must have a name");
}
this.counterName = MetricRegistry.name(getClass(), names);
this.meterRegistry = meterRegistry;
}
public <T> void compareResult(final T expected, final CompletionStage<T> experimentStage) {
experimentStage.whenComplete((actual, cause) -> {
if (cause != null) {
meterRegistry.counter(counterName,
List.of(Tag.of(OUTCOME_TAG, ERROR_OUTCOME), Tag.of(CAUSE_TAG, cause.getClass().getSimpleName())))
.increment();
} else {
final boolean shouldIgnore = actual == null && expected != null;
if (!shouldIgnore) {
meterRegistry.counter(counterName,
List.of(Tag.of(OUTCOME_TAG, Objects.equals(expected, actual) ? MATCH_OUTCOME : MISMATCH_OUTCOME)))
.increment();
}
}
});
}
}

105
service/src/test/java/org/whispersystems/textsecuregcm/experiment/ExperimentTest.java Normal file
View File

@ -0,0 +1,105 @@
package org.whispersystems.textsecuregcm.experiment;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tag;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatchers;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import static org.junit.Assert.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.anyVararg;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class ExperimentTest {
private MeterRegistry meterRegistry;
@Before
public void setUp() {
meterRegistry = mock(MeterRegistry.class);
}
@Test
public void compareResultMatch() {
final Counter counter = mock(Counter.class);
when(meterRegistry.counter(anyString(), ArgumentMatchers.<Iterable<Tag>>any())).thenReturn(counter);
new Experiment(meterRegistry, "test").compareResult(12, CompletableFuture.completedFuture(12));
@SuppressWarnings("unchecked") final ArgumentCaptor<Iterable<Tag>> tagCaptor = ArgumentCaptor.forClass(Iterable.class);
verify(meterRegistry).counter(anyString(), tagCaptor.capture());
final Set<Tag> tags = getTagSet(tagCaptor.getValue());
assertEquals(tags, Set.of(Tag.of(Experiment.OUTCOME_TAG, Experiment.MATCH_OUTCOME)));
verify(counter).increment();
}
@Test
public void compareResultMismatch() {
final Counter counter = mock(Counter.class);
when(meterRegistry.counter(anyString(), ArgumentMatchers.<Iterable<Tag>>any())).thenReturn(counter);
new Experiment(meterRegistry, "test").compareResult(12, CompletableFuture.completedFuture(77));
@SuppressWarnings("unchecked") final ArgumentCaptor<Iterable<Tag>> tagCaptor = ArgumentCaptor.forClass(Iterable.class);
verify(meterRegistry).counter(anyString(), tagCaptor.capture());
final Set<Tag> tags = getTagSet(tagCaptor.getValue());
assertEquals(tags, Set.of(Tag.of(Experiment.OUTCOME_TAG, Experiment.MISMATCH_OUTCOME)));
verify(counter).increment();
}
@Test
public void compareResultError() {
final Counter counter = mock(Counter.class);
when(meterRegistry.counter(anyString(), ArgumentMatchers.<Iterable<Tag>>any())).thenReturn(counter);
new Experiment(meterRegistry, "test").compareResult(12, CompletableFuture.failedFuture(new RuntimeException("OH NO")));
@SuppressWarnings("unchecked") final ArgumentCaptor<Iterable<Tag>> tagCaptor = ArgumentCaptor.forClass(Iterable.class);
verify(meterRegistry).counter(anyString(), tagCaptor.capture());
final Set<Tag> tags = getTagSet(tagCaptor.getValue());
assertEquals(tags, Set.of(Tag.of(Experiment.OUTCOME_TAG, Experiment.ERROR_OUTCOME), Tag.of(Experiment.CAUSE_TAG, "RuntimeException")));
verify(counter).increment();
}
@Test
public void compareResultNoExperimentData() {
final Counter counter = mock(Counter.class);
when(meterRegistry.counter(anyString(), ArgumentMatchers.<Iterable<Tag>>any())).thenReturn(counter);
new Experiment(meterRegistry, "test").compareResult(12, CompletableFuture.completedFuture(null));
verify(counter, never()).increment();
}
private static Set<Tag> getTagSet(final Iterable<Tag> tags) {
final Set<Tag> tagSet = new HashSet<>();
for (final Tag tag : tags) {
tagSet.add(tag);
}
return tagSet;
}
}