Update KT search requests to include a value and maybe an unidentified access key

This commit is contained in:
Katherine 2024-10-23 10:21:38 -04:00 committed by GitHub
parent 3fdb691702
commit 013e45596e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 140 additions and 34 deletions

View File

@ -56,8 +56,10 @@ public class KeyTransparencyController {
private static final Logger LOGGER = LoggerFactory.getLogger(KeyTransparencyController.class);
@VisibleForTesting
static final Duration KEY_TRANSPARENCY_RPC_TIMEOUT = Duration.ofSeconds(15);
private static final byte USERNAME_PREFIX = (byte) 'u';
private static final byte E164_PREFIX = (byte) 'n';
@VisibleForTesting
static final byte USERNAME_PREFIX = (byte) 'u';
@VisibleForTesting
static final byte E164_PREFIX = (byte) 'n';
@VisibleForTesting
static final byte ACI_PREFIX = (byte) 'a';
private final KeyTransparencyServiceClient keyTransparencyServiceClient;
@ -76,7 +78,7 @@ public class KeyTransparencyController {
@ApiResponse(responseCode = "200", description = "All search key lookups were successful", useReturnTypeSchema = true)
@ApiResponse(responseCode = "403", description = "At least one search key lookup to value mapping was invalid")
@ApiResponse(responseCode = "404", description = "At least one search key lookup did not find the key")
@ApiResponse(responseCode = "413", description = "Ratelimited")
@ApiResponse(responseCode = "429", description = "Ratelimited")
@ApiResponse(responseCode = "422", description = "Invalid request format")
@POST
@Path("/search")
@ -92,6 +94,8 @@ public class KeyTransparencyController {
try {
final CompletableFuture<byte[]> aciSearchKeyResponseFuture = keyTransparencyServiceClient.search(
getFullSearchKeyByteString(ACI_PREFIX, request.aci().toCompactByteArray()),
ByteString.copyFrom(request.aciIdentityKey().serialize()),
Optional.empty(),
request.lastTreeHeadSize(),
request.distinguishedTreeHeadSize(),
KEY_TRANSPARENCY_RPC_TIMEOUT);
@ -99,6 +103,8 @@ public class KeyTransparencyController {
final CompletableFuture<byte[]> e164SearchKeyResponseFuture = request.e164()
.map(e164 -> keyTransparencyServiceClient.search(
getFullSearchKeyByteString(E164_PREFIX, e164.getBytes(StandardCharsets.UTF_8)),
ByteString.copyFrom(request.aci().toCompactByteArray()),
Optional.of(ByteString.copyFrom(request.unidentifiedAccessKey().get())),
request.lastTreeHeadSize(),
request.distinguishedTreeHeadSize(),
KEY_TRANSPARENCY_RPC_TIMEOUT))
@ -107,6 +113,8 @@ public class KeyTransparencyController {
final CompletableFuture<byte[]> usernameHashSearchKeyResponseFuture = request.usernameHash()
.map(usernameHash -> keyTransparencyServiceClient.search(
getFullSearchKeyByteString(USERNAME_PREFIX, request.usernameHash().get()),
ByteString.copyFrom(request.aci().toCompactByteArray()),
Optional.empty(),
request.lastTreeHeadSize(),
request.distinguishedTreeHeadSize(),
KEY_TRANSPARENCY_RPC_TIMEOUT))
@ -138,7 +146,7 @@ public class KeyTransparencyController {
)
@ApiResponse(responseCode = "200", description = "All search keys exist in the log", useReturnTypeSchema = true)
@ApiResponse(responseCode = "404", description = "At least one search key lookup did not find the key")
@ApiResponse(responseCode = "413", description = "Ratelimited")
@ApiResponse(responseCode = "429", description = "Ratelimited")
@ApiResponse(responseCode = "422", description = "Invalid request format")
@POST
@Path("/monitor")

View File

@ -8,11 +8,15 @@ package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import io.swagger.v3.oas.annotations.media.Schema;
import org.signal.libsignal.protocol.IdentityKey;
import org.whispersystems.textsecuregcm.identity.AciServiceIdentifier;
import org.whispersystems.textsecuregcm.util.ByteArrayBase64UrlAdapter;
import org.whispersystems.textsecuregcm.util.ByteArrayBase64WithPaddingAdapter;
import org.whispersystems.textsecuregcm.util.E164;
import org.whispersystems.textsecuregcm.util.IdentityKeyAdapter;
import org.whispersystems.textsecuregcm.util.ServiceIdentifierAdapter;
import javax.validation.constraints.AssertTrue;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Positive;
import java.util.Optional;
@ -33,9 +37,25 @@ public record KeyTransparencySearchRequest(
@Schema(description = "The username hash to look up, encoded in web-safe unpadded base64.")
Optional<byte[]> usernameHash,
@NotNull
@JsonSerialize(using = IdentityKeyAdapter.Serializer.class)
@JsonDeserialize(using = IdentityKeyAdapter.Deserializer.class)
@Schema(description="The public aci identity key associated with the provided aci")
IdentityKey aciIdentityKey,
@JsonSerialize(contentUsing = ByteArrayBase64WithPaddingAdapter.Serializing.class)
@JsonDeserialize(contentUsing = ByteArrayBase64WithPaddingAdapter.Deserializing.class)
@Schema(description="The unidentified access key associated with the account")
Optional<byte[]> unidentifiedAccessKey,
@Schema(description = "The non-distinguished tree head size to prove consistency against.")
Optional<@Positive Long> lastTreeHeadSize,
@Schema(description = "The distinguished tree head size to prove consistency against.")
Optional<@Positive Long> distinguishedTreeHeadSize
) {}
) {
@AssertTrue
public boolean isUnidentifiedAccessKeyProvidedWithE164() {
return unidentifiedAccessKey.isPresent() == e164.isPresent();
}
}

View File

@ -52,13 +52,19 @@ public class KeyTransparencyServiceClient implements Managed {
this.callbackExecutor = callbackExecutor;
}
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
public CompletableFuture<byte[]> search(
final ByteString searchKey,
final ByteString mappedValue,
final Optional<ByteString> unidentifiedAccessKey,
final Optional<Long> lastTreeHeadSize,
final Optional<Long> distinguishedTreeHeadSize,
final Duration timeout) {
final SearchRequest.Builder searchRequestBuilder = SearchRequest.newBuilder()
.setSearchKey(searchKey);
.setSearchKey(searchKey)
.setMappedValue(mappedValue);
unidentifiedAccessKey.ifPresent(searchRequestBuilder::setUnidentifiedAccessKey);
final ConsistencyParameters.Builder consistency = ConsistencyParameters.newBuilder();
lastTreeHeadSize.ifPresent(consistency::setLast);
@ -71,6 +77,7 @@ public class KeyTransparencyServiceClient implements Managed {
.thenApply(AbstractMessageLite::toByteArray);
}
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
public CompletableFuture<byte[]> monitor(final List<MonitorKey> monitorKeys,
final Optional<Long> lastTreeHeadSize,
final Optional<Long> distinguishedTreeHeadSize,

View File

@ -48,8 +48,6 @@ message ConsistencyParameters {
optional uint64 distinguished = 2;
}
// TODO: add a `value` field so that the KT server can verify that the given search key is mapped
// to the provided value.
message SearchRequest {
/**
* The key to look up in the log tree.
@ -67,6 +65,16 @@ message SearchRequest {
* The tree head size(s) to prove consistency against.
*/
ConsistencyParameters consistency = 3;
/**
* Clients need to prove that they know the search key to value mapping
* to avoid the key transparency service leaking user data.
* If the client is looking up the distinguished key, this field may be empty.
*/
optional bytes mapped_value = 4;
/**
* Clients should only provide the unidentified_access_key if the search key is a phone number.
*/
optional bytes unidentified_access_key = 5;
}
message SearchResponse {

View File

@ -8,6 +8,7 @@ package org.whispersystems.textsecuregcm.controllers;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.net.HttpHeaders;
import com.google.i18n.phonenumbers.PhoneNumberUtil;
import com.google.protobuf.ByteString;
import io.dropwizard.auth.AuthValueFactoryProvider;
import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
import io.dropwizard.testing.junit5.ResourceExtension;
@ -21,6 +22,9 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.ArgumentCaptor;
import org.signal.libsignal.protocol.IdentityKey;
import org.signal.libsignal.protocol.ecc.Curve;
import org.whispersystems.textsecuregcm.auth.AuthenticatedDevice;
import org.whispersystems.textsecuregcm.entities.KeyTransparencyMonitorRequest;
import org.whispersystems.textsecuregcm.entities.KeyTransparencyMonitorResponse;
@ -41,10 +45,13 @@ import javax.ws.rs.client.Entity;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.core.Response;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
@ -62,6 +69,10 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.whispersystems.textsecuregcm.controllers.KeyTransparencyController.ACI_PREFIX;
import static org.whispersystems.textsecuregcm.controllers.KeyTransparencyController.E164_PREFIX;
import static org.whispersystems.textsecuregcm.controllers.KeyTransparencyController.USERNAME_PREFIX;
import static org.whispersystems.textsecuregcm.controllers.KeyTransparencyController.getFullSearchKeyByteString;
@ExtendWith(DropwizardExtensionsSupport.class)
public class KeyTransparencyControllerTest {
@ -70,8 +81,11 @@ public class KeyTransparencyControllerTest {
PhoneNumberUtil.getInstance().getExampleNumber("US"),
PhoneNumberUtil.PhoneNumberFormat.E164);
private static final AciServiceIdentifier ACI = new AciServiceIdentifier(UUID.randomUUID());
private static final byte[] USERNAME_HASH = TestRandomUtil.nextBytes(20);
private static final TestRemoteAddressFilterProvider TEST_REMOTE_ADDRESS_FILTER_PROVIDER
= new TestRemoteAddressFilterProvider("127.0.0.1");
private static final IdentityKey ACI_IDENTITY_KEY = new IdentityKey(Curve.generateKeyPair().getPublicKey());
private static final byte[] UNIDENTIFIED_ACCESS_KEY = new byte[16];
private final KeyTransparencyServiceClient keyTransparencyServiceClient = mock(KeyTransparencyServiceClient.class);
private static final RateLimiters rateLimiters = mock(RateLimiters.class);
private static final RateLimiter searchRatelimiter = mock(RateLimiter.class);
@ -103,29 +117,34 @@ public class KeyTransparencyControllerTest {
@Test
void getFullSearchKey() {
final byte[] charBytes = new byte[]{KeyTransparencyController.ACI_PREFIX};
final byte[] charBytes = new byte[]{ACI_PREFIX};
final byte[] aci = ACI.toCompactByteArray();
final byte[] expectedFullSearchKey = new byte[aci.length + 1];
System.arraycopy(charBytes, 0, expectedFullSearchKey, 0, charBytes.length);
System.arraycopy(aci, 0, expectedFullSearchKey, charBytes.length, aci.length);
assertArrayEquals(expectedFullSearchKey,
KeyTransparencyController.getFullSearchKeyByteString(KeyTransparencyController.ACI_PREFIX, aci).toByteArray());
assertArrayEquals(expectedFullSearchKey, getFullSearchKeyByteString(KeyTransparencyController.ACI_PREFIX, aci).toByteArray());
}
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@ParameterizedTest
@MethodSource
void searchSuccess(final Optional<String> e164, final Optional<byte[]> usernameHash, final int expectedNumClientCalls) {
when(keyTransparencyServiceClient.search(any(), any(), any(), any()))
void searchSuccess(final Optional<String> e164, final Optional<byte[]> usernameHash, final int expectedNumClientCalls,
final Set<ByteString> expectedSearchKeys,
final Set<ByteString> expectedValues,
final List<Optional<ByteString>> expectedUnidentifiedAccessKey) {
when(keyTransparencyServiceClient.search(any(), any(), any(), any(), any(), any()))
.thenReturn(CompletableFuture.completedFuture(TestRandomUtil.nextBytes(16)));
final Invocation.Builder request = resources.getJerseyTest()
.target("/v1/key-transparency/search")
.request();
final String searchJson = createSearchRequestJson(ACI, e164, usernameHash, Optional.of(3L), Optional.of(4L));
final Optional<byte[]> unidentifiedAccessKey = e164.isPresent() ? Optional.of(UNIDENTIFIED_ACCESS_KEY) : Optional.empty();
final String searchJson = createSearchRequestJson(ACI, e164, usernameHash, ACI_IDENTITY_KEY,
unidentifiedAccessKey, Optional.of(3L), Optional.of(4L));
try (Response response = request.post(Entity.json(searchJson))) {
assertEquals(200, response.getStatus());
@ -140,16 +159,45 @@ public class KeyTransparencyControllerTest {
e164.ifPresentOrElse(ignored -> assertTrue(keyTransparencySearchResponse.e164SearchResponse().isPresent()),
() -> assertTrue(keyTransparencySearchResponse.e164SearchResponse().isEmpty()));
verify(keyTransparencyServiceClient, times(expectedNumClientCalls)).search(any(), eq(Optional.of(3L)), eq(Optional.of(4L)),
ArgumentCaptor<ByteString> valueArguments = ArgumentCaptor.forClass(ByteString.class);
ArgumentCaptor<ByteString> searchKeyArguments = ArgumentCaptor.forClass(ByteString.class);
ArgumentCaptor<Optional<ByteString>> unidentifiedAccessKeyArgument = ArgumentCaptor.forClass(Optional.class);
verify(keyTransparencyServiceClient, times(expectedNumClientCalls)).search(searchKeyArguments.capture(), valueArguments.capture(), unidentifiedAccessKeyArgument.capture(), eq(Optional.of(3L)), eq(Optional.of(4L)),
eq(KeyTransparencyController.KEY_TRANSPARENCY_RPC_TIMEOUT));
assertEquals(expectedSearchKeys, new HashSet<>(searchKeyArguments.getAllValues()));
assertEquals(expectedValues, new HashSet<>(valueArguments.getAllValues()));
assertEquals(expectedUnidentifiedAccessKey, unidentifiedAccessKeyArgument.getAllValues());
}
}
private static Stream<Arguments> searchSuccess() {
final byte[] aciBytes = ACI.toCompactByteArray();
final ByteString aciValueByteString = ByteString.copyFrom(aciBytes);
final byte[] aciIdentityKeyBytes = ACI_IDENTITY_KEY.serialize();
final ByteString aciIdentityKeyValueByteString = ByteString.copyFrom(aciIdentityKeyBytes);
return Stream.of(
Arguments.of(Optional.empty(), Optional.empty(), 1),
Arguments.of(Optional.empty(), Optional.of(TestRandomUtil.nextBytes(20)), 2),
Arguments.of(Optional.of(NUMBER), Optional.empty(), 2)
// Only looking up ACI; ACI identity key should be the only value provided; no UAK
Arguments.of(Optional.empty(), Optional.empty(), 1,
Set.of(getFullSearchKeyByteString(ACI_PREFIX, aciBytes)),
Set.of(aciIdentityKeyValueByteString),
List.of(Optional.empty())),
// Looking up ACI and username hash; ACI identity key and ACI should be the values provided; no UAK
Arguments.of(Optional.empty(), Optional.of(USERNAME_HASH), 2,
Set.of(getFullSearchKeyByteString(ACI_PREFIX, aciBytes),
getFullSearchKeyByteString(USERNAME_PREFIX, USERNAME_HASH)),
Set.of(aciIdentityKeyValueByteString, aciValueByteString),
List.of(Optional.empty(), Optional.empty())),
// Looking up ACI and phone number; ACI identity key and ACI should be the values provided; must provide UAK
Arguments.of(Optional.of(NUMBER), Optional.empty(), 2,
Set.of(getFullSearchKeyByteString(ACI_PREFIX, aciBytes),
getFullSearchKeyByteString(E164_PREFIX, NUMBER.getBytes(StandardCharsets.UTF_8))),
Set.of(aciValueByteString, aciIdentityKeyValueByteString),
List.of(Optional.empty(), Optional.of(ByteString.copyFrom(UNIDENTIFIED_ACCESS_KEY))))
);
}
@ -159,24 +207,26 @@ public class KeyTransparencyControllerTest {
.target("/v1/key-transparency/search")
.request()
.header(HttpHeaders.AUTHORIZATION, AuthHelper.getAuthHeader(AuthHelper.VALID_UUID, AuthHelper.VALID_PASSWORD));
try (Response response = request.post(Entity.json(createSearchRequestJson(ACI, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())))) {
try (Response response = request.post(Entity.json(createSearchRequestJson(ACI, Optional.empty(), Optional.empty(),
ACI_IDENTITY_KEY, Optional.empty(), Optional.empty(), Optional.empty())))) {
assertEquals(400, response.getStatus());
}
verify(keyTransparencyServiceClient, never()).search(any(), any(), any(), any());
verify(keyTransparencyServiceClient, never()).search(any(), any(), any(), any(), any(), any());
}
@ParameterizedTest
@MethodSource
void searchGrpcErrors(final Status grpcStatus, final int httpStatus) {
when(keyTransparencyServiceClient.search(any(), any(), any(), any()))
when(keyTransparencyServiceClient.search(any(), any(), any(), any(), any(), any()))
.thenReturn(CompletableFuture.failedFuture(new CompletionException(new StatusRuntimeException(grpcStatus))));
final Invocation.Builder request = resources.getJerseyTest()
.target("/v1/key-transparency/search")
.request();
try (Response response = request.post(Entity.json(createSearchRequestJson(ACI, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())))) {
try (Response response = request.post(Entity.json(createSearchRequestJson(ACI, Optional.empty(), Optional.empty(),
ACI_IDENTITY_KEY, Optional.empty(),Optional.empty(), Optional.empty())))) {
assertEquals(httpStatus, response.getStatus());
verify(keyTransparencyServiceClient, times(1)).search(any(), any(), any(), any());
verify(keyTransparencyServiceClient, times(1)).search(any(), any(), any(), any(), any(), any());
}
}
@ -193,27 +243,37 @@ public class KeyTransparencyControllerTest {
@ParameterizedTest
@MethodSource
void searchInvalidRequest(final AciServiceIdentifier aci,
final IdentityKey aciIdentityKey,
final Optional<String> e164,
final Optional<byte[]> unidentifiedAccessKey,
final Optional<Long> lastTreeHeadSize,
final Optional<Long> distinguishedTreeHeadSize) {
final Invocation.Builder request = resources.getJerseyTest()
.target("/v1/key-transparency/search")
.request();
try (Response response = request.post(Entity.json(
createSearchRequestJson(aci, Optional.empty(), Optional.empty(), lastTreeHeadSize, distinguishedTreeHeadSize)))) {
createSearchRequestJson(aci, e164, Optional.empty(),
aciIdentityKey, unidentifiedAccessKey, lastTreeHeadSize, distinguishedTreeHeadSize)))) {
assertEquals(422, response.getStatus());
verify(keyTransparencyServiceClient, never()).search(any(), any(), any(), any());
verify(keyTransparencyServiceClient, never()).search(any(), any(), any(), any(), any(), any());
}
}
private static Stream<Arguments> searchInvalidRequest() {
return Stream.of(
// ACI can't be null
Arguments.of(null, Optional.empty(), Optional.empty()),
Arguments.of(null, ACI_IDENTITY_KEY, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()),
// ACI identity key can't be null
Arguments.of(ACI, null, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()),
// lastNonDistinguishedTreeHeadSize must be positive
Arguments.of(ACI, Optional.of(0L), Optional.empty()),
Arguments.of(ACI, ACI_IDENTITY_KEY, Optional.empty(), Optional.empty(), Optional.of(0L), Optional.empty()),
// lastDistinguishedTreeHeadSize must be positive
Arguments.of(ACI, Optional.empty(), Optional.of(0L))
);
Arguments.of(ACI, ACI_IDENTITY_KEY, Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(0L)),
// E164 can't be provided without an unidentified access key
Arguments.of(ACI, ACI_IDENTITY_KEY, Optional.of(NUMBER), Optional.empty(), Optional.empty(), Optional.empty()),
// ...and an unidentified access key can't be provided without an E164
Arguments.of(ACI, ACI_IDENTITY_KEY, Optional.empty(), Optional.of(UNIDENTIFIED_ACCESS_KEY), Optional.empty(), Optional.empty())
);
}
@Test
@ -223,9 +283,10 @@ public class KeyTransparencyControllerTest {
final Invocation.Builder request = resources.getJerseyTest()
.target("/v1/key-transparency/search")
.request();
try (Response response = request.post(Entity.json(createSearchRequestJson(ACI, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())))) {
try (Response response = request.post(Entity.json(createSearchRequestJson(ACI, Optional.empty(), Optional.empty(),
ACI_IDENTITY_KEY, Optional.empty(),Optional.empty(), Optional.empty())))) {
assertEquals(429, response.getStatus());
verify(keyTransparencyServiceClient, never()).search(any(), any(), any(), any());
verify(keyTransparencyServiceClient, never()).search(any(), any(), any(), any(), any(), any());
}
}
@ -319,10 +380,10 @@ public class KeyTransparencyControllerTest {
Arguments.of(createMonitorRequestJson(ACI, List.of(4L), Optional.empty(), Optional.of(List.of(5L)),
Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())),
// usernameHashPosition cannot be empty if usernameHash isn't
Arguments.of(createMonitorRequestJson(ACI, List.of(4L), Optional.of(TestRandomUtil.nextBytes(20)),
Arguments.of(createMonitorRequestJson(ACI, List.of(4L), Optional.of(USERNAME_HASH),
Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())),
// usernameHashPositions list cannot be empty
Arguments.of(createMonitorRequestJson(ACI, List.of(4L), Optional.of(TestRandomUtil.nextBytes(20)),
Arguments.of(createMonitorRequestJson(ACI, List.of(4L), Optional.of(USERNAME_HASH),
Optional.of(Collections.emptyList()), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())),
// e164 cannot be empty if e164Positions isn't
Arguments.of(
@ -382,9 +443,11 @@ public class KeyTransparencyControllerTest {
final AciServiceIdentifier aci,
final Optional<String> e164,
final Optional<byte[]> usernameHash,
final IdentityKey aciIdentityKey,
final Optional<byte[]> unidentifiedAccessKey,
final Optional<Long> lastTreeHeadSize,
final Optional<Long> distinguishedTreeHeadSize) {
final KeyTransparencySearchRequest request = new KeyTransparencySearchRequest(aci, e164, usernameHash, lastTreeHeadSize, distinguishedTreeHeadSize);
final KeyTransparencySearchRequest request = new KeyTransparencySearchRequest(aci, e164, usernameHash, aciIdentityKey, unidentifiedAccessKey, lastTreeHeadSize, distinguishedTreeHeadSize);
try {
return SystemMapper.jsonMapper().writeValueAsString(request);
} catch (final JsonProcessingException e) {