MirrorMirror
/
Signal-Server
mirror of https://github.com/signalapp/Signal-Server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support for accessing attachments via CDN

This commit is contained in:
Moxie Marlinspike 2019-04-09 10:22:08 -07:00
parent 07c22ed5bc
commit 305b4148bd
11 changed files with 237 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
View File

@ -30,7 +30,8 @@ import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
import org.whispersystems.textsecuregcm.auth.DirectoryCredentialsGenerator;
import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
import org.whispersystems.textsecuregcm.controllers.AccountController;
import org.whispersystems.textsecuregcm.controllers.AttachmentController;
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV1;
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV2;
import org.whispersystems.textsecuregcm.controllers.CertificateController;
import org.whispersystems.textsecuregcm.controllers.DeviceController;
import org.whispersystems.textsecuregcm.controllers.DirectoryController;
@ -62,7 +63,6 @@ import org.whispersystems.textsecuregcm.push.ReceiptSender;
import org.whispersystems.textsecuregcm.push.WebsocketSender;
import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient;
import org.whispersystems.textsecuregcm.redis.ReplicatedJedisPool;
import org.whispersystems.textsecuregcm.s3.UrlSigner;
import org.whispersystems.textsecuregcm.sms.SmsSender;
import org.whispersystems.textsecuregcm.sms.TwilioSmsSender;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
@ -190,7 +190,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
ApnFallbackManager apnFallbackManager = new ApnFallbackManager(pushSchedulerClient, apnSender, accountsManager);
TwilioSmsSender twilioSmsSender = new TwilioSmsSender(config.getTwilioConfiguration());
SmsSender smsSender = new SmsSender(twilioSmsSender);
UrlSigner urlSigner = new UrlSigner(config.getAttachmentsConfiguration());
PushSender pushSender = new PushSender(apnFallbackManager, gcmSender, apnSender, websocketSender, config.getPushConfiguration().getQueueSize());
ReceiptSender receiptSender = new ReceiptSender(accountsManager, pushSender);
TurnTokenGenerator turnTokenGenerator = new TurnTokenGenerator(config.getTurnConfiguration());
@ -216,10 +215,11 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
environment.lifecycle().manage(messagesCache);
environment.lifecycle().manage(accountDatabaseCrawler);
AttachmentController attachmentController = new AttachmentController(rateLimiters, urlSigner);
KeysController keysController = new KeysController(rateLimiters, keys, accountsManager, directoryQueue);
MessageController messageController = new MessageController(rateLimiters, pushSender, receiptSender, accountsManager, messagesManager, apnFallbackManager);
ProfileController profileController = new ProfileController(rateLimiters , accountsManager, config.getProfilesConfiguration());
AttachmentControllerV1 attachmentControllerV1 = new AttachmentControllerV1(rateLimiters, config.getAttachmentsConfiguration().getAccessKey(), config.getAttachmentsConfiguration().getAccessSecret(), config.getAttachmentsConfiguration().getBucket() );
AttachmentControllerV2 attachmentControllerV2 = new AttachmentControllerV2(rateLimiters, config.getAttachmentsConfiguration().getAccessKey(), config.getAttachmentsConfiguration().getAccessSecret(), config.getAttachmentsConfiguration().getRegion(), config.getAttachmentsConfiguration().getBucket());
KeysController keysController = new KeysController(rateLimiters, keys, accountsManager, directoryQueue);
MessageController messageController = new MessageController(rateLimiters, pushSender, receiptSender, accountsManager, messagesManager, apnFallbackManager);
ProfileController profileController = new ProfileController(rateLimiters, accountsManager, config.getProfilesConfiguration());
environment.jersey().register(new AuthDynamicFeature(new BasicCredentialAuthFilter.Builder<Account>()
.setAuthenticator(deviceAuthenticator)
@ -233,7 +233,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
environment.jersey().register(new CertificateController(new CertificateGenerator(config.getDeliveryCertificate().getCertificate(), config.getDeliveryCertificate().getPrivateKey(), config.getDeliveryCertificate().getExpiresDays())));
environment.jersey().register(new VoiceVerificationController(config.getVoiceVerificationConfiguration().getUrl(), config.getVoiceVerificationConfiguration().getLocales()));
environment.jersey().register(new TransparentDataController(accountsManager, config.getTransparentDataIndex()));
environment.jersey().register(attachmentController);
environment.jersey().register(attachmentControllerV1);
environment.jersey().register(attachmentControllerV2);
environment.jersey().register(keysController);
environment.jersey().register(messageController);
environment.jersey().register(profileController);
@ -245,6 +246,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
webSocketEnvironment.jersey().register(new KeepAliveController(pubSubManager));
webSocketEnvironment.jersey().register(messageController);
webSocketEnvironment.jersey().register(profileController);
webSocketEnvironment.jersey().register(attachmentControllerV1);
webSocketEnvironment.jersey().register(attachmentControllerV2);
WebSocketEnvironment provisioningEnvironment = new WebSocketEnvironment(environment, webSocketEnvironment.getRequestLog(), 60000);
provisioningEnvironment.setConnectListener(new ProvisioningConnectListener(pubSubManager));

8
src/main/java/org/whispersystems/textsecuregcm/configuration/AttachmentsConfiguration.java
View File

@ -33,6 +33,10 @@ public class AttachmentsConfiguration {
@JsonProperty
private String bucket;
@NotEmpty
@JsonProperty
private String region;
public String getAccessKey() {
return accessKey;
}
@ -44,4 +48,8 @@ public class AttachmentsConfiguration {
public String getBucket() {
return bucket;
}
public String getRegion() {
return region;
}
}

17
src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerBase.java Normal file
View File

@ -0,0 +1,17 @@
package org.whispersystems.textsecuregcm.controllers;
import org.whispersystems.textsecuregcm.util.Conversions;
import java.security.SecureRandom;
public class AttachmentControllerBase {
protected long generateAttachmentId() {
byte[] attachmentBytes = new byte[8];
new SecureRandom().nextBytes(attachmentBytes);
attachmentBytes[0] = (byte)(attachmentBytes[0] & 0x7F);
return Conversions.byteArrayToLong(attachmentBytes);
}
}

22
src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentController.java → src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV1.java
View File

@ -20,7 +20,7 @@ import com.amazonaws.HttpMethod;
import com.codahale.metrics.annotation.Timed;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptor;
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV1;
import org.whispersystems.textsecuregcm.entities.AttachmentUri;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.s3.UrlSigner;
@ -41,26 +41,25 @@ import io.dropwizard.auth.Auth;
@Path("/v1/attachments")
public class AttachmentController {
public class AttachmentControllerV1 extends AttachmentControllerBase {
@SuppressWarnings("unused")
private final Logger logger = LoggerFactory.getLogger(AttachmentController.class);
private final Logger logger = LoggerFactory.getLogger(AttachmentControllerV1.class);
private static final String[] UNACCELERATED_REGIONS = {"+20", "+971", "+968", "+974"};
private final RateLimiters rateLimiters;
private final UrlSigner urlSigner;
public AttachmentController(RateLimiters rateLimiters, UrlSigner urlSigner)
{
public AttachmentControllerV1(RateLimiters rateLimiters, String accessKey, String accessSecret, String bucket) {
this.rateLimiters = rateLimiters;
this.urlSigner = urlSigner;
this.urlSigner = new UrlSigner(accessKey, accessSecret, bucket);
}
@Timed
@GET
@Produces(MediaType.APPLICATION_JSON)
public AttachmentDescriptor allocateAttachment(@Auth Account account)
public AttachmentDescriptorV1 allocateAttachment(@Auth Account account)
throws RateLimitExceededException
{
if (account.isRateLimited()) {
@ -70,7 +69,7 @@ public class AttachmentController {
long attachmentId = generateAttachmentId();
URL url = urlSigner.getPreSignedUrl(attachmentId, HttpMethod.PUT, Stream.of(UNACCELERATED_REGIONS).anyMatch(region -> account.getNumber().startsWith(region)));
return new AttachmentDescriptor(attachmentId, url.toExternalForm());
return new AttachmentDescriptorV1(attachmentId, url.toExternalForm());
}
@ -85,11 +84,4 @@ public class AttachmentController {
return new AttachmentUri(urlSigner.getPreSignedUrl(attachmentId, HttpMethod.GET, Stream.of(UNACCELERATED_REGIONS).anyMatch(region -> account.getNumber().startsWith(region))));
}
private long generateAttachmentId() {
byte[] attachmentBytes = new byte[8];
new SecureRandom().nextBytes(attachmentBytes);
attachmentBytes[0] = (byte)(attachmentBytes[0] & 0x7F);
return Conversions.byteArrayToLong(attachmentBytes);
}
}

54
src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV2.java Normal file
View File

@ -0,0 +1,54 @@
package org.whispersystems.textsecuregcm.controllers;
import com.codahale.metrics.annotation.Timed;
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV2;
import org.whispersystems.textsecuregcm.limits.RateLimiter;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.s3.PolicySigner;
import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.util.Pair;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import io.dropwizard.auth.Auth;
@Path("/v2/attachments")
public class AttachmentControllerV2 extends AttachmentControllerBase {
private final PostPolicyGenerator policyGenerator;
private final PolicySigner policySigner;
private final RateLimiter rateLimiter;
public AttachmentControllerV2(RateLimiters rateLimiters, String accessKey, String accessSecret, String region, String bucket) {
this.rateLimiter = rateLimiters.getAttachmentLimiter();
this.policyGenerator = new PostPolicyGenerator(region, bucket, accessKey);
this.policySigner = new PolicySigner(accessSecret, region);
}
@Timed
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/form/upload")
public AttachmentDescriptorV2 getAttachmentUploadForm(@Auth Account account) throws RateLimitExceededException {
rateLimiter.validate(account.getNumber());
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
long attachmentId = generateAttachmentId();
String objectName = String.valueOf(attachmentId);
Pair<String, String> policy = policyGenerator.createFor(now, String.valueOf(objectName));
String signature = policySigner.getSignature(now, policy.second());
return new AttachmentDescriptorV2(attachmentId, objectName, policy.first(),
"private", "AWS4-HMAC-SHA256",
now.format(PostPolicyGenerator.AWS_DATE_TIME),
policy.second(), signature);
}
}

6
src/main/java/org/whispersystems/textsecuregcm/entities/AttachmentDescriptor.java → src/main/java/org/whispersystems/textsecuregcm/entities/AttachmentDescriptorV1.java
View File

@ -18,7 +18,7 @@ package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
public class AttachmentDescriptor {
public class AttachmentDescriptorV1 {
@JsonProperty
private long id;
@ -29,13 +29,13 @@ public class AttachmentDescriptor {
@JsonProperty
private String location;
public AttachmentDescriptor(long id, String location) {
public AttachmentDescriptorV1(long id, String location) {
this.id = id;
this.idString = String.valueOf(id);
this.location = location;
}
public AttachmentDescriptor() {}
public AttachmentDescriptorV1() {}
public long getId() {
return id;

88
src/main/java/org/whispersystems/textsecuregcm/entities/AttachmentDescriptorV2.java Normal file
View File

@ -0,0 +1,88 @@
package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
public class AttachmentDescriptorV2 {
@JsonProperty
private String key;
@JsonProperty
private String credential;
@JsonProperty
private String acl;
@JsonProperty
private String algorithm;
@JsonProperty
private String date;
@JsonProperty
private String policy;
@JsonProperty
private String signature;
@JsonProperty
private long attachmentId;
@JsonProperty
private String attachmentIdString;
public AttachmentDescriptorV2() {}
public AttachmentDescriptorV2(long attachmentId,
String key, String credential,
String acl, String algorithm,
String date, String policy,
String signature)
{
this.attachmentId = attachmentId;
this.attachmentIdString = String.valueOf(attachmentId);
this.key = key;
this.credential = credential;
this.acl = acl;
this.algorithm = algorithm;
this.date = date;
this.policy = policy;
this.signature = signature;
}
public String getKey() {
return key;
}
public String getCredential() {
return credential;
}
public String getAcl() {
return acl;
}
public String getAlgorithm() {
return algorithm;
}
public String getDate() {
return date;
}
public String getPolicy() {
return policy;
}
public String getSignature() {
return signature;
}
public long getAttachmentId() {
return attachmentId;
}
public String getAttachmentIdString() {
return attachmentIdString;
}
}

1
src/main/java/org/whispersystems/textsecuregcm/s3/PostPolicyGenerator.java
View File

@ -35,6 +35,7 @@ public class PostPolicyGenerator {
" {\"key\": \"%s\"},\n" +
" {\"acl\": \"private\"},\n" +
" [\"starts-with\", \"$Content-Type\", \"\"],\n" +
" [\"content-length-range\", 1, 104857600],\n" +
"\n" +
" {\"x-amz-credential\": \"%s\"},\n" +
" {\"x-amz-algorithm\": \"AWS4-HMAC-SHA256\"},\n" +

9
src/main/java/org/whispersystems/textsecuregcm/s3/UrlSigner.java
View File

@ -1,4 +1,4 @@
/**
/*
* Copyright (C) 2013 Open WhisperSystems
*
* This program is free software: you can redistribute it and/or modify
@ -23,7 +23,6 @@ import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration;
import java.net.URL;
import java.util.Date;
@ -35,9 +34,9 @@ public class UrlSigner {
private final AWSCredentials credentials;
private final String bucket;
public UrlSigner(AttachmentsConfiguration config) {
this.credentials = new BasicAWSCredentials(config.getAccessKey(), config.getAccessSecret());
this.bucket = config.getBucket();
public UrlSigner(String accessKey, String accessSecret, String bucket) {
this.credentials = new BasicAWSCredentials(accessKey, accessSecret);
this.bucket = bucket;
}
public URL getPreSignedUrl(long attachmentId, HttpMethod method, boolean unaccelerated) {

65
src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AttachmentControllerTest.java
View File

@ -3,13 +3,13 @@ package org.whispersystems.textsecuregcm.tests.controllers;
import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
import org.junit.ClassRule;
import org.junit.Test;
import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration;
import org.whispersystems.textsecuregcm.controllers.AttachmentController;
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptor;
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV1;
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV2;
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV1;
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV2;
import org.whispersystems.textsecuregcm.entities.AttachmentUri;
import org.whispersystems.textsecuregcm.limits.RateLimiter;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.s3.UrlSigner;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
import org.whispersystems.textsecuregcm.util.SystemMapper;
@ -24,19 +24,11 @@ import static org.mockito.Mockito.when;
public class AttachmentControllerTest {
private static AttachmentsConfiguration configuration = mock(AttachmentsConfiguration.class);
private static RateLimiters rateLimiters = mock(RateLimiters.class );
private static RateLimiter rateLimiter = mock(RateLimiter.class );
private static UrlSigner urlSigner;
static {
when(configuration.getAccessKey()).thenReturn("accessKey");
when(configuration.getAccessSecret()).thenReturn("accessSecret");
when(configuration.getBucket()).thenReturn("attachment-bucket");
when(rateLimiters.getAttachmentLimiter()).thenReturn(rateLimiter);
urlSigner = new UrlSigner(configuration);
}
@ClassRule
@ -45,16 +37,43 @@ public class AttachmentControllerTest {
.addProvider(new AuthValueFactoryProvider.Binder<>(Account.class))
.setMapper(SystemMapper.getMapper())
.setTestContainerFactory(new GrizzlyWebTestContainerFactory())
.addResource(new AttachmentController(rateLimiters, urlSigner))
.addResource(new AttachmentControllerV1(rateLimiters, "accessKey", "accessSecret", "attachment-bucket"))
.addResource(new AttachmentControllerV2(rateLimiters, "accessKey", "accessSecret", "us-east-1", "attachmentv2-bucket"))
.build();
@Test
public void testV2Form() {
AttachmentDescriptorV2 descriptor = resources.getJerseyTest()
.target("/v2/attachments/form/upload")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
.get(AttachmentDescriptorV2.class);
assertThat(descriptor.getKey()).isEqualTo(descriptor.getAttachmentIdString());
assertThat(descriptor.getAcl()).isEqualTo("private");
assertThat(descriptor.getAlgorithm()).isEqualTo("AWS4-HMAC-SHA256");
assertThat(descriptor.getAttachmentId()).isGreaterThan(0);
assertThat(String.valueOf(descriptor.getAttachmentId())).isEqualTo(descriptor.getAttachmentIdString());
String[] credentialParts = descriptor.getCredential().split("/");
assertThat(credentialParts[0]).isEqualTo("accessKey");
assertThat(credentialParts[2]).isEqualTo("us-east-1");
assertThat(credentialParts[3]).isEqualTo("s3");
assertThat(credentialParts[4]).isEqualTo("aws4_request");
assertThat(descriptor.getDate()).isNotBlank();
assertThat(descriptor.getPolicy()).isNotBlank();
assertThat(descriptor.getSignature()).isNotBlank();
}
@Test
public void testAcceleratedPut() {
AttachmentDescriptor descriptor = resources.getJerseyTest()
.target("/v1/attachments/")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
.get(AttachmentDescriptor.class);
AttachmentDescriptorV1 descriptor = resources.getJerseyTest()
.target("/v1/attachments/")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
.get(AttachmentDescriptorV1.class);
assertThat(descriptor.getLocation()).startsWith("https://attachment-bucket.s3-accelerate.amazonaws.com");
assertThat(descriptor.getId()).isGreaterThan(0);
@ -63,11 +82,11 @@ public class AttachmentControllerTest {
@Test
public void testUnacceleratedPut() {
AttachmentDescriptor descriptor = resources.getJerseyTest()
.target("/v1/attachments/")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER_TWO, AuthHelper.VALID_PASSWORD_TWO))
.get(AttachmentDescriptor.class);
AttachmentDescriptorV1 descriptor = resources.getJerseyTest()
.target("/v1/attachments/")
.request()
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER_TWO, AuthHelper.VALID_PASSWORD_TWO))
.get(AttachmentDescriptorV1.class);
assertThat(descriptor.getLocation()).startsWith("https://s3.amazonaws.com");
assertThat(descriptor.getId()).isGreaterThan(0);

17
src/test/java/org/whispersystems/textsecuregcm/tests/util/UrlSignerTest.java
View File

@ -2,25 +2,17 @@ package org.whispersystems.textsecuregcm.tests.util;
import com.amazonaws.HttpMethod;
import org.junit.Test;
import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration;
import org.whispersystems.textsecuregcm.s3.UrlSigner;
import java.net.URL;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class UrlSignerTest {
@Test
public void testTransferAcceleration() {
AttachmentsConfiguration configuration = mock(AttachmentsConfiguration.class);
when(configuration.getAccessKey()).thenReturn("foo");
when(configuration.getAccessSecret()).thenReturn("bar");
when(configuration.getBucket()).thenReturn("attachments-test");
UrlSigner signer = new UrlSigner(configuration);
UrlSigner signer = new UrlSigner("foo", "bar", "attachments-test");
URL url = signer.getPreSignedUrl(1234, HttpMethod.GET, false);
assertThat(url).hasHost("attachments-test.s3-accelerate.amazonaws.com");
@ -28,12 +20,7 @@ public class UrlSignerTest {
@Test
public void testTransferUnaccelerated() {
AttachmentsConfiguration configuration = mock(AttachmentsConfiguration.class);
when(configuration.getAccessKey()).thenReturn("foo");
when(configuration.getAccessSecret()).thenReturn("bar");
when(configuration.getBucket()).thenReturn("attachments-test");
UrlSigner signer = new UrlSigner(configuration);
UrlSigner signer = new UrlSigner("foo", "bar", "attachments-test");
URL url = signer.getPreSignedUrl(1234, HttpMethod.GET, true);
assertThat(url).hasHost("s3.amazonaws.com");