Support for accessing attachments via CDN

src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java

@@ -30,7 +30,8 @@ import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
 import org.whispersystems.textsecuregcm.auth.DirectoryCredentialsGenerator;
 import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
 import org.whispersystems.textsecuregcm.controllers.AccountController;
-import org.whispersystems.textsecuregcm.controllers.AttachmentController;
+import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV1;
+import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV2;
 import org.whispersystems.textsecuregcm.controllers.CertificateController;
 import org.whispersystems.textsecuregcm.controllers.DeviceController;
 import org.whispersystems.textsecuregcm.controllers.DirectoryController;
@@ -62,7 +63,6 @@ import org.whispersystems.textsecuregcm.push.ReceiptSender;
 import org.whispersystems.textsecuregcm.push.WebsocketSender;
 import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient;
 import org.whispersystems.textsecuregcm.redis.ReplicatedJedisPool;
-import org.whispersystems.textsecuregcm.s3.UrlSigner;
 import org.whispersystems.textsecuregcm.sms.SmsSender;
 import org.whispersystems.textsecuregcm.sms.TwilioSmsSender;
 import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
@@ -190,7 +190,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
     ApnFallbackManager       apnFallbackManager  = new ApnFallbackManager(pushSchedulerClient, apnSender, accountsManager);
     TwilioSmsSender          twilioSmsSender     = new TwilioSmsSender(config.getTwilioConfiguration());
     SmsSender                smsSender           = new SmsSender(twilioSmsSender);
-    UrlSigner                urlSigner           = new UrlSigner(config.getAttachmentsConfiguration());
     PushSender               pushSender          = new PushSender(apnFallbackManager, gcmSender, apnSender, websocketSender, config.getPushConfiguration().getQueueSize());
     ReceiptSender            receiptSender       = new ReceiptSender(accountsManager, pushSender);
     TurnTokenGenerator       turnTokenGenerator  = new TurnTokenGenerator(config.getTurnConfiguration());
@@ -216,7 +215,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
     environment.lifecycle().manage(messagesCache);
     environment.lifecycle().manage(accountDatabaseCrawler);
 
-    AttachmentController attachmentController = new AttachmentController(rateLimiters, urlSigner);
+    AttachmentControllerV1 attachmentControllerV1 = new AttachmentControllerV1(rateLimiters, config.getAttachmentsConfiguration().getAccessKey(), config.getAttachmentsConfiguration().getAccessSecret(), config.getAttachmentsConfiguration().getBucket()                                                  );
+    AttachmentControllerV2 attachmentControllerV2 = new AttachmentControllerV2(rateLimiters, config.getAttachmentsConfiguration().getAccessKey(), config.getAttachmentsConfiguration().getAccessSecret(), config.getAttachmentsConfiguration().getRegion(), config.getAttachmentsConfiguration().getBucket());
     KeysController         keysController         = new KeysController(rateLimiters, keys, accountsManager, directoryQueue);
     MessageController      messageController      = new MessageController(rateLimiters, pushSender, receiptSender, accountsManager, messagesManager, apnFallbackManager);
     ProfileController      profileController      = new ProfileController(rateLimiters, accountsManager, config.getProfilesConfiguration());
@@ -233,7 +233,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
     environment.jersey().register(new CertificateController(new CertificateGenerator(config.getDeliveryCertificate().getCertificate(), config.getDeliveryCertificate().getPrivateKey(), config.getDeliveryCertificate().getExpiresDays())));
     environment.jersey().register(new VoiceVerificationController(config.getVoiceVerificationConfiguration().getUrl(), config.getVoiceVerificationConfiguration().getLocales()));
     environment.jersey().register(new TransparentDataController(accountsManager, config.getTransparentDataIndex()));
-    environment.jersey().register(attachmentController);
+    environment.jersey().register(attachmentControllerV1);
+    environment.jersey().register(attachmentControllerV2);
     environment.jersey().register(keysController);
     environment.jersey().register(messageController);
     environment.jersey().register(profileController);
@@ -245,6 +246,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
     webSocketEnvironment.jersey().register(new KeepAliveController(pubSubManager));
     webSocketEnvironment.jersey().register(messageController);
     webSocketEnvironment.jersey().register(profileController);
+    webSocketEnvironment.jersey().register(attachmentControllerV1);
+    webSocketEnvironment.jersey().register(attachmentControllerV2);
 
     WebSocketEnvironment provisioningEnvironment = new WebSocketEnvironment(environment, webSocketEnvironment.getRequestLog(), 60000);
     provisioningEnvironment.setConnectListener(new ProvisioningConnectListener(pubSubManager));

src/main/java/org/whispersystems/textsecuregcm/configuration/AttachmentsConfiguration.java

@@ -33,6 +33,10 @@ public class AttachmentsConfiguration {
   @JsonProperty
   private String bucket;
 
+  @NotEmpty
+  @JsonProperty
+  private String region;
+
   public String getAccessKey() {
     return accessKey;
   }
@@ -44,4 +48,8 @@ public class AttachmentsConfiguration {
   public String getBucket() {
     return bucket;
   }
+
+  public String getRegion() {
+    return region;
+  }
 }

src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerBase.java

@@ -0,0 +1,17 @@
+package org.whispersystems.textsecuregcm.controllers;
+
+import org.whispersystems.textsecuregcm.util.Conversions;
+
+import java.security.SecureRandom;
+
+public class AttachmentControllerBase {
+
+  protected long generateAttachmentId() {
+    byte[] attachmentBytes = new byte[8];
+    new SecureRandom().nextBytes(attachmentBytes);
+
+    attachmentBytes[0] = (byte)(attachmentBytes[0] & 0x7F);
+    return Conversions.byteArrayToLong(attachmentBytes);
+  }
+
+}

src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV1.java

@@ -20,7 +20,7 @@ import com.amazonaws.HttpMethod;
 import com.codahale.metrics.annotation.Timed;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.whispersystems.textsecuregcm.entities.AttachmentDescriptor;
+import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV1;
 import org.whispersystems.textsecuregcm.entities.AttachmentUri;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
 import org.whispersystems.textsecuregcm.s3.UrlSigner;
@@ -41,26 +41,25 @@ import io.dropwizard.auth.Auth;
 
 
 @Path("/v1/attachments")
-public class AttachmentController {
+public class AttachmentControllerV1 extends AttachmentControllerBase {
 
   @SuppressWarnings("unused")
-  private final Logger logger = LoggerFactory.getLogger(AttachmentController.class);
+  private final Logger logger = LoggerFactory.getLogger(AttachmentControllerV1.class);
 
   private static final String[] UNACCELERATED_REGIONS = {"+20", "+971", "+968", "+974"};
 
   private final RateLimiters rateLimiters;
   private final UrlSigner    urlSigner;
 
-  public AttachmentController(RateLimiters rateLimiters, UrlSigner urlSigner)
+  public AttachmentControllerV1(RateLimiters rateLimiters, String accessKey, String accessSecret, String bucket) {
-  {
     this.rateLimiters = rateLimiters;
-    this.urlSigner    = urlSigner;
+    this.urlSigner    = new UrlSigner(accessKey, accessSecret, bucket);
   }
 
   @Timed
   @GET
   @Produces(MediaType.APPLICATION_JSON)
-  public AttachmentDescriptor allocateAttachment(@Auth Account account)
+  public AttachmentDescriptorV1 allocateAttachment(@Auth Account account)
       throws RateLimitExceededException
   {
     if (account.isRateLimited()) {
@@ -70,7 +69,7 @@ public class AttachmentController {
     long attachmentId = generateAttachmentId();
     URL  url          = urlSigner.getPreSignedUrl(attachmentId, HttpMethod.PUT, Stream.of(UNACCELERATED_REGIONS).anyMatch(region -> account.getNumber().startsWith(region)));
 
-    return new AttachmentDescriptor(attachmentId, url.toExternalForm());
+    return new AttachmentDescriptorV1(attachmentId, url.toExternalForm());
 
   }
 
@@ -85,11 +84,4 @@ public class AttachmentController {
     return new AttachmentUri(urlSigner.getPreSignedUrl(attachmentId, HttpMethod.GET, Stream.of(UNACCELERATED_REGIONS).anyMatch(region -> account.getNumber().startsWith(region))));
   }
 
-  private long generateAttachmentId() {
-    byte[] attachmentBytes = new byte[8];
-    new SecureRandom().nextBytes(attachmentBytes);
-
-    attachmentBytes[0] = (byte)(attachmentBytes[0] & 0x7F);
-    return Conversions.byteArrayToLong(attachmentBytes);
-  }
 }

src/main/java/org/whispersystems/textsecuregcm/controllers/AttachmentControllerV2.java

@@ -0,0 +1,54 @@
+package org.whispersystems.textsecuregcm.controllers;
+
+import com.codahale.metrics.annotation.Timed;
+import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV2;
+import org.whispersystems.textsecuregcm.limits.RateLimiter;
+import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.s3.PolicySigner;
+import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
+import org.whispersystems.textsecuregcm.storage.Account;
+import org.whispersystems.textsecuregcm.util.Pair;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+
+import io.dropwizard.auth.Auth;
+
+@Path("/v2/attachments")
+public class AttachmentControllerV2 extends AttachmentControllerBase {
+
+  private final PostPolicyGenerator policyGenerator;
+  private final PolicySigner        policySigner;
+  private final RateLimiter         rateLimiter;
+
+  public AttachmentControllerV2(RateLimiters rateLimiters, String accessKey, String accessSecret, String region, String bucket) {
+    this.rateLimiter      = rateLimiters.getAttachmentLimiter();
+    this.policyGenerator  = new PostPolicyGenerator(region, bucket, accessKey);
+    this.policySigner     = new PolicySigner(accessSecret, region);
+  }
+
+  @Timed
+  @GET
+  @Produces(MediaType.APPLICATION_JSON)
+  @Path("/form/upload")
+  public AttachmentDescriptorV2 getAttachmentUploadForm(@Auth Account account) throws RateLimitExceededException {
+    rateLimiter.validate(account.getNumber());
+
+    ZonedDateTime        now          = ZonedDateTime.now(ZoneOffset.UTC);
+    long                 attachmentId = generateAttachmentId();
+    String               objectName   = String.valueOf(attachmentId);
+    Pair<String, String> policy       = policyGenerator.createFor(now, String.valueOf(objectName));
+    String               signature    = policySigner.getSignature(now, policy.second());
+
+    return new AttachmentDescriptorV2(attachmentId, objectName, policy.first(),
+                                      "private", "AWS4-HMAC-SHA256",
+                                      now.format(PostPolicyGenerator.AWS_DATE_TIME),
+                                      policy.second(), signature);
+  }
+
+
+}

src/main/java/org/whispersystems/textsecuregcm/entities/AttachmentDescriptorV1.java

@@ -18,7 +18,7 @@ package org.whispersystems.textsecuregcm.entities;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-public class AttachmentDescriptor {
+public class AttachmentDescriptorV1 {
 
   @JsonProperty
   private long id;
@@ -29,13 +29,13 @@ public class AttachmentDescriptor {
   @JsonProperty
   private String location;
 
-  public AttachmentDescriptor(long id, String location) {
+  public AttachmentDescriptorV1(long id, String location) {
     this.id       = id;
     this.idString = String.valueOf(id);
     this.location = location;
   }
 
-  public AttachmentDescriptor() {}
+  public AttachmentDescriptorV1() {}
 
   public long getId() {
     return id;

src/main/java/org/whispersystems/textsecuregcm/entities/AttachmentDescriptorV2.java

@@ -0,0 +1,88 @@
+package org.whispersystems.textsecuregcm.entities;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class AttachmentDescriptorV2 {
+
+  @JsonProperty
+  private String key;
+
+  @JsonProperty
+  private String credential;
+
+  @JsonProperty
+  private String acl;
+
+  @JsonProperty
+  private String algorithm;
+
+  @JsonProperty
+  private String date;
+
+  @JsonProperty
+  private String policy;
+
+  @JsonProperty
+  private String signature;
+
+  @JsonProperty
+  private long attachmentId;
+
+  @JsonProperty
+  private String attachmentIdString;
+
+  public AttachmentDescriptorV2() {}
+
+  public AttachmentDescriptorV2(long attachmentId,
+                                String key, String credential,
+                                String acl,  String algorithm,
+                                String date, String policy,
+                                String signature)
+  {
+    this.attachmentId       = attachmentId;
+    this.attachmentIdString = String.valueOf(attachmentId);
+    this.key                = key;
+    this.credential         = credential;
+    this.acl                = acl;
+    this.algorithm          = algorithm;
+    this.date               = date;
+    this.policy             = policy;
+    this.signature          = signature;
+  }
+
+  public String getKey() {
+    return key;
+  }
+
+  public String getCredential() {
+    return credential;
+  }
+
+  public String getAcl() {
+    return acl;
+  }
+
+  public String getAlgorithm() {
+    return algorithm;
+  }
+
+  public String getDate() {
+    return date;
+  }
+
+  public String getPolicy() {
+    return policy;
+  }
+
+  public String getSignature() {
+    return signature;
+  }
+
+  public long getAttachmentId() {
+    return attachmentId;
+  }
+
+  public String getAttachmentIdString() {
+    return attachmentIdString;
+  }
+}

src/main/java/org/whispersystems/textsecuregcm/s3/PostPolicyGenerator.java

@@ -35,6 +35,7 @@ public class PostPolicyGenerator {
                                         "    {\"key\": \"%s\"},\n" +
                                         "    {\"acl\": \"private\"},\n" +
                                         "    [\"starts-with\", \"$Content-Type\", \"\"],\n" +
+                                        "    [\"content-length-range\", 1, 104857600],\n" +
                                         "\n" +
                                         "    {\"x-amz-credential\": \"%s\"},\n" +
                                         "    {\"x-amz-algorithm\": \"AWS4-HMAC-SHA256\"},\n" +

src/main/java/org/whispersystems/textsecuregcm/s3/UrlSigner.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Copyright (C) 2013 Open WhisperSystems
  *
  * This program is free software: you can redistribute it and/or modify
@@ -23,7 +23,6 @@ import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.S3ClientOptions;
 import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
-import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration;
 
 import java.net.URL;
 import java.util.Date;
@@ -35,9 +34,9 @@ public class UrlSigner {
   private final AWSCredentials credentials;
   private final String bucket;
 
-  public UrlSigner(AttachmentsConfiguration config) {
+  public UrlSigner(String accessKey, String accessSecret, String bucket) {
-    this.credentials = new BasicAWSCredentials(config.getAccessKey(), config.getAccessSecret());
+    this.credentials = new BasicAWSCredentials(accessKey, accessSecret);
-    this.bucket      = config.getBucket();
+    this.bucket      = bucket;
   }
 
   public URL getPreSignedUrl(long attachmentId, HttpMethod method, boolean unaccelerated) {

src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AttachmentControllerTest.java

@@ -3,13 +3,13 @@ package org.whispersystems.textsecuregcm.tests.controllers;
 import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
 import org.junit.ClassRule;
 import org.junit.Test;
-import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration;
+import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV1;
-import org.whispersystems.textsecuregcm.controllers.AttachmentController;
+import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV2;
-import org.whispersystems.textsecuregcm.entities.AttachmentDescriptor;
+import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV1;
+import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV2;
 import org.whispersystems.textsecuregcm.entities.AttachmentUri;
 import org.whispersystems.textsecuregcm.limits.RateLimiter;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
-import org.whispersystems.textsecuregcm.s3.UrlSigner;
 import org.whispersystems.textsecuregcm.storage.Account;
 import org.whispersystems.textsecuregcm.tests.util.AuthHelper;
 import org.whispersystems.textsecuregcm.util.SystemMapper;
@@ -24,19 +24,11 @@ import static org.mockito.Mockito.when;
 
 public class AttachmentControllerTest {
 
-  private static AttachmentsConfiguration configuration = mock(AttachmentsConfiguration.class);
   private static RateLimiters             rateLimiters  = mock(RateLimiters.class            );
   private static RateLimiter              rateLimiter   = mock(RateLimiter.class             );
 
-  private static UrlSigner urlSigner;
-
   static {
-    when(configuration.getAccessKey()).thenReturn("accessKey");
-    when(configuration.getAccessSecret()).thenReturn("accessSecret");
-    when(configuration.getBucket()).thenReturn("attachment-bucket");
-
     when(rateLimiters.getAttachmentLimiter()).thenReturn(rateLimiter);
-    urlSigner = new UrlSigner(configuration);
   }
 
   @ClassRule
@@ -45,16 +37,43 @@ public class AttachmentControllerTest {
                                                                    .addProvider(new AuthValueFactoryProvider.Binder<>(Account.class))
                                                                    .setMapper(SystemMapper.getMapper())
                                                                    .setTestContainerFactory(new GrizzlyWebTestContainerFactory())
-                                                                   .addResource(new AttachmentController(rateLimiters, urlSigner))
+                                                                   .addResource(new AttachmentControllerV1(rateLimiters, "accessKey", "accessSecret", "attachment-bucket"))
+                                                                   .addResource(new AttachmentControllerV2(rateLimiters, "accessKey", "accessSecret", "us-east-1", "attachmentv2-bucket"))
                                                                    .build();
 
+  @Test
+  public void testV2Form() {
+    AttachmentDescriptorV2 descriptor = resources.getJerseyTest()
+                                                 .target("/v2/attachments/form/upload")
+                                                 .request()
+                                                 .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
+                                                 .get(AttachmentDescriptorV2.class);
+
+    assertThat(descriptor.getKey()).isEqualTo(descriptor.getAttachmentIdString());
+    assertThat(descriptor.getAcl()).isEqualTo("private");
+    assertThat(descriptor.getAlgorithm()).isEqualTo("AWS4-HMAC-SHA256");
+    assertThat(descriptor.getAttachmentId()).isGreaterThan(0);
+    assertThat(String.valueOf(descriptor.getAttachmentId())).isEqualTo(descriptor.getAttachmentIdString());
+
+    String[] credentialParts = descriptor.getCredential().split("/");
+
+    assertThat(credentialParts[0]).isEqualTo("accessKey");
+    assertThat(credentialParts[2]).isEqualTo("us-east-1");
+    assertThat(credentialParts[3]).isEqualTo("s3");
+    assertThat(credentialParts[4]).isEqualTo("aws4_request");
+
+    assertThat(descriptor.getDate()).isNotBlank();
+    assertThat(descriptor.getPolicy()).isNotBlank();
+    assertThat(descriptor.getSignature()).isNotBlank();
+  }
+
   @Test
   public void testAcceleratedPut() {
-    AttachmentDescriptor descriptor = resources.getJerseyTest()
+    AttachmentDescriptorV1 descriptor = resources.getJerseyTest()
                                                  .target("/v1/attachments/")
                                                  .request()
                                                  .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
-                                               .get(AttachmentDescriptor.class);
+                                                 .get(AttachmentDescriptorV1.class);
 
     assertThat(descriptor.getLocation()).startsWith("https://attachment-bucket.s3-accelerate.amazonaws.com");
     assertThat(descriptor.getId()).isGreaterThan(0);
@@ -63,11 +82,11 @@ public class AttachmentControllerTest {
 
   @Test
   public void testUnacceleratedPut() {
-    AttachmentDescriptor descriptor = resources.getJerseyTest()
+    AttachmentDescriptorV1 descriptor = resources.getJerseyTest()
                                                  .target("/v1/attachments/")
                                                  .request()
                                                  .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER_TWO, AuthHelper.VALID_PASSWORD_TWO))
-                                               .get(AttachmentDescriptor.class);
+                                                 .get(AttachmentDescriptorV1.class);
 
     assertThat(descriptor.getLocation()).startsWith("https://s3.amazonaws.com");
     assertThat(descriptor.getId()).isGreaterThan(0);

src/test/java/org/whispersystems/textsecuregcm/tests/util/UrlSignerTest.java

@@ -2,25 +2,17 @@ package org.whispersystems.textsecuregcm.tests.util;
 
 import com.amazonaws.HttpMethod;
 import org.junit.Test;
-import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration;
 import org.whispersystems.textsecuregcm.s3.UrlSigner;
 
 import java.net.URL;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class UrlSignerTest {
 
   @Test
   public void testTransferAcceleration() {
-    AttachmentsConfiguration configuration = mock(AttachmentsConfiguration.class);
+    UrlSigner signer = new UrlSigner("foo", "bar", "attachments-test");
-    when(configuration.getAccessKey()).thenReturn("foo");
-    when(configuration.getAccessSecret()).thenReturn("bar");
-    when(configuration.getBucket()).thenReturn("attachments-test");
-
-    UrlSigner signer = new UrlSigner(configuration);
     URL url = signer.getPreSignedUrl(1234, HttpMethod.GET, false);
 
     assertThat(url).hasHost("attachments-test.s3-accelerate.amazonaws.com");
@@ -28,12 +20,7 @@ public class UrlSignerTest {
 
   @Test
   public void testTransferUnaccelerated() {
-    AttachmentsConfiguration configuration = mock(AttachmentsConfiguration.class);
+    UrlSigner signer = new UrlSigner("foo", "bar", "attachments-test");
-    when(configuration.getAccessKey()).thenReturn("foo");
-    when(configuration.getAccessSecret()).thenReturn("bar");
-    when(configuration.getBucket()).thenReturn("attachments-test");
-
-    UrlSigner signer = new UrlSigner(configuration);
     URL url = signer.getPreSignedUrl(1234, HttpMethod.GET, true);
 
     assertThat(url).hasHost("s3.amazonaws.com");