From 79ab85c632eb4121d87033a5ca54ab9f81ef70b0 Mon Sep 17 00:00:00 2001 From: Moxie Marlinspike Date: Tue, 3 Oct 2017 14:38:12 -0700 Subject: [PATCH] Support for unaccelerated regions // FREEBIE --- .../textsecuregcm/WhisperServerService.java | 2 - .../controllers/AttachmentController.java | 7 +- .../controllers/ReceiptController.java | 47 -------- .../textsecuregcm/s3/UrlSigner.java | 8 +- .../controllers/AttachmentControllerTest.java | 100 ++++++++++++++++++ .../controllers/ReceiptControllerTest.java | 100 ------------------ .../textsecuregcm/tests/util/AuthHelper.java | 2 +- .../tests/util/UrlSignerTest.java | 16 ++- 8 files changed, 126 insertions(+), 156 deletions(-) delete mode 100644 src/main/java/org/whispersystems/textsecuregcm/controllers/ReceiptController.java create mode 100644 src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AttachmentControllerTest.java delete mode 100644 src/test/java/org/whispersystems/textsecuregcm/tests/controllers/ReceiptControllerTest.java diff --git a/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index fa5e7df24..6debf8685 100644 --- a/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -43,7 +43,6 @@ import org.whispersystems.textsecuregcm.controllers.KeysController; import org.whispersystems.textsecuregcm.controllers.MessageController; import org.whispersystems.textsecuregcm.controllers.ProfileController; import org.whispersystems.textsecuregcm.controllers.ProvisioningController; -import org.whispersystems.textsecuregcm.controllers.ReceiptController; import org.whispersystems.textsecuregcm.federation.FederatedClientManager; import org.whispersystems.textsecuregcm.federation.FederatedPeer; import org.whispersystems.textsecuregcm.limits.RateLimiters; @@ -214,7 +213,6 @@ public class WhisperServerService extends Application account.getNumber().startsWith(region))); return new AttachmentDescriptor(attachmentId, url.toExternalForm()); @@ -91,7 +94,7 @@ public class AttachmentController { { try { if (!relay.isPresent()) { - return new AttachmentUri(urlSigner.getPreSignedUrl(attachmentId, HttpMethod.GET)); + return new AttachmentUri(urlSigner.getPreSignedUrl(attachmentId, HttpMethod.GET, Stream.of(UNACCELERATED_REGIONS).anyMatch(region -> account.getNumber().startsWith(region)))); } else { return new AttachmentUri(federatedClientManager.getClient(relay.get()).getSignedAttachmentUri(attachmentId)); } diff --git a/src/main/java/org/whispersystems/textsecuregcm/controllers/ReceiptController.java b/src/main/java/org/whispersystems/textsecuregcm/controllers/ReceiptController.java deleted file mode 100644 index 6d56e72ff..000000000 --- a/src/main/java/org/whispersystems/textsecuregcm/controllers/ReceiptController.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.whispersystems.textsecuregcm.controllers; - -import com.codahale.metrics.annotation.Timed; -import com.google.common.base.Optional; -import org.whispersystems.textsecuregcm.push.NotPushRegisteredException; -import org.whispersystems.textsecuregcm.push.ReceiptSender; -import org.whispersystems.textsecuregcm.push.TransientPushFailureException; -import org.whispersystems.textsecuregcm.storage.Account; - -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; -import java.io.IOException; - -import io.dropwizard.auth.Auth; - -@Path("/v1/receipt") -public class ReceiptController { - - private final ReceiptSender receiptSender; - - public ReceiptController(ReceiptSender receiptSender) { - this.receiptSender = receiptSender; - } - - @Timed - @PUT - @Path("/{destination}/{messageId}") - public void sendDeliveryReceipt(@Auth Account source, - @PathParam("destination") String destination, - @PathParam("messageId") long messageId, - @QueryParam("relay") Optional relay) - throws IOException - { - try { - receiptSender.sendReceipt(source, destination, messageId, relay); - } catch (NoSuchUserException | NotPushRegisteredException e) { - throw new WebApplicationException(Response.Status.NOT_FOUND); - } catch (TransientPushFailureException e) { - throw new IOException(e); - } - } - -} diff --git a/src/main/java/org/whispersystems/textsecuregcm/s3/UrlSigner.java b/src/main/java/org/whispersystems/textsecuregcm/s3/UrlSigner.java index 05457a5aa..dfd19ef1a 100644 --- a/src/main/java/org/whispersystems/textsecuregcm/s3/UrlSigner.java +++ b/src/main/java/org/whispersystems/textsecuregcm/s3/UrlSigner.java @@ -40,14 +40,18 @@ public class UrlSigner { this.bucket = config.getBucket(); } - public URL getPreSignedUrl(long attachmentId, HttpMethod method) { + public URL getPreSignedUrl(long attachmentId, HttpMethod method, boolean unaccelerated) { AmazonS3 client = new AmazonS3Client(credentials); GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, String.valueOf(attachmentId), method); request.setExpiration(new Date(System.currentTimeMillis() + DURATION)); request.setContentType("application/octet-stream"); - client.setS3ClientOptions(S3ClientOptions.builder().setAccelerateModeEnabled(true).build()); + if (unaccelerated) { + client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); + } else { + client.setS3ClientOptions(S3ClientOptions.builder().setAccelerateModeEnabled(true).build()); + } return client.generatePresignedUrl(request); } diff --git a/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AttachmentControllerTest.java b/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AttachmentControllerTest.java new file mode 100644 index 000000000..f65451bf9 --- /dev/null +++ b/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/AttachmentControllerTest.java @@ -0,0 +1,100 @@ +package org.whispersystems.textsecuregcm.tests.controllers; + +import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory; +import org.junit.ClassRule; +import org.junit.Test; +import org.whispersystems.dropwizard.simpleauth.AuthValueFactoryProvider; +import org.whispersystems.textsecuregcm.configuration.AttachmentsConfiguration; +import org.whispersystems.textsecuregcm.controllers.AttachmentController; +import org.whispersystems.textsecuregcm.entities.AttachmentDescriptor; +import org.whispersystems.textsecuregcm.entities.AttachmentUri; +import org.whispersystems.textsecuregcm.federation.FederatedClientManager; +import org.whispersystems.textsecuregcm.limits.RateLimiter; +import org.whispersystems.textsecuregcm.limits.RateLimiters; +import org.whispersystems.textsecuregcm.s3.UrlSigner; +import org.whispersystems.textsecuregcm.tests.util.AuthHelper; +import org.whispersystems.textsecuregcm.util.SystemMapper; + +import java.net.MalformedURLException; + +import io.dropwizard.testing.junit.ResourceTestRule; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AttachmentControllerTest { + + private static AttachmentsConfiguration configuration = mock(AttachmentsConfiguration.class); + private static FederatedClientManager federatedClientManager = mock(FederatedClientManager.class ); + private static RateLimiters rateLimiters = mock(RateLimiters.class ); + private static RateLimiter rateLimiter = mock(RateLimiter.class ); + + private static UrlSigner urlSigner; + + static { + when(configuration.getAccessKey()).thenReturn("accessKey"); + when(configuration.getAccessSecret()).thenReturn("accessSecret"); + when(configuration.getBucket()).thenReturn("attachment-bucket"); + + when(rateLimiters.getAttachmentLimiter()).thenReturn(rateLimiter); + urlSigner = new UrlSigner(configuration); + } + + @ClassRule + public static final ResourceTestRule resources = ResourceTestRule.builder() + .addProvider(AuthHelper.getAuthFilter()) + .addProvider(new AuthValueFactoryProvider.Binder()) + .setMapper(SystemMapper.getMapper()) + .setTestContainerFactory(new GrizzlyWebTestContainerFactory()) + .addResource(new AttachmentController(rateLimiters, federatedClientManager, urlSigner)) + .build(); + + @Test + public void testAcceleratedPut() { + AttachmentDescriptor descriptor = resources.getJerseyTest() + .target("/v1/attachments/") + .request() + .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD)) + .get(AttachmentDescriptor.class); + + assertThat(descriptor.getLocation()).startsWith("https://attachment-bucket.s3-accelerate.amazonaws.com"); + assertThat(descriptor.getId()).isGreaterThan(0); + assertThat(descriptor.getIdString()).isNotBlank(); + } + + @Test + public void testUnacceleratedPut() { + AttachmentDescriptor descriptor = resources.getJerseyTest() + .target("/v1/attachments/") + .request() + .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER_TWO, AuthHelper.VALID_PASSWORD_TWO)) + .get(AttachmentDescriptor.class); + + assertThat(descriptor.getLocation()).startsWith("https://s3.amazonaws.com"); + assertThat(descriptor.getId()).isGreaterThan(0); + assertThat(descriptor.getIdString()).isNotBlank(); + } + + @Test + public void testAcceleratedGet() throws MalformedURLException { + AttachmentUri uri = resources.getJerseyTest() + .target("/v1/attachments/1234") + .request() + .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD)) + .get(AttachmentUri.class); + + assertThat(uri.getLocation().getHost()).isEqualTo("attachment-bucket.s3-accelerate.amazonaws.com"); + } + + @Test + public void testUnacceleratedGet() throws MalformedURLException { + AttachmentUri uri = resources.getJerseyTest() + .target("/v1/attachments/1234") + .request() + .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER_TWO, AuthHelper.VALID_PASSWORD_TWO)) + .get(AttachmentUri.class); + + assertThat(uri.getLocation().getHost()).isEqualTo("s3.amazonaws.com"); + } + +} diff --git a/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/ReceiptControllerTest.java b/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/ReceiptControllerTest.java deleted file mode 100644 index 529009fbc..000000000 --- a/src/test/java/org/whispersystems/textsecuregcm/tests/controllers/ReceiptControllerTest.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.whispersystems.textsecuregcm.tests.controllers; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Optional; -import org.glassfish.jersey.client.ClientProperties; -import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.whispersystems.dropwizard.simpleauth.AuthValueFactoryProvider; -import org.whispersystems.textsecuregcm.controllers.ReceiptController; -import org.whispersystems.textsecuregcm.entities.MessageProtos.Envelope; -import org.whispersystems.textsecuregcm.federation.FederatedClientManager; -import org.whispersystems.textsecuregcm.push.PushSender; -import org.whispersystems.textsecuregcm.push.ReceiptSender; -import org.whispersystems.textsecuregcm.storage.Account; -import org.whispersystems.textsecuregcm.storage.AccountsManager; -import org.whispersystems.textsecuregcm.storage.Device; -import org.whispersystems.textsecuregcm.tests.util.AuthHelper; - -import javax.ws.rs.core.Response; -import java.util.HashSet; -import java.util.Set; - -import io.dropwizard.testing.junit.ResourceTestRule; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.*; - -public class ReceiptControllerTest { - - private static final String SINGLE_DEVICE_RECIPIENT = "+14151111111"; - private static final String MULTI_DEVICE_RECIPIENT = "+14152222222"; - - private final PushSender pushSender = mock(PushSender.class ); - private final FederatedClientManager federatedClientManager = mock(FederatedClientManager.class); - private final AccountsManager accountsManager = mock(AccountsManager.class ); - - private final ReceiptSender receiptSender = new ReceiptSender(accountsManager, pushSender, federatedClientManager); - - private final ObjectMapper mapper = new ObjectMapper(); - - @Rule - public final ResourceTestRule resources = ResourceTestRule.builder() - .addProvider(AuthHelper.getAuthFilter()) - .addProvider(new AuthValueFactoryProvider.Binder()) - .setTestContainerFactory(new GrizzlyWebTestContainerFactory()) - .addResource(new ReceiptController(receiptSender)) - .build(); - - @Before - public void setup() throws Exception { - Set singleDeviceList = new HashSet() {{ - add(new Device(1, null, "foo", "bar", "baz", "isgcm", null, null, false, 111, null, System.currentTimeMillis(), System.currentTimeMillis(), false, false, "Test")); - }}; - - Set multiDeviceList = new HashSet() {{ - add(new Device(1, null, "foo", "bar", "baz", "isgcm", null, null, false, 222, null, System.currentTimeMillis(), System.currentTimeMillis(), false, false, "Test")); - add(new Device(2, null, "foo", "bar", "baz", "isgcm", null, null, false, 333, null, System.currentTimeMillis(), System.currentTimeMillis(), false, false, "Test")); - }}; - - Account singleDeviceAccount = new Account(SINGLE_DEVICE_RECIPIENT, singleDeviceList); - Account multiDeviceAccount = new Account(MULTI_DEVICE_RECIPIENT, multiDeviceList); - - when(accountsManager.get(eq(SINGLE_DEVICE_RECIPIENT))).thenReturn(Optional.of(singleDeviceAccount)); - when(accountsManager.get(eq(MULTI_DEVICE_RECIPIENT))).thenReturn(Optional.of(multiDeviceAccount)); - } - - @Test - public synchronized void testSingleDeviceCurrent() throws Exception { - Response response = - resources.getJerseyTest() - .target(String.format("/v1/receipt/%s/%d", SINGLE_DEVICE_RECIPIENT, 1234)) - .request() - .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true) - .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD)) - .put(null); - - assertThat(response.getStatus() == 204); - - verify(pushSender, times(1)).sendMessage(any(Account.class), any(Device.class), any(Envelope.class), eq(true)); - } - - @Test - public synchronized void testMultiDeviceCurrent() throws Exception { - Response response = - resources.getJerseyTest() - .target(String.format("/v1/receipt/%s/%d", MULTI_DEVICE_RECIPIENT, 12345)) - .request() - .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true) - .header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD)) - .put(null); - - assertThat(response.getStatus() == 204); - - verify(pushSender, times(2)).sendMessage(any(Account.class), any(Device.class), any(Envelope.class), eq(true)); - } - - -} diff --git a/src/test/java/org/whispersystems/textsecuregcm/tests/util/AuthHelper.java b/src/test/java/org/whispersystems/textsecuregcm/tests/util/AuthHelper.java index 2f70497ca..69e511f6c 100644 --- a/src/test/java/org/whispersystems/textsecuregcm/tests/util/AuthHelper.java +++ b/src/test/java/org/whispersystems/textsecuregcm/tests/util/AuthHelper.java @@ -25,7 +25,7 @@ public class AuthHelper { public static final String VALID_NUMBER = "+14150000000"; public static final String VALID_PASSWORD = "foo"; - public static final String VALID_NUMBER_TWO = "+14151111111"; + public static final String VALID_NUMBER_TWO = "+201511111110"; public static final String VALID_PASSWORD_TWO = "baz"; public static final String INVVALID_NUMBER = "+14151111111"; diff --git a/src/test/java/org/whispersystems/textsecuregcm/tests/util/UrlSignerTest.java b/src/test/java/org/whispersystems/textsecuregcm/tests/util/UrlSignerTest.java index b5bf1054a..851caaf56 100644 --- a/src/test/java/org/whispersystems/textsecuregcm/tests/util/UrlSignerTest.java +++ b/src/test/java/org/whispersystems/textsecuregcm/tests/util/UrlSignerTest.java @@ -21,10 +21,22 @@ public class UrlSignerTest { when(configuration.getBucket()).thenReturn("attachments-test"); UrlSigner signer = new UrlSigner(configuration); - URL url = signer.getPreSignedUrl(1234, HttpMethod.GET); + URL url = signer.getPreSignedUrl(1234, HttpMethod.GET, false); - System.out.println("The URL: " + url); assertThat(url).hasHost("attachments-test.s3-accelerate.amazonaws.com"); } + @Test + public void testTransferUnaccelerated() { + AttachmentsConfiguration configuration = mock(AttachmentsConfiguration.class); + when(configuration.getAccessKey()).thenReturn("foo"); + when(configuration.getAccessSecret()).thenReturn("bar"); + when(configuration.getBucket()).thenReturn("attachments-test"); + + UrlSigner signer = new UrlSigner(configuration); + URL url = signer.getPreSignedUrl(1234, HttpMethod.GET, true); + + assertThat(url).hasHost("s3.amazonaws.com"); + } + }