Create attachments V3 endpoint for CDN2 on GCP
In preparation for resumable uploads, this creates a separate attachment authorization endpoint that creates a signed URL for accessing GCP Storage through Signal's CDN2. This should allow Signal clients to do byte-level resume of media uploads.
This commit is contained in:
Ehren Kret
parent
2aca007a59
commit
41286650cc
|
|
@ -56,10 +56,18 @@ messageStore: # Postgresql database configuration for message store
|
|||
password:
|
||||
url:
|
||||
|
||||
attachments: # AWS S3 configuration
|
||||
awsAttachments: # AWS S3 configuration
|
||||
accessKey:
|
||||
accessSecret:
|
||||
bucket:
|
||||
region:
|
||||
|
||||
gcpAttachments: # GCP Storage configuration
|
||||
domain:
|
||||
email:
|
||||
maxSizeInBytes:
|
||||
pathPrefix:
|
||||
rsaSigningKey:
|
||||
|
||||
profiles: # AWS S3 configuration
|
||||
accessKey:
|
||||
|
|
|
|||
|
|
@ -46,7 +46,12 @@ public class WhisperServerConfiguration extends Configuration {
|
|||
@NotNull
|
||||
@Valid
|
||||
@JsonProperty
|
||||
private AttachmentsConfiguration attachments;
|
||||
private AwsAttachmentsConfiguration awsAttachments;
|
||||
|
||||
@NotNull
|
||||
@Valid
|
||||
@JsonProperty
|
||||
private GcpAttachmentsConfiguration gcpAttachments;
|
||||
|
||||
@NotNull
|
||||
@Valid
|
||||
|
|
@ -199,8 +204,12 @@ public class WhisperServerConfiguration extends Configuration {
|
|||
return httpClient;
|
||||
}
|
||||
|
||||
public AttachmentsConfiguration getAttachmentsConfiguration() {
|
||||
return attachments;
|
||||
public AwsAttachmentsConfiguration getAwsAttachmentsConfiguration() {
|
||||
return awsAttachments;
|
||||
}
|
||||
|
||||
public GcpAttachmentsConfiguration getGcpAttachmentsConfiguration() {
|
||||
return gcpAttachments;
|
||||
}
|
||||
|
||||
public RedisConfiguration getCacheConfiguration() {
|
||||
|
|
|
|||
|
|
@ -29,6 +29,17 @@ import com.fasterxml.jackson.annotation.PropertyAccessor;
|
|||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.dropwizard.Application;
|
||||
import io.dropwizard.auth.AuthFilter;
|
||||
import io.dropwizard.auth.PolymorphicAuthDynamicFeature;
|
||||
import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
|
||||
import io.dropwizard.auth.basic.BasicCredentialAuthFilter;
|
||||
import io.dropwizard.auth.basic.BasicCredentials;
|
||||
import io.dropwizard.db.DataSourceFactory;
|
||||
import io.dropwizard.db.PooledDataSourceFactory;
|
||||
import io.dropwizard.jdbi3.JdbiFactory;
|
||||
import io.dropwizard.setup.Bootstrap;
|
||||
import io.dropwizard.setup.Environment;
|
||||
import org.bouncycastle.jce.provider.BouncyCastleProvider;
|
||||
import org.eclipse.jetty.servlets.CrossOriginFilter;
|
||||
import org.jdbi.v3.core.Jdbi;
|
||||
|
|
@ -42,7 +53,23 @@ import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccount;
|
|||
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccountAuthenticator;
|
||||
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
|
||||
import org.whispersystems.textsecuregcm.auth.TurnTokenGenerator;
|
||||
import org.whispersystems.textsecuregcm.controllers.*;
|
||||
import org.whispersystems.textsecuregcm.controllers.AccountController;
|
||||
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV1;
|
||||
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV2;
|
||||
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV3;
|
||||
import org.whispersystems.textsecuregcm.controllers.CertificateController;
|
||||
import org.whispersystems.textsecuregcm.controllers.DeviceController;
|
||||
import org.whispersystems.textsecuregcm.controllers.DirectoryController;
|
||||
import org.whispersystems.textsecuregcm.controllers.KeepAliveController;
|
||||
import org.whispersystems.textsecuregcm.controllers.KeysController;
|
||||
import org.whispersystems.textsecuregcm.controllers.MessageController;
|
||||
import org.whispersystems.textsecuregcm.controllers.ProfileController;
|
||||
import org.whispersystems.textsecuregcm.controllers.ProvisioningController;
|
||||
import org.whispersystems.textsecuregcm.controllers.RemoteConfigController;
|
||||
import org.whispersystems.textsecuregcm.controllers.SecureBackupController;
|
||||
import org.whispersystems.textsecuregcm.controllers.SecureStorageController;
|
||||
import org.whispersystems.textsecuregcm.controllers.StickerController;
|
||||
import org.whispersystems.textsecuregcm.controllers.VoiceVerificationController;
|
||||
import org.whispersystems.textsecuregcm.limits.RateLimiters;
|
||||
import org.whispersystems.textsecuregcm.liquibase.NameableMigrationsBundle;
|
||||
import org.whispersystems.textsecuregcm.mappers.DeviceLimitExceededExceptionMapper;
|
||||
|
|
@ -69,7 +96,36 @@ import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
|
|||
import org.whispersystems.textsecuregcm.sms.SmsSender;
|
||||
import org.whispersystems.textsecuregcm.sms.TwilioSmsSender;
|
||||
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
|
||||
import org.whispersystems.textsecuregcm.storage.*;
|
||||
import org.whispersystems.textsecuregcm.storage.AbusiveHostRules;
|
||||
import org.whispersystems.textsecuregcm.storage.Account;
|
||||
import org.whispersystems.textsecuregcm.storage.AccountCleaner;
|
||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawler;
|
||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerCache;
|
||||
import org.whispersystems.textsecuregcm.storage.AccountDatabaseCrawlerListener;
|
||||
import org.whispersystems.textsecuregcm.storage.Accounts;
|
||||
import org.whispersystems.textsecuregcm.storage.AccountsManager;
|
||||
import org.whispersystems.textsecuregcm.storage.ActiveUserCounter;
|
||||
import org.whispersystems.textsecuregcm.storage.DirectoryManager;
|
||||
import org.whispersystems.textsecuregcm.storage.DirectoryReconciler;
|
||||
import org.whispersystems.textsecuregcm.storage.DirectoryReconciliationClient;
|
||||
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
|
||||
import org.whispersystems.textsecuregcm.storage.Keys;
|
||||
import org.whispersystems.textsecuregcm.storage.Messages;
|
||||
import org.whispersystems.textsecuregcm.storage.MessagesCache;
|
||||
import org.whispersystems.textsecuregcm.storage.MessagesManager;
|
||||
import org.whispersystems.textsecuregcm.storage.PendingAccounts;
|
||||
import org.whispersystems.textsecuregcm.storage.PendingAccountsManager;
|
||||
import org.whispersystems.textsecuregcm.storage.PendingDevices;
|
||||
import org.whispersystems.textsecuregcm.storage.PendingDevicesManager;
|
||||
import org.whispersystems.textsecuregcm.storage.Profiles;
|
||||
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
|
||||
import org.whispersystems.textsecuregcm.storage.PubSubManager;
|
||||
import org.whispersystems.textsecuregcm.storage.PushFeedbackProcessor;
|
||||
import org.whispersystems.textsecuregcm.storage.RemoteConfigs;
|
||||
import org.whispersystems.textsecuregcm.storage.RemoteConfigsManager;
|
||||
import org.whispersystems.textsecuregcm.storage.ReservedUsernames;
|
||||
import org.whispersystems.textsecuregcm.storage.Usernames;
|
||||
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
|
||||
import org.whispersystems.textsecuregcm.util.Constants;
|
||||
import org.whispersystems.textsecuregcm.websocket.AuthenticatedConnectListener;
|
||||
import org.whispersystems.textsecuregcm.websocket.DeadLetterHandler;
|
||||
|
|
@ -91,17 +147,6 @@ import java.util.List;
|
|||
import java.util.Optional;
|
||||
|
||||
import static com.codahale.metrics.MetricRegistry.name;
|
||||
import io.dropwizard.Application;
|
||||
import io.dropwizard.auth.AuthFilter;
|
||||
import io.dropwizard.auth.PolymorphicAuthDynamicFeature;
|
||||
import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
|
||||
import io.dropwizard.auth.basic.BasicCredentialAuthFilter;
|
||||
import io.dropwizard.auth.basic.BasicCredentials;
|
||||
import io.dropwizard.db.DataSourceFactory;
|
||||
import io.dropwizard.db.PooledDataSourceFactory;
|
||||
import io.dropwizard.jdbi3.JdbiFactory;
|
||||
import io.dropwizard.setup.Bootstrap;
|
||||
import io.dropwizard.setup.Environment;
|
||||
|
||||
public class WhisperServerService extends Application<WhisperServerConfiguration> {
|
||||
|
||||
|
|
@ -243,24 +288,25 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
|||
environment.lifecycle().manage(accountDatabaseCrawler);
|
||||
environment.lifecycle().manage(remoteConfigsManager);
|
||||
|
||||
AWSCredentials credentials = new BasicAWSCredentials(config.getCdnConfiguration().getAccessKey(), config.getCdnConfiguration().getAccessSecret());
|
||||
AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(credentials);
|
||||
AmazonS3 cdnS3Client = AmazonS3Client.builder().withCredentials(credentialsProvider).withRegion(config.getCdnConfiguration().getRegion()).build();
|
||||
PostPolicyGenerator cdnPolicyGenerator = new PostPolicyGenerator(config.getCdnConfiguration().getRegion(), config.getCdnConfiguration().getBucket(), config.getCdnConfiguration().getAccessKey());
|
||||
PolicySigner cdnPolicySigner = new PolicySigner(config.getCdnConfiguration().getAccessSecret(), config.getCdnConfiguration().getRegion());
|
||||
AWSCredentials credentials = new BasicAWSCredentials(config.getCdnConfiguration().getAccessKey(), config.getCdnConfiguration().getAccessSecret());
|
||||
AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(credentials);
|
||||
AmazonS3 cdnS3Client = AmazonS3Client.builder().withCredentials(credentialsProvider).withRegion(config.getCdnConfiguration().getRegion()).build();
|
||||
PostPolicyGenerator profileCdnPolicyGenerator = new PostPolicyGenerator(config.getCdnConfiguration().getRegion(), config.getCdnConfiguration().getBucket(), config.getCdnConfiguration().getAccessKey());
|
||||
PolicySigner profileCdnPolicySigner = new PolicySigner(config.getCdnConfiguration().getAccessSecret(), config.getCdnConfiguration().getRegion());
|
||||
|
||||
ServerSecretParams zkSecretParams = new ServerSecretParams(config.getZkConfig().getServerSecret());
|
||||
ServerZkProfileOperations zkProfileOperations = new ServerZkProfileOperations(zkSecretParams);
|
||||
ServerZkAuthOperations zkAuthOperations = new ServerZkAuthOperations(zkSecretParams);
|
||||
boolean isZkEnabled = config.getZkConfig().isEnabled();
|
||||
ServerSecretParams zkSecretParams = new ServerSecretParams(config.getZkConfig().getServerSecret());
|
||||
ServerZkProfileOperations zkProfileOperations = new ServerZkProfileOperations(zkSecretParams);
|
||||
ServerZkAuthOperations zkAuthOperations = new ServerZkAuthOperations(zkSecretParams);
|
||||
boolean isZkEnabled = config.getZkConfig().isEnabled();
|
||||
|
||||
AttachmentControllerV1 attachmentControllerV1 = new AttachmentControllerV1(rateLimiters, config.getAttachmentsConfiguration().getAccessKey(), config.getAttachmentsConfiguration().getAccessSecret(), config.getAttachmentsConfiguration().getBucket() );
|
||||
AttachmentControllerV2 attachmentControllerV2 = new AttachmentControllerV2(rateLimiters, config.getAttachmentsConfiguration().getAccessKey(), config.getAttachmentsConfiguration().getAccessSecret(), config.getAttachmentsConfiguration().getRegion(), config.getAttachmentsConfiguration().getBucket());
|
||||
KeysController keysController = new KeysController(rateLimiters, keys, accountsManager, directoryQueue);
|
||||
MessageController messageController = new MessageController(rateLimiters, pushSender, receiptSender, accountsManager, messagesManager, apnFallbackManager);
|
||||
ProfileController profileController = new ProfileController(rateLimiters, accountsManager, profilesManager, usernamesManager, cdnS3Client, cdnPolicyGenerator, cdnPolicySigner, config.getCdnConfiguration().getBucket(), zkProfileOperations, isZkEnabled);
|
||||
StickerController stickerController = new StickerController(rateLimiters, config.getCdnConfiguration().getAccessKey(), config.getCdnConfiguration().getAccessSecret(), config.getCdnConfiguration().getRegion(), config.getCdnConfiguration().getBucket());
|
||||
RemoteConfigController remoteConfigController = new RemoteConfigController(remoteConfigsManager, config.getRemoteConfigConfiguration().getAuthorizedTokens());
|
||||
AttachmentControllerV1 attachmentControllerV1 = new AttachmentControllerV1(rateLimiters, config.getAwsAttachmentsConfiguration().getAccessKey(), config.getAwsAttachmentsConfiguration().getAccessSecret(), config.getAwsAttachmentsConfiguration().getBucket());
|
||||
AttachmentControllerV2 attachmentControllerV2 = new AttachmentControllerV2(rateLimiters, config.getAwsAttachmentsConfiguration().getAccessKey(), config.getAwsAttachmentsConfiguration().getAccessSecret(), config.getAwsAttachmentsConfiguration().getRegion(), config.getAwsAttachmentsConfiguration().getBucket());
|
||||
AttachmentControllerV3 attachmentControllerV3 = new AttachmentControllerV3(rateLimiters, config.getGcpAttachmentsConfiguration().getDomain(), config.getGcpAttachmentsConfiguration().getEmail(), config.getGcpAttachmentsConfiguration().getMaxSizeInBytes(), config.getGcpAttachmentsConfiguration().getPathPrefix(), config.getGcpAttachmentsConfiguration().getRsaSigningKey());
|
||||
KeysController keysController = new KeysController(rateLimiters, keys, accountsManager, directoryQueue);
|
||||
MessageController messageController = new MessageController(rateLimiters, pushSender, receiptSender, accountsManager, messagesManager, apnFallbackManager);
|
||||
ProfileController profileController = new ProfileController(rateLimiters, accountsManager, profilesManager, usernamesManager, cdnS3Client, profileCdnPolicyGenerator, profileCdnPolicySigner, config.getCdnConfiguration().getBucket(), zkProfileOperations, isZkEnabled);
|
||||
StickerController stickerController = new StickerController(rateLimiters, config.getCdnConfiguration().getAccessKey(), config.getCdnConfiguration().getAccessSecret(), config.getCdnConfiguration().getRegion(), config.getCdnConfiguration().getBucket());
|
||||
RemoteConfigController remoteConfigController = new RemoteConfigController(remoteConfigsManager, config.getRemoteConfigConfiguration().getAuthorizedTokens());
|
||||
|
||||
AuthFilter<BasicCredentials, Account> accountAuthFilter = new BasicCredentialAuthFilter.Builder<Account>().setAuthenticator(accountAuthenticator).buildAuthFilter ();
|
||||
AuthFilter<BasicCredentials, DisabledPermittedAccount> disabledPermittedAccountAuthFilter = new BasicCredentialAuthFilter.Builder<DisabledPermittedAccount>().setAuthenticator(disabledPermittedAccountAuthenticator).buildAuthFilter();
|
||||
|
|
@ -279,6 +325,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
|||
environment.jersey().register(new SecureBackupController(backupCredentialsGenerator));
|
||||
environment.jersey().register(attachmentControllerV1);
|
||||
environment.jersey().register(attachmentControllerV2);
|
||||
environment.jersey().register(attachmentControllerV3);
|
||||
environment.jersey().register(keysController);
|
||||
environment.jersey().register(messageController);
|
||||
environment.jersey().register(profileController);
|
||||
|
|
@ -294,6 +341,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
|
|||
webSocketEnvironment.jersey().register(profileController);
|
||||
webSocketEnvironment.jersey().register(attachmentControllerV1);
|
||||
webSocketEnvironment.jersey().register(attachmentControllerV2);
|
||||
webSocketEnvironment.jersey().register(attachmentControllerV3);
|
||||
webSocketEnvironment.jersey().register(remoteConfigController);
|
||||
|
||||
WebSocketEnvironment<Account> provisioningEnvironment = new WebSocketEnvironment<>(environment, webSocketEnvironment.getRequestLog(), 60000);
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ package org.whispersystems.textsecuregcm.configuration;
|
|||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.hibernate.validator.constraints.NotEmpty;
|
||||
|
||||
public class AttachmentsConfiguration {
|
||||
public class AwsAttachmentsConfiguration {
|
||||
|
||||
@NotEmpty
|
||||
@JsonProperty
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
package org.whispersystems.textsecuregcm.configuration;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.dropwizard.util.Strings;
|
||||
import io.dropwizard.validation.ValidationMethod;
|
||||
import org.hibernate.validator.constraints.NotEmpty;
|
||||
|
||||
import javax.validation.constraints.Min;
|
||||
|
||||
public class GcpAttachmentsConfiguration {
|
||||
|
||||
@NotEmpty
|
||||
@JsonProperty
|
||||
private String domain;
|
||||
|
||||
@NotEmpty
|
||||
@JsonProperty
|
||||
private String email;
|
||||
|
||||
@JsonProperty
|
||||
@Min(1)
|
||||
private int maxSizeInBytes;
|
||||
|
||||
@JsonProperty
|
||||
private String pathPrefix;
|
||||
|
||||
@NotEmpty
|
||||
@JsonProperty
|
||||
private String rsaSigningKey;
|
||||
|
||||
public String getDomain() {
|
||||
return domain;
|
||||
}
|
||||
|
||||
public String getEmail() {
|
||||
return email;
|
||||
}
|
||||
|
||||
public int getMaxSizeInBytes() {
|
||||
return maxSizeInBytes;
|
||||
}
|
||||
|
||||
public String getPathPrefix() {
|
||||
return pathPrefix;
|
||||
}
|
||||
|
||||
public String getRsaSigningKey() {
|
||||
return rsaSigningKey;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@ValidationMethod(message = "pathPrefix must be empty or start with /")
|
||||
public boolean isPathPrefixValid() {
|
||||
return Strings.isNullOrEmpty(pathPrefix) || pathPrefix.startsWith("/");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
package org.whispersystems.textsecuregcm.controllers;
|
||||
|
||||
import com.codahale.metrics.annotation.Timed;
|
||||
import io.dropwizard.auth.Auth;
|
||||
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV3;
|
||||
import org.whispersystems.textsecuregcm.gcp.CanonicalRequest;
|
||||
import org.whispersystems.textsecuregcm.gcp.CanonicalRequestGenerator;
|
||||
import org.whispersystems.textsecuregcm.gcp.CanonicalRequestSigner;
|
||||
import org.whispersystems.textsecuregcm.limits.RateLimiter;
|
||||
import org.whispersystems.textsecuregcm.limits.RateLimiters;
|
||||
import org.whispersystems.textsecuregcm.storage.Account;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import javax.ws.rs.GET;
|
||||
import javax.ws.rs.Path;
|
||||
import javax.ws.rs.Produces;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import java.io.IOException;
|
||||
import java.security.InvalidKeyException;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Path("/v3/attachments")
|
||||
public class AttachmentControllerV3 extends AttachmentControllerBase {
|
||||
|
||||
@Nonnull
|
||||
private final RateLimiter rateLimiter;
|
||||
|
||||
@Nonnull
|
||||
private final CanonicalRequestGenerator canonicalRequestGenerator;
|
||||
|
||||
@Nonnull
|
||||
private final CanonicalRequestSigner canonicalRequestSigner;
|
||||
|
||||
public AttachmentControllerV3(@Nonnull RateLimiters rateLimiters, @Nonnull String domain, @Nonnull String email, int maxSizeInBytes, @Nonnull String pathPrefix, @Nonnull String rsaSigningKey)
|
||||
throws IOException, InvalidKeyException {
|
||||
this.rateLimiter = rateLimiters.getAttachmentLimiter();
|
||||
this.canonicalRequestGenerator = new CanonicalRequestGenerator(domain, email, maxSizeInBytes, pathPrefix);
|
||||
this.canonicalRequestSigner = new CanonicalRequestSigner(rsaSigningKey);
|
||||
}
|
||||
|
||||
@Timed
|
||||
@GET
|
||||
@Produces(MediaType.APPLICATION_JSON)
|
||||
@Path("/form/upload")
|
||||
public AttachmentDescriptorV3 getAttachmentUploadForm(@Auth Account account) throws RateLimitExceededException {
|
||||
rateLimiter.validate(account.getNumber());
|
||||
|
||||
final ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
|
||||
final String key = String.valueOf(generateAttachmentId());
|
||||
final CanonicalRequest canonicalRequest = canonicalRequestGenerator.createFor(key, now);
|
||||
|
||||
return new AttachmentDescriptorV3(2, key, getHeaderMap(canonicalRequest), getSignedUploadLocation(canonicalRequest));
|
||||
}
|
||||
|
||||
public String getSignedUploadLocation(@Nonnull CanonicalRequest canonicalRequest) {
|
||||
return "https://" + canonicalRequest.getDomain() + canonicalRequest.getResourcePath()
|
||||
+ '?' + canonicalRequest.getCanonicalQuery()
|
||||
+ "&X-Goog-Signature=" + canonicalRequestSigner.sign(canonicalRequest);
|
||||
}
|
||||
|
||||
public static Map<String, String> getHeaderMap(@Nonnull CanonicalRequest canonicalRequest) {
|
||||
Map<String, String> result = new HashMap<>(3);
|
||||
result.put("host", canonicalRequest.getDomain());
|
||||
result.put("x-goog-content-length-range", "1," + canonicalRequest.getMaxSizeInBytes());
|
||||
result.put("x-goog-resumable", "start");
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package org.whispersystems.textsecuregcm.entities;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class AttachmentDescriptorV3 {
|
||||
|
||||
@JsonProperty
|
||||
private int cdn;
|
||||
|
||||
@JsonProperty
|
||||
private String key;
|
||||
|
||||
@JsonProperty
|
||||
private Map<String, String> headers;
|
||||
|
||||
@JsonProperty
|
||||
private String signedUploadLocation;
|
||||
|
||||
public AttachmentDescriptorV3() {
|
||||
}
|
||||
|
||||
public AttachmentDescriptorV3(int cdn, String key, Map<String, String> headers, String signedUploadLocation) {
|
||||
this.cdn = cdn;
|
||||
this.key = key;
|
||||
this.headers = headers;
|
||||
this.signedUploadLocation = signedUploadLocation;
|
||||
}
|
||||
|
||||
public int getCdn() {
|
||||
return cdn;
|
||||
}
|
||||
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public Map<String, String> getHeaders() {
|
||||
return headers;
|
||||
}
|
||||
|
||||
public String getSignedUploadLocation() {
|
||||
return signedUploadLocation;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
package org.whispersystems.textsecuregcm.gcp;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class CanonicalRequest {
|
||||
|
||||
@Nonnull
|
||||
private final String canonicalRequest;
|
||||
|
||||
@Nonnull
|
||||
private final String resourcePath;
|
||||
|
||||
@Nonnull
|
||||
private final String canonicalQuery;
|
||||
|
||||
@Nonnull
|
||||
private final String activeDatetime;
|
||||
|
||||
@Nonnull
|
||||
private final String credentialScope;
|
||||
|
||||
@Nonnull
|
||||
private final String domain;
|
||||
|
||||
private final int maxSizeInBytes;
|
||||
|
||||
public CanonicalRequest(@Nonnull String canonicalRequest, @Nonnull String resourcePath, @Nonnull String canonicalQuery, @Nonnull String activeDatetime, @Nonnull String credentialScope, @Nonnull String domain, int maxSizeInBytes) {
|
||||
this.canonicalRequest = canonicalRequest;
|
||||
this.resourcePath = resourcePath;
|
||||
this.canonicalQuery = canonicalQuery;
|
||||
this.activeDatetime = activeDatetime;
|
||||
this.credentialScope = credentialScope;
|
||||
this.domain = domain;
|
||||
this.maxSizeInBytes = maxSizeInBytes;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
String getCanonicalRequest() {
|
||||
return canonicalRequest;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public String getResourcePath() {
|
||||
return resourcePath;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public String getCanonicalQuery() {
|
||||
return canonicalQuery;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
String getActiveDatetime() {
|
||||
return activeDatetime;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
String getCredentialScope() {
|
||||
return credentialScope;
|
||||
}
|
||||
|
||||
@Nonnull
|
||||
public String getDomain() {
|
||||
return domain;
|
||||
}
|
||||
|
||||
public int getMaxSizeInBytes() {
|
||||
return maxSizeInBytes;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
package org.whispersystems.textsecuregcm.gcp;
|
||||
|
||||
import io.dropwizard.util.Strings;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Locale;
|
||||
|
||||
public class CanonicalRequestGenerator {
|
||||
private static final DateTimeFormatter SIMPLE_UTC_DATE = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.US).withZone(ZoneOffset.UTC);
|
||||
private static final DateTimeFormatter SIMPLE_UTC_DATE_TIME = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'", Locale.US).withZone(ZoneOffset.UTC);
|
||||
|
||||
@Nonnull
|
||||
private final String domain;
|
||||
|
||||
@Nonnull
|
||||
private final String email;
|
||||
|
||||
private final int maxSizeBytes;
|
||||
|
||||
@Nonnull
|
||||
private final String pathPrefix;
|
||||
|
||||
public CanonicalRequestGenerator(@Nonnull String domain, @Nonnull String email, int maxSizeBytes, @Nonnull String pathPrefix) {
|
||||
this.domain = domain;
|
||||
this.email = email;
|
||||
this.maxSizeBytes = maxSizeBytes;
|
||||
this.pathPrefix = pathPrefix;
|
||||
}
|
||||
|
||||
public CanonicalRequest createFor(@Nonnull final String key, @Nonnull final ZonedDateTime now) {
|
||||
final StringBuilder result = new StringBuilder("POST\n");
|
||||
|
||||
final StringBuilder resourcePathBuilder = new StringBuilder();
|
||||
if (!Strings.isNullOrEmpty(pathPrefix)) {
|
||||
resourcePathBuilder.append(pathPrefix);
|
||||
}
|
||||
resourcePathBuilder.append('/').append(URLEncoder.encode(key, StandardCharsets.UTF_8));
|
||||
final String resourcePath = resourcePathBuilder.toString();
|
||||
result.append(resourcePath).append('\n');
|
||||
|
||||
final String activeDatetime = SIMPLE_UTC_DATE_TIME.format(now);
|
||||
final String canonicalQuery = "X-Goog-Algorithm=GOOG4-RSA-SHA256" +
|
||||
"&X-Goog-Credential=" + URLEncoder.encode(makeCredential(email, now), StandardCharsets.UTF_8) +
|
||||
"&X-Goog-Date=" + URLEncoder.encode(activeDatetime, StandardCharsets.UTF_8) +
|
||||
"&X-Goog-Expires=" + Duration.of(25, ChronoUnit.HOURS).toSeconds() +
|
||||
"&X-Goog-SignedHeaders=host%3Bx-goog-content-length-range%3Bx-goog-resumable";
|
||||
result.append(canonicalQuery).append('\n');
|
||||
|
||||
result.append("host:").append(domain).append('\n');
|
||||
result.append("x-goog-content-length-range:1,").append(maxSizeBytes).append('\n');
|
||||
result.append("x-goog-resumable:start\n");
|
||||
result.append('\n');
|
||||
|
||||
result.append("host;x-goog-content-length-range;x-goog-resumable\n");
|
||||
|
||||
result.append("UNSIGNED-PAYLOAD");
|
||||
|
||||
return new CanonicalRequest(result.toString(), resourcePath, canonicalQuery, activeDatetime, makeCredentialScope(now), domain, maxSizeBytes);
|
||||
}
|
||||
|
||||
private String makeCredentialScope(@Nonnull ZonedDateTime now) {
|
||||
return SIMPLE_UTC_DATE.format(now) + "/auto/storage/goog4_request";
|
||||
}
|
||||
|
||||
private String makeCredential(@Nonnull String email, @Nonnull ZonedDateTime now) {
|
||||
return email + '/' + makeCredentialScope(now);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package org.whispersystems.textsecuregcm.gcp;
|
||||
|
||||
import org.apache.commons.codec.binary.Hex;
|
||||
import org.bouncycastle.openssl.PEMReader;
|
||||
|
||||
import javax.annotation.Nonnull;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.InvalidKeyException;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.PrivateKey;
|
||||
import java.security.Signature;
|
||||
import java.security.SignatureException;
|
||||
|
||||
public class CanonicalRequestSigner {
|
||||
|
||||
@Nonnull
|
||||
private final PrivateKey rsaSigningKey;
|
||||
|
||||
public CanonicalRequestSigner(@Nonnull String rsaSigningKey) throws IOException, InvalidKeyException {
|
||||
this.rsaSigningKey = initializeRsaSigningKey(rsaSigningKey);
|
||||
}
|
||||
|
||||
public String sign(@Nonnull CanonicalRequest canonicalRequest) {
|
||||
return sign(makeStringToSign(canonicalRequest));
|
||||
}
|
||||
|
||||
private String makeStringToSign(@Nonnull final CanonicalRequest canonicalRequest) {
|
||||
final StringBuilder result = new StringBuilder("GOOG4-RSA-SHA256\n");
|
||||
|
||||
result.append(canonicalRequest.getActiveDatetime()).append('\n');
|
||||
|
||||
result.append(canonicalRequest.getCredentialScope()).append('\n');
|
||||
|
||||
final MessageDigest sha256;
|
||||
try {
|
||||
sha256 = MessageDigest.getInstance("SHA-256");
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
sha256.update(canonicalRequest.getCanonicalRequest().getBytes(StandardCharsets.UTF_8));
|
||||
result.append(Hex.encodeHex(sha256.digest()));
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
private String sign(@Nonnull String stringToSign) {
|
||||
final byte[] signature;
|
||||
try {
|
||||
final Signature sha256rsa = Signature.getInstance("SHA256WITHRSA");
|
||||
sha256rsa.initSign(rsaSigningKey);
|
||||
sha256rsa.update(stringToSign.getBytes(StandardCharsets.UTF_8));
|
||||
signature = sha256rsa.sign();
|
||||
} catch (NoSuchAlgorithmException | InvalidKeyException | SignatureException e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
return Hex.encodeHexString(signature);
|
||||
}
|
||||
|
||||
private static PrivateKey initializeRsaSigningKey(String rsaSigningKey) throws IOException, InvalidKeyException {
|
||||
final PEMReader pemReader = new PEMReader(new StringReader(rsaSigningKey));
|
||||
final PrivateKey key = (PrivateKey) pemReader.readObject();
|
||||
testKeyIsValidForSigning(key);
|
||||
return key;
|
||||
}
|
||||
|
||||
private static void testKeyIsValidForSigning(PrivateKey key) throws InvalidKeyException {
|
||||
final Signature sha256rsa;
|
||||
try {
|
||||
sha256rsa = Signature.getInstance("SHA256WITHRSA");
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
sha256rsa.initSign(key);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
package org.whispersystems.textsecuregcm.s3;
|
||||
|
||||
import com.amazonaws.util.Base16Lower;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
|
|
|||
|
|
@ -1,14 +1,26 @@
|
|||
package org.whispersystems.textsecuregcm.tests.controllers;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
|
||||
import io.dropwizard.testing.junit.ResourceTestRule;
|
||||
import org.assertj.core.api.Assertions;
|
||||
import org.assertj.core.api.Condition;
|
||||
import org.assertj.core.api.InstanceOfAssertFactories;
|
||||
import org.bouncycastle.jce.provider.BouncyCastleProvider;
|
||||
import org.bouncycastle.openssl.PEMWriter;
|
||||
import org.bouncycastle.openssl.PKCS8Generator;
|
||||
import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAccount;
|
||||
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV1;
|
||||
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV2;
|
||||
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV3;
|
||||
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV1;
|
||||
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV2;
|
||||
import org.whispersystems.textsecuregcm.entities.AttachmentDescriptorV3;
|
||||
import org.whispersystems.textsecuregcm.entities.AttachmentUri;
|
||||
import org.whispersystems.textsecuregcm.limits.RateLimiter;
|
||||
import org.whispersystems.textsecuregcm.limits.RateLimiters;
|
||||
|
|
@ -19,10 +31,20 @@ import org.whispersystems.textsecuregcm.util.SystemMapper;
|
|||
|
||||
import javax.ws.rs.core.Response;
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.InvalidKeyException;
|
||||
import java.security.KeyPair;
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.Provider;
|
||||
import java.security.Security;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
|
||||
import io.dropwizard.testing.junit.ResourceTestRule;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
|
@ -36,15 +58,122 @@ public class AttachmentControllerTest {
|
|||
when(rateLimiters.getAttachmentLimiter()).thenReturn(rateLimiter);
|
||||
}
|
||||
|
||||
public static final String RSA_PRIVATE_KEY_PEM;
|
||||
|
||||
static {
|
||||
try {
|
||||
final Provider provider = new BouncyCastleProvider();
|
||||
final KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA", provider);
|
||||
keyPairGenerator.initialize(1024);
|
||||
final KeyPair keyPair = keyPairGenerator.generateKeyPair();
|
||||
final StringWriter stringWriter = new StringWriter();
|
||||
final PEMWriter pemWriter = new PEMWriter(stringWriter);
|
||||
final PKCS8Generator pkcs8Generator = new PKCS8Generator(keyPair.getPrivate());
|
||||
pemWriter.writeObject(pkcs8Generator);
|
||||
pemWriter.close();
|
||||
RSA_PRIVATE_KEY_PEM = stringWriter.toString();
|
||||
} catch (NoSuchAlgorithmException | IOException e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
}
|
||||
|
||||
@ClassRule
|
||||
public static final ResourceTestRule resources = ResourceTestRule.builder()
|
||||
.addProvider(AuthHelper.getAuthFilter())
|
||||
.addProvider(new PolymorphicAuthValueFactoryProvider.Binder<>(ImmutableSet.of(Account.class, DisabledPermittedAccount.class)))
|
||||
.setMapper(SystemMapper.getMapper())
|
||||
.setTestContainerFactory(new GrizzlyWebTestContainerFactory())
|
||||
.addResource(new AttachmentControllerV1(rateLimiters, "accessKey", "accessSecret", "attachment-bucket"))
|
||||
.addResource(new AttachmentControllerV2(rateLimiters, "accessKey", "accessSecret", "us-east-1", "attachmentv2-bucket"))
|
||||
.build();
|
||||
public static final ResourceTestRule resources;
|
||||
|
||||
static {
|
||||
try {
|
||||
Security.insertProviderAt(new BouncyCastleProvider(), 0);
|
||||
resources = ResourceTestRule.builder()
|
||||
.addProvider(AuthHelper.getAuthFilter())
|
||||
.addProvider(new PolymorphicAuthValueFactoryProvider.Binder<>(ImmutableSet.of(Account.class, DisabledPermittedAccount.class)))
|
||||
.setMapper(SystemMapper.getMapper())
|
||||
.setTestContainerFactory(new GrizzlyWebTestContainerFactory())
|
||||
.addResource(new AttachmentControllerV1(rateLimiters, "accessKey", "accessSecret", "attachment-bucket"))
|
||||
.addResource(new AttachmentControllerV2(rateLimiters, "accessKey", "accessSecret", "us-east-1", "attachmentv2-bucket"))
|
||||
.addResource(new AttachmentControllerV3(rateLimiters, "some-cdn.signal.org", "signal@example.com", 1000, "/attach-here", RSA_PRIVATE_KEY_PEM))
|
||||
.build();
|
||||
Security.removeProvider(BouncyCastleProvider.PROVIDER_NAME);
|
||||
} catch (IOException | InvalidKeyException e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void setup() {
|
||||
Security.insertProviderAt(new BouncyCastleProvider(), 0);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() {
|
||||
Security.removeProvider(BouncyCastleProvider.PROVIDER_NAME);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV3Form() {
|
||||
AttachmentDescriptorV3 descriptor = resources.getJerseyTest()
|
||||
.target("/v3/attachments/form/upload")
|
||||
.request()
|
||||
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.VALID_NUMBER, AuthHelper.VALID_PASSWORD))
|
||||
.get(AttachmentDescriptorV3.class);
|
||||
|
||||
assertThat(descriptor.getKey()).isNotBlank();
|
||||
assertThat(descriptor.getCdn()).isEqualTo(2);
|
||||
assertThat(descriptor.getHeaders()).hasSize(3);
|
||||
assertThat(descriptor.getHeaders()).extractingByKey("host").isEqualTo("some-cdn.signal.org");
|
||||
assertThat(descriptor.getHeaders()).extractingByKey("x-goog-resumable").isEqualTo("start");
|
||||
assertThat(descriptor.getHeaders()).extractingByKey("x-goog-content-length-range").isEqualTo("1,1000");
|
||||
assertThat(descriptor.getSignedUploadLocation()).isNotEmpty();
|
||||
assertThat(descriptor.getSignedUploadLocation()).contains("X-Goog-Signature");
|
||||
assertThat(descriptor.getSignedUploadLocation()).is(new Condition<>(x -> {
|
||||
try {
|
||||
new URL(x);
|
||||
} catch (MalformedURLException e) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}, "convertible to a URL", (Object[]) null));
|
||||
|
||||
final URL signedUploadLocation;
|
||||
try {
|
||||
signedUploadLocation = new URL(descriptor.getSignedUploadLocation());
|
||||
} catch (MalformedURLException e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
assertThat(signedUploadLocation.getHost()).isEqualTo("some-cdn.signal.org");
|
||||
assertThat(signedUploadLocation.getPath()).startsWith("/attach-here/");
|
||||
final Map<String, String> queryParamMap = new HashMap<>();
|
||||
final String[] queryTerms = signedUploadLocation.getQuery().split("&");
|
||||
for (final String queryTerm : queryTerms) {
|
||||
final String[] keyValueArray = queryTerm.split("=", 2);
|
||||
queryParamMap.put(
|
||||
URLDecoder.decode(keyValueArray[0], StandardCharsets.UTF_8),
|
||||
URLDecoder.decode(keyValueArray[1], StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
assertThat(queryParamMap).extractingByKey("X-Goog-Algorithm").isEqualTo("GOOG4-RSA-SHA256");
|
||||
assertThat(queryParamMap).extractingByKey("X-Goog-Expires").isEqualTo("90000");
|
||||
assertThat(queryParamMap).extractingByKey("X-Goog-SignedHeaders").isEqualTo("host;x-goog-content-length-range;x-goog-resumable");
|
||||
assertThat(queryParamMap).extractingByKey("X-Goog-Date", Assertions.as(InstanceOfAssertFactories.STRING)).isNotEmpty();
|
||||
|
||||
final String credential = queryParamMap.get("X-Goog-Credential");
|
||||
String[] credentialParts = credential.split("/");
|
||||
assertThat(credentialParts).hasSize(5);
|
||||
assertThat(credentialParts[0]).isEqualTo("signal@example.com");
|
||||
assertThat(credentialParts[2]).isEqualTo("auto");
|
||||
assertThat(credentialParts[3]).isEqualTo("storage");
|
||||
assertThat(credentialParts[4]).isEqualTo("goog4_request");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV3FormDisabled() {
|
||||
Response response = resources.getJerseyTest()
|
||||
.target("/v3/attachments/form/upload")
|
||||
.request()
|
||||
.header("Authorization", AuthHelper.getAuthHeader(AuthHelper.DISABLED_NUMBER, AuthHelper.DISABLED_PASSWORD))
|
||||
.get();
|
||||
|
||||
assertThat(response.getStatus()).isEqualTo(401);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV2Form() throws IOException {
|
||||
|
|
|
|||
Loading…
Reference in New Issue