Add copy endpoint to ArchiveController

Co-authored-by: Jonathan Klabunde Tomer <[email protected]> Co-authored-by: Chris Eager <[email protected]>
signalapp · Nov 28, 2023 · 202dd8e · 202dd8e
1 parent 1da3f96
commit 202dd8e
Show file tree

Hide file tree

Showing 24 changed files with 1,918 additions and 248 deletions.
diff --git a/service/config/sample.yml b/service/config/sample.yml
@@ -87,6 +87,8 @@ dynamoDbTables:
     usernamesTableName: Example_Accounts_Usernames
   backups:
     tableName: Example_Backups
+  backupMedia:
+    tableName: Example_BackupMedia
   clientReleases:
     tableName: Example_ClientReleases
   deletedAccounts:
@@ -219,6 +221,33 @@ cdn:
   bucket: cdn        # S3 Bucket name
   region: us-west-2  # AWS region
 
+clientCdn:
+  attachmentUrls:
+    2: https://cdn2.example.com/attachments/
+  caCertificates:
+    - |
+      -----BEGIN CERTIFICATE-----
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
+      AAAAAAAAAAAAAAAAAAAA
+      -----END CERTIFICATE-----
+
 dogstatsd:
   environment: dev
 

diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerConfiguration.java
@@ -24,6 +24,7 @@
 import org.whispersystems.textsecuregcm.configuration.BadgesConfiguration;
 import org.whispersystems.textsecuregcm.configuration.BraintreeConfiguration;
 import org.whispersystems.textsecuregcm.configuration.CdnConfiguration;
+import org.whispersystems.textsecuregcm.configuration.ClientCdnConfiguration;
 import org.whispersystems.textsecuregcm.configuration.ClientReleaseConfiguration;
 import org.whispersystems.textsecuregcm.configuration.CommandStopListenerConfiguration;
 import org.whispersystems.textsecuregcm.configuration.DogstatsdConfiguration;
@@ -101,6 +102,11 @@ public class WhisperServerConfiguration extends Configuration {
   @JsonProperty
   private CdnConfiguration cdn;
 
+  @NotNull
+  @Valid
+  @JsonProperty
+  private ClientCdnConfiguration clientCdn;
+
   @NotNull
   @Valid
   @JsonProperty
@@ -405,6 +411,10 @@ public CdnConfiguration getCdnConfiguration() {
     return cdn;
   }
 
+  public ClientCdnConfiguration getClientCdn() {
+    return clientCdn;
+  }
+
   public DogstatsdConfiguration getDatadogConfiguration() {
     return dogstatsd;
   }

diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
@@ -79,7 +79,9 @@
 import org.whispersystems.textsecuregcm.auth.grpc.BasicCredentialAuthenticationInterceptor;
 import org.whispersystems.textsecuregcm.backup.BackupAuthManager;
 import org.whispersystems.textsecuregcm.backup.BackupManager;
-import org.whispersystems.textsecuregcm.backup.TusBackupCredentialGenerator;
+import org.whispersystems.textsecuregcm.backup.BackupsDb;
+import org.whispersystems.textsecuregcm.backup.Cdn3BackupCredentialGenerator;
+import org.whispersystems.textsecuregcm.backup.Cdn3RemoteStorageManager;
 import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter;
 import org.whispersystems.textsecuregcm.badges.ResourceBundleLevelTranslator;
 import org.whispersystems.textsecuregcm.captcha.CaptchaChecker;
@@ -420,6 +422,8 @@ public void run(WhisperServerConfiguration config, Environment environment) thro
         .scheduledExecutorService(name(getClass(), "storageServiceRetry-%d")).threads(1).build();
     ScheduledExecutorService hcaptchaRetryExecutor = environment.lifecycle()
         .scheduledExecutorService(name(getClass(), "hCaptchaRetry-%d")).threads(1).build();
+    ScheduledExecutorService remoteStorageExecutor = environment.lifecycle()
+        .scheduledExecutorService(name(getClass(), "remoteStorageRetry-%d")).threads(1).build();
 
     Scheduler messageDeliveryScheduler = Schedulers.fromExecutorService(
         ExecutorServiceMetrics.monitor(Metrics.globalRegistry,
@@ -650,10 +654,23 @@ public void run(WhisperServerConfiguration config, Environment environment) thro
     ServerZkAuthOperations zkAuthOperations = new ServerZkAuthOperations(zkSecretParams);
     ServerZkReceiptOperations zkReceiptOperations = new ServerZkReceiptOperations(zkSecretParams);
 
-    TusBackupCredentialGenerator tusBackupCredentialGenerator = new TusBackupCredentialGenerator(config.getTus());
+    Cdn3BackupCredentialGenerator cdn3BackupCredentialGenerator = new Cdn3BackupCredentialGenerator(config.getTus());
     BackupAuthManager backupAuthManager = new BackupAuthManager(dynamicConfigurationManager, rateLimiters, accountsManager, backupsGenericZkSecretParams, clock);
-    BackupManager backupManager = new BackupManager(backupsGenericZkSecretParams, tusBackupCredentialGenerator, dynamoDbAsyncClient,
+    BackupsDb backupsDb = new BackupsDb(
+        dynamoDbAsyncClient,
         config.getDynamoDbTables().getBackups().getTableName(),
+        config.getDynamoDbTables().getBackupMedia().getTableName(),
+        clock);
+    BackupManager backupManager = new BackupManager(
+        backupsDb,
+        backupsGenericZkSecretParams,
+        cdn3BackupCredentialGenerator,
+        new Cdn3RemoteStorageManager(
+            remoteStorageExecutor,
+            config.getClientCdn().getCircuitBreaker(),
+            config.getClientCdn().getRetry(),
+            config.getClientCdn().getCaCertificates()),
+        config.getClientCdn().getAttachmentUrls(),
         clock);
 
     AuthFilter<BasicCredentials, AuthenticatedAccount> accountAuthFilter = new BasicCredentialAuthFilter.Builder<AuthenticatedAccount>().setAuthenticator(