Add readPreference option to mongodump and multiPartUploadSize to Amazon S3 sync

doc/config/source/mongodump.json

@@ -10,6 +10,7 @@
     "excludeCollections": "myCollectionToExclude1,myCollectionToExclude2",
     "excludeCollectionsWithPrefix": "myExcludePrefix1,myExcludePrefix2",
     "ipv6": true,
-    "pathToMongodump": "/path/to/custom/bin"
+    "pathToMongodump": "/path/to/custom/bin",
+    "readPreference": "secondary"
   }
 }

doc/config/source/mongodump.xml

@@ -27,6 +27,9 @@
   <!-- optional, default=false -->
   <option name="ipv6" value="true"/>
 
+  <!-- optional, default=none -->
+  <option name="readPreference" value="secondary"/>
+
   <!-- define your custom mongodump executable location -->
   <option name="pathToMongodump" value="/path/to/custom/bin" />
 </source>

doc/config/sync/amazons3.json

@@ -6,6 +6,7 @@
     "bucket": "some-s3-bucket",
     "region": "aws-region",
     "path": "backup",
-    "useMultiPartUpload": false
+    "useMultiPartUpload": false,
+    "multiPartUploadPartSize": 104857600
   }
 }

doc/config/sync/amazons3.xml

@@ -17,4 +17,7 @@
 
   <!-- optional, default false -->
   <option name="useMultiPartUpload" value="true" />
+
+  <!-- optional, multipart upload part size in bytes, default AWS SDK default (5MB) -->
+  <option name="multiPartUploadPartSize" value="104857600" />
 </sync>

src/Backup/Source/Mongodump.php

@@ -107,6 +107,14 @@ class Mongodump extends SimulatorExecutable implements Simulator
      */
     private $excludeCollectionsWithPrefix;
 
+    /**
+     * Read preference
+     * --readPreference <string>
+     *
+     * @var string
+     */
+    private $readPreference;
+
     /**
      * (No PHPDoc)
      *
@@ -121,6 +129,7 @@ public function setup(array $conf = [])
 
         $this->pathToMongodump = Util\Arr::getValue($conf, 'pathToMongodump', '');
         $this->useIPv6         = Util\Str::toBoolean(Util\Arr::getValue($conf, 'ipv6', ''), false);
+        $this->readPreference  = Util\Arr::getValue($conf, 'readPreference', '');
 
         $this->setupValidation();
     }
@@ -213,7 +222,8 @@ protected function createExecutable(Target $target) : Executable
                    ->dumpDatabases($this->databases)
                    ->dumpCollections($this->collections)
                    ->excludeCollections($this->excludeCollections)
-                   ->excludeCollectionsWithPrefix($this->excludeCollectionsWithPrefix);
+                   ->excludeCollectionsWithPrefix($this->excludeCollectionsWithPrefix)
+                   ->useReadPreference($this->readPreference);
         return $executable;
     }
 

src/Backup/Sync/AmazonS3.php

@@ -110,6 +110,13 @@ abstract class AmazonS3 implements Simulator
      */
     protected $signatureVersion;
 
+    /**
+     * Multi part upload part size in bytes
+     *
+     * @var int|null
+     */
+    protected $multiPartUploadSize;
+
     /**
      * Min multi part upload size
      *
@@ -151,8 +158,9 @@ public function setup(array $config)
         $this->path             = Util\Path::replaceDatePlaceholders($pathCleaned, $this->time);
         $this->pathRaw          = $pathCleaned;
         $this->acl              = Util\Arr::getValue($config, 'acl', 'private');
-        $this->multiPartUpload  = Util\Str::toBoolean(Util\Arr::getValue($config, 'useMultiPartUpload'), false);
-        $this->usePathStyle     = Util\Str::toBoolean(Util\Arr::getValue($config, 'usePathStyleEndpoint'), false);
+        $this->multiPartUpload     = Util\Str::toBoolean(Util\Arr::getValue($config, 'useMultiPartUpload'), false);
+        $this->multiPartUploadSize = Util\Arr::getValue($config, 'multiPartUploadPartSize');
+        $this->usePathStyle        = Util\Str::toBoolean(Util\Arr::getValue($config, 'usePathStyleEndpoint'), false);
         $this->endpoint         = Util\Arr::getValue($config, 'endpoint');
         $this->signatureVersion = Util\Arr::getValue($config, 'signatureVersion');
     }

src/Backup/Sync/AmazonS3v3.php

@@ -108,10 +108,14 @@ protected function createClient() : S3Client
      */
     protected function createUploader(Target $target, S3Client $s3) : MultipartUploader
     {
-        return new MultipartUploader($s3, $target->getPathname(), [
+        $options = [
             'bucket' => $this->bucket,
             'key'    => $this->getUploadPath($target),
-        ]);
+        ];
+        if ($this->multiPartUploadSize) {
+            $options['part_size'] = (int) $this->multiPartUploadSize;
+        }
+        return new MultipartUploader($s3, $target->getPathname(), $options);
     }
 
     /**

src/Cli/Executable/Mongodump.php

@@ -107,6 +107,14 @@ class Mongodump extends Abstraction
      */
     private $excludeCollectionsWithPrefix = [];
 
+    /**
+     * Read preference
+     * --readPreference <string>
+     *
+     * @var string
+     */
+    private $readPreference;
+
     /**
      * Constructor.
      *
@@ -230,6 +238,18 @@ public function excludeCollectionsWithPrefix(array $prefixes) : Mongodump
         return $this;
     }
 
+    /**
+     * Set read preference.
+     *
+     * @param  string $readPreference
+     * @return Mongodump
+     */
+    public function useReadPreference(string $readPreference) : Mongodump
+    {
+        $this->readPreference = $readPreference;
+        return $this;
+    }
+
     /**
      * Mongodump CommandLine generator.
      *
@@ -252,6 +272,7 @@ protected function createCommandLine() : CommandLine
         $cmd->addOptionIfNotEmpty('--username', $this->user);
         $cmd->addOptionIfNotEmpty('--password', $this->password);
         $cmd->addOptionIfNotEmpty('--authenticationDatabase', $this->authenticationDatabase);
+        $cmd->addOptionIfNotEmpty('--readPreference', $this->readPreference);
 
         foreach ($this->databases as $db) {
             $cmd->addOption('--db', $db);

tests/phpbu/Backup/Sync/AmazonS3v3Test.php

@@ -202,6 +202,39 @@ public function testSimulate()
         $amazonS3->simulate($targetStub, $resultStub);
     }
 
+    /**
+     * Tests AmazonS3V3::sync with multiPartUploadPartSize
+     */
+    public function testSyncWithMultiPartUploadPartSize()
+    {
+        $target = $this->createTargetMock('foo.txt', 'foo.txt.gz');
+
+        $result = $this->createMock(Result::class);
+        $result->expects($this->once())->method('debug');
+
+        $clientMock = $this->createAWSS3Mock();
+        $clientMock->expects($this->once())->method('doesBucketExist')->willReturn(true);
+
+        $uploaderMock = $this->createAWSS3UploaderMock();
+        $uploaderMock->expects($this->once())->method('upload');
+
+        $aws = $this->createPartialMock(AmazonS3v3::class, ['createClient', 'createUploader']);
+        $aws->method('createClient')->willReturn($clientMock);
+        $aws->method('createUploader')->willReturn($uploaderMock);
+
+        $aws->setup([
+            'key'                      => 'some-key',
+            'secret'                   => 'some-secret',
+            'bucket'                   => 'backup',
+            'region'                   => 'frankfurt',
+            'path'                     => 'backup',
+            'useMultiPartUpload'       => 'true',
+            'multiPartUploadPartSize'  => '104857600'
+        ]);
+
+        $aws->sync($target, $result);
+    }
+
     /**
      * Tests AmazonS3::setUp
      */

tests/phpbu/Cli/Executable/MongodumpTest.php

@@ -168,4 +168,18 @@ public function testUri()
             $mongo->getCommand()
         );
     }
+
+    /**
+     * Tests Mongodump::createCommandLine
+     */
+    public function testReadPreference()
+    {
+        $mongo = new Mongodump(PHPBU_TEST_BIN);
+        $mongo->dumpToDirectory('./dump')->useReadPreference('secondary');
+
+        $this->assertEquals(
+            '"' . PHPBU_TEST_BIN . '/mongodump" --out=\'./dump\' --readPreference=\'secondary\'',
+            $mongo->getCommand()
+        );
+    }
 }