Single File Upload/Download

cmake/sdksCommon.cmake

@@ -108,6 +108,7 @@ list(APPEND SDK_TEST_PROJECT_LIST "redshift:tests/aws-cpp-sdk-redshift-integrati
 list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-unit-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3-crt:tests/aws-cpp-sdk-s3-crt-integration-tests")
+list(APPEND SDK_TEST_PROJECT_LIST "s3-transfer:tests/aws-cpp-sdk-s3-transfer-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3-encryption:tests/aws-cpp-sdk-s3-encryption-tests,tests/aws-cpp-sdk-s3-encryption-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3control:tests/aws-cpp-sdk-s3control-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "sns:tests/aws-cpp-sdk-sns-integration-tests")

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadDataReceiver.h

@@ -0,0 +1,26 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+#pragma once
+#include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/s3-transfer/S3DownloadBuffer.h>
+
+namespace Aws {
+namespace S3 {
+namespace Transfer {
+
+/**
+ * Callback interface for zero-copy downloads. The transfer manager delivers each part of the
+ * object to OnDataReceived as it arrives, in object order. Keep a copy of the buffer to retain
+ * the bytes past this call.
+ */
+class AWS_S3_TRANSFER_API DownloadDataReceiver {
+ public:
+  virtual ~DownloadDataReceiver() = default;
+  virtual void OnDataReceived(S3DownloadBuffer buffer) = 0;
+};
+
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadHandle.h

@@ -1,42 +1,41 @@
 /**
-* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
 #include <future>
 #include <memory>
 #include <aws/s3-transfer/DownloadResponse.h>
+#include <aws/core/utils/memory/AWSMemory.h>
 
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
-/**
- * Returned from S3TransferManager::Download to represent a single in-flight download. The
- * handle is move-only and owns the underlying transfer state.
- */
+class DownloadHandleImpl;
+
+// Move-only handle for a single in-flight download.
 class AWS_S3_TRANSFER_API DownloadHandle final {
 public:
-  DownloadHandle();
+  explicit DownloadHandle(Aws::UniquePtr<DownloadHandleImpl> impl);
   ~DownloadHandle();
+  DownloadHandle(const DownloadHandle&) = delete;
+  DownloadHandle& operator=(const DownloadHandle&) = delete;
   DownloadHandle(DownloadHandle&&) noexcept;
   DownloadHandle& operator=(DownloadHandle&&) noexcept;
 
-  /**
-   * Returns a future that resolves once the transfer finishes, succeeds, or fails.
-   */
+  // Resolves once the transfer finishes, succeeds, or fails.
   std::future<DownloadOutcome> CompletionFuture();
 
-  /**
-   * Requests cancellation of the in-flight download. Returns immediately; the future
-   * returned by CompletionFuture will resolve with a failure once the cancel takes effect.
-   */
+  // Returns immediately; the completion future resolves with a failure once the cancel takes effect.
   void Cancel();
-};
 
+private:
+  Aws::UniquePtr<DownloadHandleImpl> m_impl;
+};
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadRequest.h

@@ -4,11 +4,15 @@
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
-#include <aws/s3-transfer/DownloadProgressListener.h>
+#include <aws/s3-transfer/ProgressListener.h>
+#include <aws/s3-transfer/DownloadDataReceiver.h>
+#include <aws/core/utils/DateTime.h>
 #include <aws/core/utils/memory/stl/AWSString.h>
 #include <aws/core/utils/memory/stl/AWSVector.h>
-#include <aws/core/utils/stream/ResponseStream.h>
+#include <aws/s3/model/ChecksumMode.h>
 #include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/RequestPayer.h>
+#include <cassert>
 #include <memory>
 #include <utility>
 
@@ -17,37 +21,102 @@ namespace S3 {
 namespace Transfer {
 
 /**
- * Request type for S3TransferManager::Download. Carries the inner S3 GetObjectRequest along
- * with the local destination (file path or stream factory) and any request-level progress
- * listeners. The transfer manager parallelizes large objects via ranged GETs internally.
+ * Request type for S3TransferManager::Download. Bucket, key, and the download destination (either
+ * a file path or a zero-copy DownloadDataReceiver) are required at construction time; the two
+ * constructors make the destination-kind choice exclusive so an invalid request is
+ * unconstructable. All other S3-level fields (version id, range constraints, SSE, etc.) are
+ * optional and set via the setters below; the customer does not construct a GetObjectRequest.
+ * Reads go through GetS3Request().
  */
 class AWS_S3_TRANSFER_API DownloadRequest final {
  public:
+  // File download.
   explicit DownloadRequest(
-    Aws::S3::Model::GetObjectRequest s3Request,
+    Aws::String bucket,
+    Aws::String key,
     Aws::String destinationFilePath,
-    Aws::IOStreamFactory responseStreamFactory,
     Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {})
-  : m_s3Request(std::move(s3Request)),
-    m_destinationFilePath(std::move(destinationFilePath)),
-    m_responseStreamFactory(std::move(responseStreamFactory)),
-    m_transferListeners(std::move(transferListeners)) {}
+  : m_destinationFilePath(std::move(destinationFilePath)),
+    m_transferListeners(std::move(transferListeners)) {
+    assert(!bucket.empty() && "DownloadRequest bucket must not be empty");
+    assert(!key.empty() && "DownloadRequest key must not be empty");
+    assert(!m_destinationFilePath.empty() && "DownloadRequest destination file path must not be empty");
+    m_s3Request.SetBucket(std::move(bucket));
+    m_s3Request.SetKey(std::move(key));
+  }
+
+  // Zero-copy stream download. Each part is delivered to the receiver in object order.
+  explicit DownloadRequest(
+    Aws::String bucket,
+    Aws::String key,
+    std::shared_ptr<DownloadDataReceiver> dataReceiver,
+    Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {})
+  : m_dataReceiver(std::move(dataReceiver)),
+    m_transferListeners(std::move(transferListeners)) {
+    assert(!bucket.empty() && "DownloadRequest bucket must not be empty");
+    assert(!key.empty() && "DownloadRequest key must not be empty");
+    assert(m_dataReceiver && "DownloadRequest data receiver must not be null");
+    m_s3Request.SetBucket(std::move(bucket));
+    m_s3Request.SetKey(std::move(key));
+  }
 
-  inline const Aws::S3::Model::GetObjectRequest& GetS3Request() const { return m_s3Request; }
   inline const Aws::String& GetDestinationFilePath() const { return m_destinationFilePath; }
-  inline const Aws::IOStreamFactory& GetResponseStreamFactory() const { return m_responseStreamFactory; }
+  inline const std::shared_ptr<DownloadDataReceiver>& GetDownloadDataReceiver() const { return m_dataReceiver; }
   inline const Aws::Vector<std::shared_ptr<DownloadProgressListener>>& GetTransferListeners() const {
     return m_transferListeners;
   }
 
+  // Bucket and key are set at construction time and cannot be changed; see the constructors above.
+  // Read these fields back via GetS3Request().
+  inline DownloadRequest& SetChecksumMode(Aws::S3::Model::ChecksumMode v) { m_s3Request.SetChecksumMode(v); return *this; }
+
+  inline DownloadRequest& SetExpectedBucketOwner(Aws::String v) { m_s3Request.SetExpectedBucketOwner(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetIfMatch(Aws::String v) { m_s3Request.SetIfMatch(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetIfModifiedSince(Aws::Utils::DateTime v) { m_s3Request.SetIfModifiedSince(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetIfNoneMatch(Aws::String v) { m_s3Request.SetIfNoneMatch(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetIfUnmodifiedSince(Aws::Utils::DateTime v) { m_s3Request.SetIfUnmodifiedSince(std::move(v)); return *this; }
+
+  // HTTP Range header (e.g. "bytes=0-999"). Not in the SEP field list but needed for byte-range
+  // downloads.
+  inline DownloadRequest& SetRange(Aws::String v) { m_s3Request.SetRange(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetRequestPayer(Aws::S3::Model::RequestPayer v) { m_s3Request.SetRequestPayer(v); return *this; }
+
+  inline DownloadRequest& SetResponseCacheControl(Aws::String v) { m_s3Request.SetResponseCacheControl(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetResponseContentDisposition(Aws::String v) { m_s3Request.SetResponseContentDisposition(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetResponseContentEncoding(Aws::String v) { m_s3Request.SetResponseContentEncoding(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetResponseContentLanguage(Aws::String v) { m_s3Request.SetResponseContentLanguage(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetResponseContentType(Aws::String v) { m_s3Request.SetResponseContentType(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetResponseExpires(Aws::Utils::DateTime v) { m_s3Request.SetResponseExpires(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetSSECustomerAlgorithm(Aws::String v) { m_s3Request.SetSSECustomerAlgorithm(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetSSECustomerKey(Aws::String v) { m_s3Request.SetSSECustomerKey(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetSSECustomerKeyMD5(Aws::String v) { m_s3Request.SetSSECustomerKeyMD5(std::move(v)); return *this; }
+
+  inline DownloadRequest& SetVersionId(Aws::String v) { m_s3Request.SetVersionId(std::move(v)); return *this; }
+
+  // Read the assembled S3 request. Use this to inspect any field set via the setters above,
+  // or bucket/key set at construction time.
+  inline const Aws::S3::Model::GetObjectRequest& GetS3Request() const { return m_s3Request; }
 
  private:
   Aws::S3::Model::GetObjectRequest m_s3Request;
   Aws::String m_destinationFilePath;
-  Aws::IOStreamFactory m_responseStreamFactory;
+  std::shared_ptr<DownloadDataReceiver> m_dataReceiver;
   Aws::Vector<std::shared_ptr<DownloadProgressListener>> m_transferListeners;
 };
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadResponse.h

@@ -15,32 +15,30 @@ namespace S3 {
 namespace Transfer {
 
 /**
- * Response type returned via the DownloadHandle's future once the transfer completes. Wraps
- * the underlying S3 GetObjectResult with whole-object content length and range, regardless
- * of how many ranged GETs were issued internally.
+ * Response type returned via the DownloadHandle's future once the transfer completes. Wraps the
+ * underlying S3 GetObjectResult with whole-object content length and range. The S3 result is set
+ * at construction time so a DownloadResponse is never in a half-populated state; access it via
+ * GetS3Result().
  */
 class AWS_S3_TRANSFER_API DownloadResponse final {
  public:
+  // Default constructor exists to satisfy Aws::Utils::Outcome<R, E>, which default-constructs
+  // its R on the error path. On the success path the S3 result is always provided via the
+  // constructor below; a default-constructed instance is unreachable through DownloadOutcome
+  // because Outcome::GetResult() is only meaningful when IsSuccess() is true.
+  DownloadResponse() = default;
+
+  explicit DownloadResponse(Aws::S3::Model::GetObjectResult s3Result)
+    : m_s3Result(std::move(s3Result)) {}
+
   inline const Aws::S3::Model::GetObjectResult& GetS3Result() const { return m_s3Result; }
-  inline bool S3ResultHasBeenSet() const { return m_s3ResultHasBeenSet; }
-  template <typename GetObjectResultT = Aws::S3::Model::GetObjectResult>
-  void SetS3Result(GetObjectResultT&& getS3Result) {
-    m_s3ResultHasBeenSet = true;
-    m_s3Result = std::forward<GetObjectResultT>(getS3Result);
-  }
-  template <typename GetObjectResultT = Aws::S3::Model::GetObjectResult>
-  DownloadResponse& WithS3Result(GetObjectResultT&& getS3Result) {
-    SetS3Result(std::forward<GetObjectResultT>(getS3Result));
-    return *this;
-  }
 
  private:
   Aws::S3::Model::GetObjectResult m_s3Result;
-  bool m_s3ResultHasBeenSet = false;
 };
 
 using DownloadOutcome = Aws::Utils::Outcome<DownloadResponse, Aws::Client::AWSError<Aws::S3::S3Errors>>;
 
-}
-}
-}
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/ProgressListener.h

@@ -4,16 +4,20 @@
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/s3-transfer/ProgressSnapshot.h>
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
+class UploadRequest;
+class DownloadRequest;
+
 /**
  * Callback interface for receiving event-driven updates throughout the lifecycle of a transfer.
  * Subclass and override the events of interest; default implementations are empty so unused
  * callbacks can be ignored. Listeners may be registered on the request or on the manager.
- * Specialized via the UploadProgressListener and DownloadProgressListener type aliases.
+ * Specialized via the UploadProgressListener and DownloadProgressListener subclasses below.
  */
 template <typename RequestT, typename SnapshotT>
 class ProgressListener {
@@ -42,6 +46,22 @@ class ProgressListener {
   virtual void OnTransferFailed(const RequestT& /*request*/, const SnapshotT& /*snapshot*/) {}
 };
 
-}
-}
-}
+/**
+ * Callback interface for receiving event-driven updates throughout the lifecycle of an upload.
+ * Subclass and override the events of interest; default implementations are empty so unused
+ * callbacks can be ignored. Listeners may be registered on the request or on the manager.
+ */
+class AWS_S3_TRANSFER_API UploadProgressListener
+    : public ProgressListener<UploadRequest, UploadProgressSnapshot> {};
+
+/**
+ * Callback interface for receiving event-driven updates throughout the lifecycle of a download.
+ * Subclass and override the events of interest; default implementations are empty so unused
+ * callbacks can be ignored. Listeners may be registered on the request or on the manager.
+ */
+class AWS_S3_TRANSFER_API DownloadProgressListener
+    : public ProgressListener<DownloadRequest, DownloadProgressSnapshot> {};
+
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws