S3.h

Go to the documentation of this file.

#pragma once
 
#include <aws/crt/Exports.h>
#include <aws/crt/Optional.h>
#include <aws/crt/Types.h>
#include <aws/crt/auth/Sigv4Signing.h>
#include <aws/crt/http/HttpConnection.h>
#include <aws/crt/http/HttpRequestResponse.h>
#include <aws/crt/io/TlsOptions.h>
#include <aws/crt/io/Uri.h>
#include <aws/crt/s3/S3BufferTicket.h>
 
#include <cstdint>
#include <functional>
#include <future>
#include <memory>
 
struct aws_s3_client;
struct aws_s3_client_config;
struct aws_s3_meta_request;
struct aws_s3_meta_request_options;
struct aws_retry_strategy;
 
namespace Aws
{
    namespace Crt
    {
        namespace Auth
        {
            class ICredentialsProvider;
        }
 
        namespace S3
        {
            // Mapped to their aws-c-s3 counterparts by a switch in S3.cpp.
            enum class S3MetaRequestType
            {
                Default,
                GetObject,
                PutObject,
                CopyObject,
            };

            enum class S3ChecksumLocation
            {
                None,
                Header,
                Trailer,
            };

            enum class S3ChecksumAlgorithm
            {
                None,
                Crc32c,
                Crc32,
                Sha1,
                Sha256,
                Crc64Nvme,
                Sha512,
                XXHash64,
                XXHash3_64,
                XXHash3_128,
            };

            enum class S3TlsMode
            {
                Enabled,
                Disabled,
            };

            enum class S3RecvFileMode
            {
                CreateOrReplace,
                CreateNew,
                CreateOrAppend,
                WriteToPosition,
            };

            enum class S3RetryStrategyType
            {
                Default,
                Standard,
                ExponentialBackoff,
                NoRetry,
            };

            struct AWS_CRT_CPP_API S3RetryStrategyExponentialBackoffOptions
            {
                size_t maxRetries = 0;
                uint32_t scaleFactorMs = 500;
                uint32_t maxBackoffSecs = 20;
            };

            class AWS_CRT_CPP_API S3RetryStrategy final
            {
              public:
                S3RetryStrategy(const S3RetryStrategy &) = delete;
                S3RetryStrategy &operator=(const S3RetryStrategy &) = delete;
                S3RetryStrategy(S3RetryStrategy &&other) noexcept = default;
                S3RetryStrategy &operator=(S3RetryStrategy &&other) noexcept = default;
                ~S3RetryStrategy() noexcept = default;

                static S3RetryStrategy CreateStandard() noexcept;

                static S3RetryStrategy CreateExponentialBackoff(
                    Io::EventLoopGroup &elGroup,
                    const S3RetryStrategyExponentialBackoffOptions &options = {}) noexcept;

                static S3RetryStrategy CreateNoRetry() noexcept;

                explicit operator bool() const noexcept { return m_strategy != nullptr; }

                explicit S3RetryStrategy(aws_retry_strategy *strategy) noexcept;

                aws_retry_strategy *GetUnderlyingHandle() const noexcept { return m_strategy.get(); }
 
              private:
                ScopedResource<struct aws_retry_strategy> m_strategy;
            };
 
            class S3Client;
            class S3MetaRequest;
            class S3MetaRequestOptions;

            struct AWS_CRT_CPP_API S3MetaRequestResult
            {
                int errorCode;

                bool IsSuccess() const noexcept;

                int responseStatus;

                Vector<Http::HttpHeader> errorResponseHeaders;

                ByteCursor errorResponseBody;

                bool didValidateChecksum;

                S3ChecksumAlgorithm validationAlgorithm;
            };

            class AWS_CRT_CPP_API S3ChecksumConfig final
            {
              public:
                S3ChecksumConfig() noexcept;

                S3ChecksumConfig &SetLocation(S3ChecksumLocation location) noexcept
                {
                    m_location = location;
                    return *this;
                }

                S3ChecksumConfig &SetChecksumAlgorithm(S3ChecksumAlgorithm algorithm) noexcept
                {
                    m_algorithm = algorithm;
                    return *this;
                }

                S3ChecksumConfig &SetValidateResponseChecksum(bool validate) noexcept
                {
                    m_validateResponseChecksum = validate;
                    return *this;
                }

                S3ChecksumLocation GetLocation() const noexcept { return m_location; }

                S3ChecksumAlgorithm GetChecksumAlgorithm() const noexcept { return m_algorithm; }

                bool GetValidateResponseChecksum() const noexcept { return m_validateResponseChecksum; }
 
              private:
                S3ChecksumLocation m_location;
                S3ChecksumAlgorithm m_algorithm;
                bool m_validateResponseChecksum;
            };

            class AWS_CRT_CPP_API S3ClientConfig final
            {
              public:
                S3ClientConfig(const S3ClientConfig &) = delete;
                S3ClientConfig(S3ClientConfig &&) = delete;
                S3ClientConfig &operator=(const S3ClientConfig &) = delete;
                S3ClientConfig &operator=(S3ClientConfig &&) = delete;

                explicit S3ClientConfig(
                    const std::shared_ptr<Auth::ICredentialsProvider> &credentialsProvider) noexcept;
 
                // Defined out-of-line in S3.cpp where Impl is a complete type
                // (required to destroy the ScopedResource<Impl> pImpl member).
                ~S3ClientConfig() noexcept;

                S3ClientConfig &SetRegion(const Crt::String &region) noexcept;

                S3ClientConfig &SetThroughputTargetGbps(double gbps) noexcept;

                S3ClientConfig &SetPartSize(uint64_t bytes) noexcept;

                S3ClientConfig &SetMultipartUploadThreshold(uint64_t bytes) noexcept;

                S3ClientConfig &SetMemoryLimit(uint64_t bytes) noexcept;

                S3ClientConfig &SetClientBootstrap(Io::ClientBootstrap &bootstrap) noexcept;

                S3ClientConfig &SetConnectTimeoutMs(uint32_t timeoutMs) noexcept;

                S3ClientConfig &SetReadBackpressure(bool enable, uint64_t initialReadWindow) noexcept;

                S3ClientConfig &SetEnableS3Express(bool enable) noexcept;

                S3ClientConfig &SetTlsMode(S3TlsMode mode) noexcept;

                S3ClientConfig &SetTlsConnectionOptions(const Io::TlsConnectionOptions &options) noexcept;

                S3ClientConfig &SetProxyOptions(const Http::HttpClientConnectionProxyOptions &proxyOptions) noexcept;

                S3ClientConfig &SetTcpKeepAlive(uint16_t keepAliveIntervalSec, uint16_t keepAliveTimeoutSec) noexcept;

                S3ClientConfig &SetConnectionMonitoring(
                    uint64_t minimumThroughputBytesPerSecond,
                    uint32_t allowableThroughputFailureIntervalSeconds) noexcept;

                S3ClientConfig &SetNetworkInterfaceNames(const Vector<String> &networkInterfaceNames) noexcept;

                S3ClientConfig &SetRetryStrategy(
                    S3RetryStrategyType type,
                    const S3RetryStrategyExponentialBackoffOptions &options = {}) noexcept;

                S3ClientConfig &SetRetryStrategyFactory(
                    std::function<S3RetryStrategy(const S3ClientConfig &)> factory) noexcept;

                S3ClientConfig &SetClientShutdownCallback(std::function<void()> callback) noexcept;

                struct aws_s3_client_config *GetUnderlyingHandle() const noexcept;

                String GetRegion() const noexcept { return m_region; }

                uint64_t GetPartSize() const noexcept;

                uint64_t GetMultipartUploadThreshold() const noexcept;

                double GetThroughputTargetGbps() const noexcept;

                S3RetryStrategyType GetRetryStrategyType() const noexcept { return m_retryStrategyType; }

                const S3RetryStrategyExponentialBackoffOptions &GetRetryStrategyOptions() const noexcept
                {
                    return m_retryStrategyOptions;
                }

                const std::function<S3RetryStrategy(const S3ClientConfig &)> &GetRetryStrategyFactory() const noexcept
                {
                    return m_retryStrategyFactory;
                }

                const Vector<String> &GetNetworkInterfaceNames() const noexcept { return m_networkInterfaceNames; }

                const std::function<void()> &GetClientShutdownCallback() const noexcept
                {
                    return m_clientShutdownCallback;
                }

                std::shared_ptr<Auth::ICredentialsProvider> GetCredentialsProvider() const noexcept
                {
                    return m_credentialsProvider;
                }
 
              private:
                // The CRT C-struct storage (aws_s3_client_config plus the proxy,
                // TCP keep-alive, and connection-monitoring option backings) lives
                // by value inside Impl, defined in S3.cpp. This keeps the C headers
                // out of this public header and collapses what were four separate
                // heap allocations into the single Impl allocation.
                struct Impl;
                ScopedResource<Impl> m_impl;
                String m_region;
                Optional<Io::TlsConnectionOptions> m_tlsConnectionOptions;
                Optional<Http::HttpClientConnectionProxyOptions> m_proxyOptions;
                Vector<String> m_networkInterfaceNames;
 
                // Strategy is materialized at S3Client construction; the
                // factory (when set) takes precedence over the enum.
                S3RetryStrategyType m_retryStrategyType;
                S3RetryStrategyExponentialBackoffOptions m_retryStrategyOptions;
                std::function<S3RetryStrategy(const S3ClientConfig &)> m_retryStrategyFactory;
 
                std::function<void()> m_clientShutdownCallback;
                std::shared_ptr<Auth::ICredentialsProvider> m_credentialsProvider;
                // Held by value (no heap): single-owner, scope-bound to this
                // config. Optional carries the "no signing config" state the
                // ctor relies on when no credentials provider is supplied.
                Optional<Auth::AwsSigningConfig> m_signingConfig;
            };

            class AWS_CRT_CPP_API S3MetaRequestOptions
            {
              public:
                // Callbacks return bool (continue/abort) so C++ callers don't have to name the C
                // AWS_OP_SUCCESS sentinel; the binding translates false to an aborted meta request.

                using BodyCallback = std::function<bool(ByteCursor body, uint64_t rangeStart)>;

                using BodyCallbackEx =
                    std::function<bool(ByteCursor body, uint64_t rangeStart, S3BufferTicket &ticket)>;

                using HeadersCallback =
                    std::function<bool(const Vector<Http::HttpHeader> &headers, int responseStatus)>;

                using ProgressCallback = std::function<void(uint64_t bytesTransferred, uint64_t contentLength)>;

                using FinishCallback = std::function<void(const S3MetaRequestResult &result)>;

                using ShutdownCallback = std::function<void()>;
 
                S3MetaRequestOptions(const S3MetaRequestOptions &) = delete;
                S3MetaRequestOptions(S3MetaRequestOptions &&) = delete;
                S3MetaRequestOptions &operator=(const S3MetaRequestOptions &) = delete;
                S3MetaRequestOptions &operator=(S3MetaRequestOptions &&) = delete;
 
                virtual ~S3MetaRequestOptions() noexcept;

                S3MetaRequestOptions &SetSigningConfig(const Auth::AwsSigningConfig &config) noexcept;

                S3MetaRequestOptions &SetSigningConfigFromEndpoint(
                    const String &region,
                    const String &signingRegion,
                    const String &signingName,
                    bool isS3Express,
                    const std::shared_ptr<Auth::ICredentialsProvider> &provider) noexcept;

                S3MetaRequestOptions &SetChecksumConfig(const S3ChecksumConfig &config) noexcept;

                S3MetaRequestOptions &SetEndpoint(const Io::Uri &endpoint) noexcept;

                S3MetaRequestOptions &SetHeadersCallback(HeadersCallback cb) noexcept;

                S3MetaRequestOptions &SetProgressCallback(ProgressCallback cb) noexcept;

                S3MetaRequestOptions &SetFinishCallback(FinishCallback cb) noexcept;

                S3MetaRequestOptions &SetShutdownCallback(ShutdownCallback cb) noexcept;

                S3MetaRequestOptions &SetRecvFileMode(S3RecvFileMode mode) noexcept;

                S3MetaRequestOptions &SetRecvFilePosition(uint64_t position) noexcept;

                S3MetaRequestOptions &SetRecvFileDeleteOnFailure(bool deleteOnFailure) noexcept;

                S3MetaRequestOptions &SetObjectSizeHint(uint64_t bytes) noexcept;

                S3MetaRequestOptions &SetPartSize(uint64_t bytes) noexcept;

                S3MetaRequestOptions &SetMultipartUploadThreshold(uint64_t bytes) noexcept;

                struct aws_s3_meta_request_options *GetUnderlyingHandle() const noexcept;

                const BodyCallback &GetBodyCallback() const noexcept { return m_bodyCb; }
                const BodyCallbackEx &GetBodyCallbackEx() const noexcept { return m_bodyCbEx; }
                const HeadersCallback &GetHeadersCallback() const noexcept { return m_headersCb; }
                const ProgressCallback &GetProgressCallback() const noexcept { return m_progressCb; }
                const FinishCallback &GetFinishCallback() const noexcept { return m_finishCb; }
                const ShutdownCallback &GetShutdownCallback() const noexcept { return m_shutdownCb; }

                int GetLastError() const noexcept { return m_lastError; }
 
              protected:
                S3MetaRequestOptions(
                    S3MetaRequestType type,
                    const std::shared_ptr<Http::HttpRequest> &request) noexcept;
 
                // The CRT C-struct storage (aws_s3_meta_request_options plus the
                // checksum-config backing) lives by value inside Impl, defined in
                // S3.cpp. Keeps the C headers out of this public header and
                // collapses two heap allocations into the single Impl allocation.
                // Protected so subclass ctors can reach it (through m_impl) to
                // populate shape-specific fields directly rather than via a setter.
                struct Impl;
                ScopedResource<Impl> m_impl;
                std::shared_ptr<Http::HttpRequest> m_httpRequest;
                String m_operationName;
                String m_sendFilepath;
                String m_recvFilepath;
                // Endpoint override parsed in place (no heap). aws_uri's internal
                // cursors point into its own buffer, so it must never be moved
                // after parsing - hence a value member parsed directly, not a
                // relocatable wrapper. m_endpointInit tracks whether it holds a
                // parsed URI needing cleanup; m_options->endpoint being non-null
                // is the "set" signal.
                aws_uri m_endpoint;
                bool m_endpointInit;
                // Backs the borrowed pointer the CRT holds via object_size_hint.
                Optional<uint64_t> m_objectSizeHint;
                BodyCallback m_bodyCb;
                BodyCallbackEx m_bodyCbEx;
                HeadersCallback m_headersCb;
                ProgressCallback m_progressCb;
                FinishCallback m_finishCb;
                ShutdownCallback m_shutdownCb;
                // Sticky validation error set by a Set* setter, surfaced at
                // MakeMetaRequest (mirrors the MqttClient builder's LastError()).
                int m_lastError = AWS_ERROR_SUCCESS;
            };

            class AWS_CRT_CPP_API S3GetObjectMetaRequestOptions final : public S3MetaRequestOptions
            {
              public:
                S3GetObjectMetaRequestOptions(const S3GetObjectMetaRequestOptions &) = delete;
                S3GetObjectMetaRequestOptions(S3GetObjectMetaRequestOptions &&) = delete;
                S3GetObjectMetaRequestOptions &operator=(const S3GetObjectMetaRequestOptions &) = delete;
                S3GetObjectMetaRequestOptions &operator=(S3GetObjectMetaRequestOptions &&) = delete;

                static ScopedResource<S3MetaRequestOptions> Create(
                    const std::shared_ptr<Http::HttpRequest> &request,
                    BodyCallback cb) noexcept;

                static ScopedResource<S3MetaRequestOptions> Create(
                    const std::shared_ptr<Http::HttpRequest> &request,
                    BodyCallbackEx cb) noexcept;

                static ScopedResource<S3MetaRequestOptions> Create(
                    const std::shared_ptr<Http::HttpRequest> &request,
                    const Crt::String &recvFilepath) noexcept;

                explicit S3GetObjectMetaRequestOptions(const std::shared_ptr<Http::HttpRequest> &request) noexcept;
            };

            class AWS_CRT_CPP_API S3PutObjectMetaRequestOptions final : public S3MetaRequestOptions
            {
              public:
                S3PutObjectMetaRequestOptions(const S3PutObjectMetaRequestOptions &) = delete;
                S3PutObjectMetaRequestOptions(S3PutObjectMetaRequestOptions &&) = delete;
                S3PutObjectMetaRequestOptions &operator=(const S3PutObjectMetaRequestOptions &) = delete;
                S3PutObjectMetaRequestOptions &operator=(S3PutObjectMetaRequestOptions &&) = delete;

                static ScopedResource<S3MetaRequestOptions> Create(
                    const std::shared_ptr<Http::HttpRequest> &request) noexcept;

                static ScopedResource<S3MetaRequestOptions> Create(
                    const std::shared_ptr<Http::HttpRequest> &request,
                    const Crt::String &sendFilepath) noexcept;

                static ScopedResource<S3MetaRequestOptions> CreateWithAsyncWrites(
                    const std::shared_ptr<Http::HttpRequest> &request) noexcept;

                explicit S3PutObjectMetaRequestOptions(const std::shared_ptr<Http::HttpRequest> &request) noexcept;
            };

            class AWS_CRT_CPP_API S3CopyObjectMetaRequestOptions final : public S3MetaRequestOptions
            {
              public:
                S3CopyObjectMetaRequestOptions(const S3CopyObjectMetaRequestOptions &) = delete;
                S3CopyObjectMetaRequestOptions(S3CopyObjectMetaRequestOptions &&) = delete;
                S3CopyObjectMetaRequestOptions &operator=(const S3CopyObjectMetaRequestOptions &) = delete;
                S3CopyObjectMetaRequestOptions &operator=(S3CopyObjectMetaRequestOptions &&) = delete;

                static ScopedResource<S3MetaRequestOptions> Create(
                    const std::shared_ptr<Http::HttpRequest> &request) noexcept;

                explicit S3CopyObjectMetaRequestOptions(const std::shared_ptr<Http::HttpRequest> &request) noexcept;
            };

            class AWS_CRT_CPP_API S3DefaultObjectMetaRequestOptions final : public S3MetaRequestOptions
            {
              public:
                S3DefaultObjectMetaRequestOptions(const S3DefaultObjectMetaRequestOptions &) = delete;
                S3DefaultObjectMetaRequestOptions(S3DefaultObjectMetaRequestOptions &&) = delete;
                S3DefaultObjectMetaRequestOptions &operator=(const S3DefaultObjectMetaRequestOptions &) = delete;
                S3DefaultObjectMetaRequestOptions &operator=(S3DefaultObjectMetaRequestOptions &&) = delete;

                static ScopedResource<S3MetaRequestOptions> Create(
                    const std::shared_ptr<Http::HttpRequest> &request,
                    const Crt::String &operationName) noexcept;

                S3DefaultObjectMetaRequestOptions(
                    const std::shared_ptr<Http::HttpRequest> &request,
                    const Crt::String &operationName) noexcept;
            };

            class AWS_CRT_CPP_API S3Client final
            {
              public:
                S3Client(const S3Client &) = delete;
                S3Client(S3Client &&) = delete;
                S3Client &operator=(const S3Client &) = delete;
                S3Client &operator=(S3Client &&) = delete;
 
                explicit S3Client(const S3ClientConfig &config) noexcept;
                ~S3Client() noexcept = default;

                std::shared_ptr<S3MetaRequest> MakeMetaRequest(S3MetaRequestOptions &options) noexcept;

                static bool MakeDefaultSigningConfig(
                    Auth::AwsSigningConfig &config,
                    const String &region,
                    const std::shared_ptr<Auth::ICredentialsProvider> &provider) noexcept;

                explicit operator bool() const noexcept { return m_client != nullptr; }

                int LastError() const noexcept;

                String GetRegion() const noexcept { return m_region; }

                uint64_t GetPartSize() const noexcept { return m_partSize; }

                uint64_t GetMultipartUploadThreshold() const noexcept { return m_multipartUploadThreshold; }

                double GetThroughputTargetGbps() const noexcept { return m_throughputTargetGbps; }

                std::shared_ptr<Auth::ICredentialsProvider> GetCredentialsProvider() const noexcept
                {
                    return m_credentialsProvider;
                }
 
              private:
                ScopedResource<struct aws_s3_client> m_client;
                int m_lastError;
                String m_region;
                uint64_t m_partSize = 0;
                uint64_t m_multipartUploadThreshold = 0;
                double m_throughputTargetGbps = 0.0;
                std::shared_ptr<Auth::ICredentialsProvider> m_credentialsProvider;
            };

            class AWS_CRT_CPP_API S3MetaRequest final
            {
              public:
                S3MetaRequest(const S3MetaRequest &) = delete;
                S3MetaRequest(S3MetaRequest &&) = delete;
                S3MetaRequest &operator=(const S3MetaRequest &) = delete;
                S3MetaRequest &operator=(S3MetaRequest &&) = delete;
 
                ~S3MetaRequest() noexcept = default;

                void Cancel() noexcept;

                void IncrementReadWindow(uint64_t bytes) noexcept;

                std::future<int> Write(ByteCursor data, bool eof) noexcept;

                int LastError() const noexcept;

                S3MetaRequest() noexcept;

                void SetUnderlyingHandle(struct aws_s3_meta_request *handle) noexcept;

                void SetLastError(int errorCode) noexcept { m_lastError = errorCode; }
 
              private:
                ScopedResource<struct aws_s3_meta_request> m_metaRequest;
                int m_lastError;
            };
 
        } // namespace S3
    } // namespace Crt
} // namespace Aws