File: s3_checksums.h

package info (click to toggle)
aws-crt-python 0.28.4%2Bdfsg-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 78,428 kB
  • sloc: ansic: 437,955; python: 27,657; makefile: 5,855; sh: 4,289; ruby: 208; java: 82; perl: 73; cpp: 25; xml: 11
file content (227 lines) | stat: -rw-r--r-- 9,271 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
#ifndef AWS_S3_CHECKSUMS_H
#define AWS_S3_CHECKSUMS_H
/**
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */
#include "aws/s3/s3_client.h"

/* TODO: consider moving the aws_checksum_stream to aws-c-checksum, and the rest about checksum headers and trailer to
 * aws-c-sdkutil. */

struct aws_s3_checksum;
struct aws_s3_upload_request_checksum_context;

/* List to check the checksum algorithm to use based on the priority. */
static const enum aws_s3_checksum_algorithm s_checksum_algo_priority_list[] = {
    AWS_SCA_CRC64NVME,
    AWS_SCA_CRC32C,
    AWS_SCA_CRC32,
    AWS_SCA_SHA1,
    AWS_SCA_SHA256,
};
AWS_STATIC_ASSERT(AWS_ARRAY_SIZE(s_checksum_algo_priority_list) == (AWS_SCA_END - AWS_SCA_INIT + 1));

struct aws_checksum_vtable {
    void (*destroy)(struct aws_s3_checksum *checksum);
    int (*update)(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf);
    int (*finalize)(struct aws_s3_checksum *checksum, struct aws_byte_buf *out);
};

struct aws_s3_checksum {
    struct aws_allocator *allocator;
    struct aws_checksum_vtable *vtable;
    size_t digest_size;
    enum aws_s3_checksum_algorithm algorithm;
    bool good;
    union {
        struct aws_hash *hash;
        uint32_t crc_val_32bit;
        uint64_t crc_val_64bit;
    } impl;
};

struct aws_s3_meta_request_checksum_config_storage {
    struct aws_allocator *allocator;
    struct aws_byte_buf full_object_checksum;
    bool has_full_object_checksum;

    aws_s3_meta_request_full_object_checksum_fn *full_object_checksum_callback;
    void *user_data;

    enum aws_s3_checksum_location location;
    enum aws_s3_checksum_algorithm checksum_algorithm;
    bool validate_response_checksum;
    struct {
        bool crc64nvme;
        bool crc32c;
        bool crc32;
        bool sha1;
        bool sha256;
    } response_checksum_algorithms;
};

/**
 * Helper stream that takes in a stream and the checksum context to help finalize the checksum from the underlying
 * stream.
 * The context will be only finalized when the checksum stream has read to the end of stream.
 *
 * Note: seek this stream will immediately fail, as it would prevent an accurate calculation of the
 * checksum.
 *
 * @param allocator
 * @param existing_stream The real content to read from. Destroying the checksum stream destroys the existing stream.
 *                        outputs the checksum of existing stream to checksum_output upon destruction. Will be kept
 *                        alive by the checksum stream
 * @param context         Checksum context to keep and get checksum requirements from.
 */
AWS_S3_API
struct aws_input_stream *aws_checksum_stream_new_with_context(
    struct aws_allocator *allocator,
    struct aws_input_stream *existing_stream,
    struct aws_s3_upload_request_checksum_context *context);

/**
 * Helper stream that takes in a stream to keep track of the checksum of the underlying stream during read.
 * Invoke `aws_checksum_stream_finalize_checksum` to get the checksum of the data has been read so far.
 *
 * Note: seek this stream will immediately fail, as it would prevent an accurate calculation of the
 * checksum.
 *
 * @param allocator
 * @param existing_stream The real content to read from. Destroying the checksum stream destroys the existing stream.
 *                        outputs the checksum of existing stream to checksum_output upon destruction. Will be kept
 *                        alive by the checksum stream
 * @param algorithm       Checksum algorithm to use.
 */
AWS_S3_API
struct aws_input_stream *aws_checksum_stream_new(
    struct aws_allocator *allocator,
    struct aws_input_stream *existing_stream,
    enum aws_s3_checksum_algorithm algorithm);

/**
 * Finalize the checksum has read so far to the output checksum buf with base64 encoding.
 * Not thread safe.
 */
AWS_S3_API
int aws_checksum_stream_finalize_checksum(struct aws_input_stream *checksum_stream, struct aws_byte_buf *checksum_buf);

/**
 * TODO: properly support chunked encoding.
 * Creates a chunked encoding stream that wraps an existing stream and adds checksum trailers.
 *
 * This function creates a stream that:
 * 1. Encodes the input stream wraps the existing_stream with aws-chunked encoded.
 * 2. Calculates a checksum of the stream content (if not already calculated)
 * 3. Appends the checksum as a trailer at the end of the aws-chunked stream
 *
 * Note: This stream does not support seeking operations, as seeking would prevent
 * accurate checksum calculation and corrupt the chunked encoding format.
 *
 * @param allocator         Memory allocator to use for stream creation and internal buffers
 * @param existing_stream   The input stream to be chunked and checksummed. This stream
 *                          will be acquired by the chunk stream and released when the
 *                          chunk stream is destroyed. Must not be NULL.
 * @param checksum_context  Context containing checksum configuration and state. Must not be NULL.
 *                          The context contains:
 *                          - algorithm: The checksum algorithm to use (CRC32, CRC32C, etc.)
 *                          - base64_checksum: Buffer for the calculated checksum result
 *                          - checksum_calculated: Whether checksum is pre-calculated or needs calculation
 *                          - encoded_checksum_size: Expected size of the base64-encoded checksum
 *
 *                          If checksum_calculated is false, the stream will wrap existing_stream
 *                          with a checksum stream to calculate the checksum during reading.
 *                          If checksum_calculated is true, the existing checksum value will be used.
 *
 * @return A new input stream that provides chunked encoding with checksum trailers,
 *         or NULL if creation fails. The returned stream must be released with
 *         aws_input_stream_release() when no longer needed.
 *
 * @note The total length of the returned stream includes:
 *       - Chunk size header (hex representation + \r\n)
 *       - Original stream content
 *       - Final chunk marker (0\r\n or \r\n0\r\n)
 *       - Checksum trailer (header name + : + base64 checksum + \r\n\r\n)
 */
AWS_S3_API
struct aws_input_stream *aws_chunk_stream_new(
    struct aws_allocator *allocator,
    struct aws_input_stream *existing_stream,
    struct aws_s3_upload_request_checksum_context *context);

/**
 * Get the size of the checksum output corresponding to the aws_s3_checksum_algorithm enum value.
 */
AWS_S3_API
size_t aws_get_digest_size_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm);

/**
 * Get header name to use for algorithm (e.g. "x-amz-checksum-crc32")
 */
AWS_S3_API
struct aws_byte_cursor aws_get_http_header_name_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm);

/**
 * Get algorithm's name (e.g. "CRC32"), to be used as the value of headers like `x-amz-checksum-algorithm`
 */
AWS_S3_API
struct aws_byte_cursor aws_get_checksum_algorithm_name(enum aws_s3_checksum_algorithm algorithm);

/**
 * Get the name of checksum algorithm to be used as the details of the parts were uploaded. Referring to
 * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompletedPart.html#AmazonS3-Type-CompletedPart
 */
AWS_S3_API
struct aws_byte_cursor aws_get_completed_part_name_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm);

/**
 * create a new aws_checksum corresponding to the aws_s3_checksum_algorithm enum value.
 */
AWS_S3_API
struct aws_s3_checksum *aws_checksum_new(struct aws_allocator *allocator, enum aws_s3_checksum_algorithm algorithm);

/**
 * Compute an aws_checksum corresponding to the provided enum, passing a function pointer around instead of using a
 * conditional would be faster, but would be a negligible improvement compared to the cost of processing data twice
 * which would be the only time this function would be used, and would be harder to follow.
 */
AWS_S3_API
int aws_checksum_compute(
    struct aws_allocator *allocator,
    enum aws_s3_checksum_algorithm algorithm,
    const struct aws_byte_cursor *input,
    struct aws_byte_buf *output);

/**
 * Cleans up and deallocates checksum.
 */
AWS_S3_API
void aws_checksum_destroy(struct aws_s3_checksum *checksum);

/**
 * Updates the running checksum with to_checksum. this can be called multiple times.
 */
AWS_S3_API
int aws_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *to_checksum);

/**
 * Completes the checksum computation and writes the final digest to output.
 * Allocation of output is the caller's responsibility.
 */
AWS_S3_API
int aws_checksum_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output);

AWS_S3_API
int aws_s3_meta_request_checksum_config_storage_init(
    struct aws_allocator *allocator,
    struct aws_s3_meta_request_checksum_config_storage *internal_config,
    const struct aws_s3_checksum_config *config,
    const struct aws_http_message *message,
    const void *log_id);

AWS_S3_API
void aws_s3_meta_request_checksum_config_storage_cleanup(
    struct aws_s3_meta_request_checksum_config_storage *internal_config);

#endif /* AWS_S3_CHECKSUMS_H */