1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
/**
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0.
*/
#ifndef AWS_S3_PARALLEL_INPUT_STREAM_H
#define AWS_S3_PARALLEL_INPUT_STREAM_H
#include <aws/s3/s3.h>
#include <aws/common/ref_count.h>
AWS_PUSH_SANE_WARNING_LEVEL
struct aws_byte_buf;
struct aws_future_bool;
struct aws_future_void;
struct aws_input_stream;
struct aws_s3_meta_request;
struct aws_s3_request;
struct aws_event_loop_group;
/**
* This should be private, but keep it public for providing your own implementation.
*/
struct aws_parallel_input_stream {
const struct aws_parallel_input_stream_vtable *vtable;
struct aws_allocator *alloc;
struct aws_ref_count ref_count;
struct aws_future_void *shutdown_future;
void *impl;
};
struct aws_parallel_input_stream_vtable {
/**
* Destroy the stream, its refcount has reached 0.
*/
void (*destroy)(struct aws_parallel_input_stream *stream);
/**
* Read from the offset until fill the dest, or EOF reached.
* It's thread safe to be called from multiple threads without waiting for other read to complete
*
* @param stream The stream to read from
* @param offset The offset in the stream from beginning to start reading
* @param max_length The maximum number of bytes to read
* @param dest The output buffer read to
* @return a future, which will contain an error code if something went wrong,
* or a result bool indicating whether EOF has been reached.
*/
struct aws_future_bool *(
*read)(struct aws_parallel_input_stream *stream, uint64_t offset, size_t max_length, struct aws_byte_buf *dest);
/**
* Get the length of the stream.
*
* @param stream The stream to get length from
* @param out_length The output length
* @return AWS_OP_SUCCESS if success, otherwise AWS_OP_ERR
*/
int (*get_length)(struct aws_parallel_input_stream *stream, int64_t *out_length);
};
AWS_EXTERN_C_BEGIN
/**
* Initialize aws_parallel_input_stream "base class"
*/
AWS_S3_API
void aws_parallel_input_stream_init_base(
struct aws_parallel_input_stream *stream,
struct aws_allocator *alloc,
const struct aws_parallel_input_stream_vtable *vtable,
void *impl);
/**
* Increment reference count.
* You may pass in NULL (has no effect).
* Returns whatever pointer was passed in.
*/
AWS_S3_API
struct aws_parallel_input_stream *aws_parallel_input_stream_acquire(struct aws_parallel_input_stream *stream);
/**
* Decrement reference count.
* You may pass in NULL (has no effect).
* Always returns NULL.
*/
AWS_S3_API
struct aws_parallel_input_stream *aws_parallel_input_stream_release(struct aws_parallel_input_stream *stream);
/**
* Read from the offset until fill the dest, or EOF reached.
* It's thread safe to be called from multiple threads without waiting for other read to complete
*
* @param stream The stream to read from
* @param offset The offset in the stream from beginning to start reading
* @param max_length The maximum number of bytes to read
* @param dest The output buffer read to
* @return a future, which will contain an error code if something went wrong,
* or a result bool indicating whether EOF has been reached.
*/
AWS_S3_API
struct aws_future_bool *aws_parallel_input_stream_read(
struct aws_parallel_input_stream *stream,
uint64_t offset,
size_t max_length,
struct aws_byte_buf *dest);
/**
* Get the total length of the parallel input stream.
*
* @param stream
* @param out_length
* @return AWS_S3_API
*/
AWS_S3_API
int aws_parallel_input_stream_get_length(struct aws_parallel_input_stream *stream, int64_t *out_length);
/**
* Creates a new parallel input stream that reads from a file.
* This stream uses an event loop group to perform file I/O operations asynchronously.
*
* Notes for direct_io_read:
* - checking `aws_file_path_read_from_offset_direct_io` for detail
* - For `AWS_ERROR_UNSUPPORTED_OPERATION`, fallback to reading with cache with warnings, instead of fail.
* - If alignment required, it's callers' responsibility to align with the page size.
*
* @param allocator The allocator to use for memory allocation
* @param file_name The name of the file to read from
* @param reading_elg The event loop group to use for file I/O operations
* @param direct_io_read Whether to use direct I/O for reading the file.
*
* @return A new parallel input stream that reads from the specified file
*/
AWS_S3_API
struct aws_parallel_input_stream *aws_parallel_input_stream_new_from_file(
struct aws_allocator *allocator,
struct aws_byte_cursor file_name,
struct aws_event_loop_group *reading_elg,
bool direct_io_read);
/**
* Get the shutdown future from the parallel input stream.
* The future will be completed when every refcount on the stream has been released.
* And all the resource has been released.
* Don't hold any refcount of the stream while waiting on the future, otherwise, deadlock can happen.
* You need to release the future after using it.
*/
AWS_S3_API
struct aws_future_void *aws_parallel_input_stream_get_shutdown_future(struct aws_parallel_input_stream *stream);
AWS_EXTERN_C_END
AWS_POP_SANE_WARNING_LEVEL
#endif /* AWS_S3_PARALLEL_INPUT_STREAM_H */
|
