File: pyzstd_build_cffi.py

package info (click to toggle)
python-pyzstd 0.17.0%2Bds-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 1,736 kB
  • sloc: python: 7,544; ansic: 3,938; makefile: 5
file content (222 lines) | stat: -rw-r--r-- 6,264 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
try:
    import cffi
except ImportError:
    # PyPy includes cffi by default
    msg = ('\n    To build the CFFI implementation '
           'of pyzstd module, need cffi module.'
           '\n    On CPython, CFFI implementation '
           'is slower than C implementation.\n')
    print(msg)
    raise

ffibuilder = cffi.FFI()

ffibuilder.cdef("""
#define ZSTD_VERSION_NUMBER ...
#define ZSTD_CONTENTSIZE_UNKNOWN ...
#define ZSTD_CONTENTSIZE_ERROR ...

typedef ... ZSTD_CDict;
typedef ... ZSTD_DDict;
typedef ... ZSTD_CCtx;
typedef ... ZSTD_DCtx;

typedef struct {
    size_t error;
    int lowerBound;
    int upperBound;
} ZSTD_bounds;

typedef enum {
    ZSTD_e_continue,
    ZSTD_e_flush,
    ZSTD_e_end
} ZSTD_EndDirective;

typedef struct ZSTD_inBuffer_s {
  const void* src;
  size_t size;
  size_t pos;
} ZSTD_inBuffer;

typedef struct ZSTD_outBuffer_s {
  void*  dst;
  size_t size;
  size_t pos;
} ZSTD_outBuffer;

typedef enum {
    /* Compression parameters */
    ZSTD_c_compressionLevel,

    /* Advanced compression parameters */
    ZSTD_c_windowLog,
    ZSTD_c_hashLog,
    ZSTD_c_chainLog,
    ZSTD_c_searchLog,
    ZSTD_c_minMatch,
    ZSTD_c_targetLength,
    ZSTD_c_strategy,
    ZSTD_c_targetCBlockSize,

    /* LDM mode parameters */
    ZSTD_c_enableLongDistanceMatching,
    ZSTD_c_ldmHashLog,
    ZSTD_c_ldmMinMatch,
    ZSTD_c_ldmBucketSizeLog,
    ZSTD_c_ldmHashRateLog,

    /* frame parameters */
    ZSTD_c_contentSizeFlag,
    ZSTD_c_checksumFlag,
    ZSTD_c_dictIDFlag,

    /* multi-threading parameters */
    ZSTD_c_nbWorkers,
    ZSTD_c_jobSize,
    ZSTD_c_overlapLog,
} ZSTD_cParameter;

typedef enum {
    ZSTD_d_windowLogMax,
} ZSTD_dParameter;

typedef enum {
    ZSTD_fast,
    ZSTD_dfast,
    ZSTD_greedy,
    ZSTD_lazy,
    ZSTD_lazy2,
    ZSTD_btlazy2,
    ZSTD_btopt,
    ZSTD_btultra,
    ZSTD_btultra2
} ZSTD_strategy;

typedef enum {
    ZSTD_reset_session_only,
    ...
} ZSTD_ResetDirective;

size_t      ZSTD_compressBound(size_t srcSize);
unsigned    ZSTD_isError(size_t code);
const char* ZSTD_getErrorName(size_t code);
int         ZSTD_minCLevel(void);
int         ZSTD_maxCLevel(void);
int         ZSTD_defaultCLevel(void);

unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);

unsigned ZSTD_versionNumber(void);
const char* ZSTD_versionString(void);

ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);

size_t ZSTD_CStreamInSize(void);
size_t ZSTD_CStreamOutSize(void);
size_t ZSTD_DStreamInSize(void);
size_t ZSTD_DStreamOutSize(void);

ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
                             int compressionLevel);
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);

ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize);

unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);

ZSTD_CCtx* ZSTD_createCCtx(void);
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
size_t ZSTD_compressStream2(ZSTD_CCtx* cctx,
                            ZSTD_outBuffer* output,
                            ZSTD_inBuffer* input,
                            ZSTD_EndDirective endOp);

ZSTD_DCtx* ZSTD_createDCtx(void);
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
size_t ZSTD_decompressStream(ZSTD_DCtx* dctx,
                             ZSTD_outBuffer* output,
                             ZSTD_inBuffer* input);

unsigned ZDICT_isError(size_t errorCode);
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
                             const void* samplesBuffer,
                             const size_t* samplesSizes, unsigned nbSamples);


typedef struct {
    int      compressionLevel;
    unsigned notificationLevel;
    unsigned dictID;
} ZDICT_params_t;

size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
                                const void* dictContent, size_t dictContentSize,
                                const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
                                ZDICT_params_t parameters);

extern int pyzstd_static_link;
""")

source = """
#include "zstd.h"
#include "zdict.h"

#if ZSTD_VERSION_NUMBER < 10400
    #error "pyzstd module requires zstd v1.4.0+"
#endif

#if ZSTD_VERSION_NUMBER < 10405
typedef struct {
    int      compressionLevel;
    unsigned notificationLevel;
    unsigned dictID;
} ZDICT_params_t;

size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize,
                                const void* dictContent, size_t dictContentSize,
                                const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
                                ZDICT_params_t parameters)
{
    return 0;
}
#endif

#if ZSTD_VERSION_NUMBER < 10500
int ZSTD_defaultCLevel(void)
{
    return ZSTD_CLEVEL_DEFAULT;
}
#endif

#if ZSTD_VERSION_NUMBER < 10506
typedef enum {
    ZSTD_c_targetCBlockSize=130
} PYZSTD_compat_c_targetCBlockSize;
#endif

#ifdef PYZSTD_STATIC_LINK
int pyzstd_static_link = 1;
#else
int pyzstd_static_link = 0;
#endif
"""

def get_extension(**kwargs):
    ffibuilder.set_source(source=source, **kwargs)
    return ffibuilder.distutils_extension()