Skip to content

Commit 1a548c0

Browse files
AA-TurnerAA-Turner
authored
gh-132983: Reduce the size of _zstdmodule.h (#133793)
1 parent 4f2f780 commit 1a548c0

File tree

10 files changed

+165
-164
lines changed

10 files changed

+165
-164
lines changed

‎Makefile.pre.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3346,7 +3346,7 @@ MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_tes
33463346
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
33473347
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h
33483348
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
3349-
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h
3349+
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
33503350

33513351
CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h
33523352
MODULE__CODECS_CN_DEPS=$(srcdir)/Modules/cjkcodecs/mappings_cn.h $(CODECS_COMMON_HEADERS)

‎Modules/_zstd/_zstdmodule.c‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,13 @@ Python module.
77
# define Py_BUILD_CORE_MODULE 1
88
#endif
99

10+
#include "Python.h"
11+
1012
#include "_zstdmodule.h"
13+
#include "zstddict.h"
14+
15+
#include <zstd.h> // ZSTD_*()
16+
#include <zdict.h> // ZDICT_*()
1117

1218
/*[clinic input]
1319
module _zstd
@@ -727,7 +733,7 @@ static struct PyModuleDef_Slot _zstd_slots[] = {
727733
{0, NULL},
728734
};
729735

730-
struct PyModuleDef _zstdmodule = {
736+
static struct PyModuleDef _zstdmodule = {
731737
.m_base = PyModuleDef_HEAD_INIT,
732738
.m_name = "_zstd",
733739
.m_doc = "Implementation module for Zstandard compression.",

‎Modules/_zstd/_zstdmodule.h‎

Lines changed: 8 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -1,132 +1,29 @@
1-
#pragma once
21
/*
32
Low level interface to Meta's zstd library for use in the compression.zstd
43
Python module.
54
*/
65

76
/* Declarations shared between different parts of the _zstd module*/
87

9-
#include "Python.h"
10-
11-
#include "zstd.h"
12-
#include "zdict.h"
13-
14-
15-
/* Forward declaration of module state */
16-
typedef struct _zstd_state _zstd_state;
17-
18-
/* Forward reference of module def */
19-
extern PyModuleDef _zstdmodule;
20-
21-
/* For clinic type calculations */
22-
static inline _zstd_state *
23-
get_zstd_state_from_type(PyTypeObject *type)
24-
{
25-
PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule);
26-
if (module == NULL) {
27-
return NULL;
28-
}
29-
void *state = PyModule_GetState(module);
30-
assert(state != NULL);
31-
return (_zstd_state *)state;
32-
}
8+
#ifndef ZSTD_MODULE_H
9+
#define ZSTD_MODULE_H
3310

11+
/* Type specs */
3412
extern PyType_Spec zstd_dict_type_spec;
3513
extern PyType_Spec zstd_compressor_type_spec;
3614
extern PyType_Spec zstd_decompressor_type_spec;
3715

38-
struct _zstd_state {
16+
typedef struct {
17+
/* Module heap types. */
3918
PyTypeObject *ZstdDict_type;
4019
PyTypeObject *ZstdCompressor_type;
4120
PyTypeObject *ZstdDecompressor_type;
4221
PyObject *ZstdError;
4322

23+
/* enum types set by set_parameter_types. */
4424
PyTypeObject *CParameter_type;
4525
PyTypeObject *DParameter_type;
46-
};
47-
48-
typedef struct {
49-
PyObject_HEAD
50-
51-
/* Reusable compress/decompress dictionary, they are created once and
52-
can be shared by multiple threads concurrently, since its usage is
53-
read-only.
54-
c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */
55-
ZSTD_DDict *d_dict;
56-
PyObject *c_dicts;
57-
58-
/* Content of the dictionary, bytes object. */
59-
PyObject *dict_content;
60-
/* Dictionary id */
61-
uint32_t dict_id;
62-
63-
/* __init__ has been called, 0 or 1. */
64-
int inited;
65-
} ZstdDict;
66-
67-
typedef struct {
68-
PyObject_HEAD
69-
70-
/* Compression context */
71-
ZSTD_CCtx *cctx;
72-
73-
/* ZstdDict object in use */
74-
PyObject *dict;
75-
76-
/* Last mode, initialized to ZSTD_e_end */
77-
int last_mode;
78-
79-
/* (nbWorker >= 1) ? 1 : 0 */
80-
int use_multithread;
81-
82-
/* Compression level */
83-
int compression_level;
84-
85-
/* __init__ has been called, 0 or 1. */
86-
int inited;
87-
} ZstdCompressor;
88-
89-
typedef struct {
90-
PyObject_HEAD
91-
92-
/* Decompression context */
93-
ZSTD_DCtx *dctx;
94-
95-
/* ZstdDict object in use */
96-
PyObject *dict;
97-
98-
/* Unconsumed input data */
99-
char *input_buffer;
100-
size_t input_buffer_size;
101-
size_t in_begin, in_end;
102-
103-
/* Unused data */
104-
PyObject *unused_data;
105-
106-
/* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
107-
char needs_input;
108-
109-
/* For decompress(), 0 or 1.
110-
1 when both input and output streams are at a frame edge, means a
111-
frame is completely decoded and fully flushed, or the decompressor
112-
just be initialized. */
113-
char at_frame_edge;
114-
115-
/* For ZstdDecompressor, 0 or 1.
116-
1 means the end of the first frame has been reached. */
117-
char eof;
118-
119-
/* Used for fast reset above three variables */
120-
char _unused_char_for_align;
121-
122-
/* __init__ has been called, 0 or 1. */
123-
int inited;
124-
} ZstdDecompressor;
125-
126-
typedef enum {
127-
TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class
128-
TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function
129-
} decompress_type;
26+
} _zstd_state;
13027

13128
typedef enum {
13229
ERR_DECOMPRESS,
@@ -149,12 +46,6 @@ typedef enum {
14946
DICT_TYPE_PREFIX = 2
15047
} dictionary_type;
15148

152-
static inline int
153-
mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
154-
{
155-
return in->size == in->pos && out->size != out->pos;
156-
}
157-
15849
/* Format error message and set ZstdError. */
15950
extern void
16051
set_zstd_error(const _zstd_state* const state,
@@ -164,4 +55,4 @@ extern void
16455
set_parameter_error(const _zstd_state* const state, int is_compress,
16556
int key_v, int value_v);
16657

167-
static const char init_twice_msg[] = "__init__ method is called twice.";
58+
#endif // !ZSTD_MODULE_H

‎Modules/_zstd/buffer.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,13 @@ Low level interface to Meta's zstd library for use in the compression.zstd
33
Python module.
44
*/
55

6-
#include "_zstdmodule.h"
6+
#ifndef ZSTD_BUFFER_H
7+
#define ZSTD_BUFFER_H
8+
79
#include "pycore_blocks_output_buffer.h"
810

11+
#include <zstd.h> // ZSTD_outBuffer
12+
913
/* Blocks output buffer wrapper code */
1014

1115
/* Initialize the buffer, and grow the buffer.
@@ -102,3 +106,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob)
102106

103107
return buffer->allocated == buffer->max_length;
104108
}
109+
110+
#endif // !ZSTD_BUFFER_H

‎Modules/_zstd/compressor.c‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,50 @@ Python module.
77

88
/*[clinic input]
99
module _zstd
10-
class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type"
10+
class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec"
1111
[clinic start generated code]*/
12-
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/
13-
12+
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/
1413

1514
#ifndef Py_BUILD_CORE_BUILTIN
1615
# define Py_BUILD_CORE_MODULE 1
1716
#endif
1817

19-
#include "_zstdmodule.h"
18+
#include "Python.h"
2019

20+
#include "_zstdmodule.h"
2121
#include "buffer.h"
22+
#include "zstddict.h"
2223

24+
#include <stdbool.h> // bool
2325
#include <stddef.h> // offsetof()
26+
#include <zstd.h> // ZSTD_*()
27+
28+
typedef struct {
29+
PyObject_HEAD
30+
31+
/* Compression context */
32+
ZSTD_CCtx *cctx;
33+
34+
/* ZstdDict object in use */
35+
PyObject *dict;
36+
37+
/* Last mode, initialized to ZSTD_e_end */
38+
int last_mode;
39+
40+
/* (nbWorker >= 1) ? 1 : 0 */
41+
int use_multithread;
2442

43+
/* Compression level */
44+
int compression_level;
45+
46+
/* __init__ has been called, 0 or 1. */
47+
bool initialized;
48+
} ZstdCompressor;
2549

2650
#define ZstdCompressor_CAST(op) ((ZstdCompressor *)op)
2751

52+
#include "clinic/compressor.c.h"
53+
2854
static int
2955
_zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
3056
const char *arg_name, const char* arg_type)
@@ -292,10 +318,6 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict)
292318
return 0;
293319
}
294320

295-
#define clinic_state() (get_zstd_state_from_type(type))
296-
#include "clinic/compressor.c.h"
297-
#undef clinic_state
298-
299321
static PyObject *
300322
_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs))
301323
{
@@ -305,7 +327,7 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject
305327
goto error;
306328
}
307329

308-
self->inited = 0;
330+
self->initialized = 0;
309331
self->dict = NULL;
310332
self->use_multithread = 0;
311333

@@ -372,12 +394,11 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level,
372394
PyObject *options, PyObject *zstd_dict)
373395
/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/
374396
{
375-
/* Only called once */
376-
if (self->inited) {
377-
PyErr_SetString(PyExc_RuntimeError, init_twice_msg);
397+
if (self->initialized) {
398+
PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
378399
return -1;
379400
}
380-
self->inited = 1;
401+
self->initialized = 1;
381402

382403
if (level != Py_None && options != Py_None) {
383404
PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
@@ -488,6 +509,12 @@ compress_impl(ZstdCompressor *self, Py_buffer *data,
488509
return NULL;
489510
}
490511

512+
static inline int
513+
mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
514+
{
515+
return in->size == in->pos && out->size != out->pos;
516+
}
517+
491518
static PyObject *
492519
compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
493520
{

0 commit comments

Comments
 (0)