From eb249eff00c295569a9e817423bb57ed31945bfa Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 4 Oct 2025 20:55:11 +0900 Subject: [PATCH 1/4] Add flags parameter on mmap.flush --- Doc/library/mmap.rst | 30 ++++++++++++++++++- Lib/test/test_mmap.py | 9 ++++++ ...5-10-04-20-48-02.gh-issue-63016.EC9QN_.rst | 1 + Modules/mmapmodule.c | 22 ++++++++++---- 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index b6ffb5cebc020e..3d5e2c9964e678 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -212,7 +212,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Writable :term:`bytes-like object` is now accepted. - .. method:: flush([offset[, size]]) + .. method:: flush([offset[, size]], *, flags=MS_SYNC) Flushes changes made to the in-memory copy of a file back to disk. Without use of this call there is no guarantee that changes are written back before @@ -221,6 +221,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length whole extent of the mapping is flushed. *offset* must be a multiple of the :const:`PAGESIZE` or :const:`ALLOCATIONGRANULARITY`. + The *flags* parameter specifies the synchronization behavior. + *flags* must be one of the :ref:`MS_* constants ` available + on the system. + + On Windows, the *flags* parameter is ignored. + ``None`` is returned to indicate success. An exception is raised when the call failed. @@ -235,6 +241,9 @@ To map anonymous memory, -1 should be passed as the fileno along with the length specified alone, and the flush operation will extend from *offset* to the end of the mmap. + .. versionchanged:: next + Added *flags* parameter to control synchronization behavior. + .. method:: madvise(option[, start[, length]]) @@ -450,3 +459,22 @@ MAP_* Constants :data:`MAP_TPRO`, :data:`MAP_TRANSLATED_ALLOW_EXECUTE`, and :data:`MAP_UNIX03` constants. +.. _ms-constants: + +MS_* Constants +++++++++++++++ + +.. data:: MS_SYNC + MS_ASYNC + MS_INVALIDATE + + These flags control the synchronization behavior for :meth:`mmap.flush`: + + * :data:`MS_SYNC` - Synchronous flush: writes are scheduled and the call + blocks until they are physically written to storage. + * :data:`MS_ASYNC` - Asynchronous flush: writes are scheduled but the call + returns immediately without waiting for completion. + * :data:`MS_INVALIDATE` - Invalidate cached data: invalidates other mappings + of the same file so they can see the changes. + + .. versionadded:: next diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 0571eed23f72dc..bd79d8c3bf7f74 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1157,6 +1157,15 @@ def test_flush_parameters(self): m.flush(PAGESIZE) m.flush(PAGESIZE, PAGESIZE) + if hasattr(mmap, 'MS_SYNC'): + m.flush(0, PAGESIZE, flags=mmap.MS_SYNC) + if hasattr(mmap, 'MS_ASYNC'): + m.flush(flags=mmap.MS_ASYNC) + if hasattr(mmap, 'MS_INVALIDATE'): + m.flush(PAGESIZE * 2, flags=mmap.MS_INVALIDATE) + if hasattr(mmap, 'MS_ASYNC') and hasattr(mmap, 'MS_INVALIDATE'): + m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) + class LargeMmapTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst new file mode 100644 index 00000000000000..a0aee6ce83a508 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst @@ -0,0 +1 @@ +Add a ``flags`` parameter to :meth:`mmap.mmap.flush` to control synchronization behavior. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 41d117162882d7..3403ce9376aad6 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -930,13 +930,16 @@ mmap_tell_method(PyObject *op, PyObject *Py_UNUSED(ignored)) } static PyObject * -mmap_flush_method(PyObject *op, PyObject *args) +mmap_flush_method(PyObject *op, PyObject *args, PyObject *kwargs) { Py_ssize_t offset = 0; Py_ssize_t size = -1; + int flags = MS_SYNC; mmap_object *self = mmap_object_CAST(op); + static char *kwlist[] = {"offset", "size", "flags", NULL}; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size)) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|nn$i:flush", kwlist, + &offset, &size, &flags)) { return NULL; } if (size == -1) { @@ -957,8 +960,7 @@ mmap_flush_method(PyObject *op, PyObject *args) } Py_RETURN_NONE; #elif defined(UNIX) - /* XXX flags for msync? */ - if (-1 == msync(self->data + offset, size, MS_SYNC)) { + if (-1 == msync(self->data + offset, size, flags)) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -1203,7 +1205,7 @@ static struct PyMethodDef mmap_object_methods[] = { {"close", mmap_close_method, METH_NOARGS}, {"find", mmap_find_method, METH_VARARGS}, {"rfind", mmap_rfind_method, METH_VARARGS}, - {"flush", mmap_flush_method, METH_VARARGS}, + {"flush", _PyCFunction_CAST(mmap_flush_method), METH_VARARGS | METH_KEYWORDS}, #ifdef HAVE_MADVISE {"madvise", mmap_madvise_method, METH_VARARGS}, #endif @@ -2047,6 +2049,16 @@ mmap_exec(PyObject *module) ADD_INT_MACRO(module, ACCESS_WRITE); ADD_INT_MACRO(module, ACCESS_COPY); +#ifdef MS_INVALIDATE + ADD_INT_MACRO(module, MS_INVALIDATE); +#endif +#ifdef MS_ASYNC + ADD_INT_MACRO(module, MS_ASYNC); +#endif +#ifdef MS_SYNC + ADD_INT_MACRO(module, MS_SYNC); +#endif + #ifdef HAVE_MADVISE // Conventional advice values #ifdef MADV_NORMAL From 1824344f2960ccb3b1bfa90d10a24a8a7d6af204 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 4 Oct 2025 21:02:41 +0900 Subject: [PATCH 2/4] Compatible on Windows --- Modules/mmapmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 3403ce9376aad6..9e3ae6d161afe0 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -934,7 +934,7 @@ mmap_flush_method(PyObject *op, PyObject *args, PyObject *kwargs) { Py_ssize_t offset = 0; Py_ssize_t size = -1; - int flags = MS_SYNC; + int flags = 0; mmap_object *self = mmap_object_CAST(op); static char *kwlist[] = {"offset", "size", "flags", NULL}; CHECK_VALID(NULL); @@ -960,6 +960,9 @@ mmap_flush_method(PyObject *op, PyObject *args, PyObject *kwargs) } Py_RETURN_NONE; #elif defined(UNIX) + if (flags == 0) { + flags = MS_SYNC; + } if (-1 == msync(self->data + offset, size, flags)) { PyErr_SetFromErrno(PyExc_OSError); return NULL; From 7215d14cb5e942fe58352bea72e0877291d6edf9 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 11 Oct 2025 21:59:43 +0900 Subject: [PATCH 3/4] Update to use Argument Clinic --- Lib/test/test_mmap.py | 2 +- Modules/clinic/mmapmodule.c.h | 67 +++++++++++++++++++++++++++++------ Modules/mmapmodule.c | 6 ++-- 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 53ba8e919b54ab..9dc628eb55032f 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1170,7 +1170,7 @@ def test_flush_parameters(self): if hasattr(mmap, 'MS_ASYNC'): m.flush(flags=mmap.MS_ASYNC) if hasattr(mmap, 'MS_INVALIDATE'): - m.flush(PAGESIZE * 2, flags=mmap.MS_INVALIDATE) + m.flush(PAGESIZE * 2, mmap.MS_INVALIDATE) if hasattr(mmap, 'MS_ASYNC') and hasattr(mmap, 'MS_INVALIDATE'): m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) diff --git a/Modules/clinic/mmapmodule.c.h b/Modules/clinic/mmapmodule.c.h index f7fc172b3af705..7cc84a092c4b11 100644 --- a/Modules/clinic/mmapmodule.c.h +++ b/Modules/clinic/mmapmodule.c.h @@ -2,6 +2,10 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() @@ -371,29 +375,63 @@ mmap_mmap_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(mmap_mmap_flush__doc__, -"flush($self, offset=0, size=-1, /)\n" +"flush($self, offset=0, size=-1, /, flags=0)\n" "--\n" "\n"); #define MMAP_MMAP_FLUSH_METHODDEF \ - {"flush", _PyCFunction_CAST(mmap_mmap_flush), METH_FASTCALL, mmap_mmap_flush__doc__}, + {"flush", _PyCFunction_CAST(mmap_mmap_flush), METH_FASTCALL|METH_KEYWORDS, mmap_mmap_flush__doc__}, static PyObject * -mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size); +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size, + int flags); static PyObject * -mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(flags), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "flags", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "flush", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; Py_ssize_t offset = 0; Py_ssize_t size = -1; + int flags = 0; - if (!_PyArg_CheckPositional("flush", nargs, 0, 2)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } if (nargs < 1) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; { Py_ssize_t ival = -1; PyObject *iobj = _PyNumber_Index(args[0]); @@ -407,8 +445,9 @@ mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) offset = ival; } if (nargs < 2) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; { Py_ssize_t ival = -1; PyObject *iobj = _PyNumber_Index(args[1]); @@ -421,9 +460,17 @@ mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } size = ival; } -skip_optional: +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_pos; + } + flags = PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: Py_BEGIN_CRITICAL_SECTION(self); - return_value = mmap_mmap_flush_impl((mmap_object *)self, offset, size); + return_value = mmap_mmap_flush_impl((mmap_object *)self, offset, size, flags); Py_END_CRITICAL_SECTION(); exit: @@ -796,4 +843,4 @@ mmap_mmap_madvise(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #ifndef MMAP_MMAP_MADVISE_METHODDEF #define MMAP_MMAP_MADVISE_METHODDEF #endif /* !defined(MMAP_MMAP_MADVISE_METHODDEF) */ -/*[clinic end generated code: output=381f6cf4986ac867 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=15895b6949afa550 input=a9049054013a1b77]*/ diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 798dc4d341dbd0..37ebb32e9b7f08 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1034,12 +1034,14 @@ mmap.mmap.flush offset: Py_ssize_t = 0 size: Py_ssize_t = -1 / + flags: int = 0 [clinic start generated code]*/ static PyObject * -mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size) -/*[clinic end generated code: output=956ced67466149cf input=c50b893bc69520ec]*/ +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size, + int flags) +/*[clinic end generated code: output=4225f4174dc75a53 input=e79195ecf4f0a271]*/ { CHECK_VALID(NULL); if (size == -1) { From 05794b7c2d015adf8070a936ad6d86a206c511fc Mon Sep 17 00:00:00 2001 From: AN Long Date: Sat, 11 Oct 2025 22:01:48 +0900 Subject: [PATCH 4/4] Fix document --- Doc/library/mmap.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 3d5e2c9964e678..ac9269255cc7e4 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -212,7 +212,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Writable :term:`bytes-like object` is now accepted. - .. method:: flush([offset[, size]], *, flags=MS_SYNC) + .. method:: flush([offset[, size]], flags=MS_SYNC) Flushes changes made to the in-memory copy of a file back to disk. Without use of this call there is no guarantee that changes are written back before