Skip to content

gh-144777: Fix data races in IncrementalNewlineDecoder #144971

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
colesbury wants to merge 1 commit into
base: main
Choose a base branch
from
+83 −7
Open

gh-144777: Fix data races in IncrementalNewlineDecoder #144971

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions Lib/test/test_free_threading/test_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
import codecs
import io
import _pyio as pyio
import threading
from unittest import TestCase
from test.support import threading_helper
from test.support.threading_helper import run_concurrently
from random import randint
from sys import getsizeof

threading_helper.requires_working_threading(module=True)


class ThreadSafetyMixin:
# Test pretty much everything that can break under free-threading.
Expand Down Expand Up @@ -115,3 +119,54 @@ class CBytesIOTest(ThreadSafetyMixin, TestCase):

class PyBytesIOTest(ThreadSafetyMixin, TestCase):
ioclass = pyio.BytesIO


class IncrementalNewlineDecoderTest(TestCase):
def make_decoder(self):
utf8_decoder = codecs.getincrementaldecoder('utf-8')()
return io.IncrementalNewlineDecoder(utf8_decoder, translate=True)

def test_concurrent_reset(self):
decoder = self.make_decoder()

def worker():
for _ in range(100):
decoder.reset()

run_concurrently(worker_func=worker, nthreads=2)

def test_concurrent_decode(self):
decoder = self.make_decoder()

def worker():
for _ in range(100):
decoder.decode(b"line\r\n", final=False)

run_concurrently(worker_func=worker, nthreads=2)

def test_concurrent_getstate_setstate(self):
decoder = self.make_decoder()
state = decoder.getstate()

def getstate_worker():
for _ in range(100):
decoder.getstate()

def setstate_worker():
for _ in range(100):
decoder.setstate(state)

run_concurrently([getstate_worker] * 2 + [setstate_worker] * 2)

def test_concurrent_decode_and_reset(self):
decoder = self.make_decoder()

def decode_worker():
for _ in range(100):
decoder.decode(b"line\r\n", final=False)

def reset_worker():
for _ in range(100):
decoder.reset()

run_concurrently([decode_worker] * 2 + [reset_worker] * 2)
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Library/2026-02-18-13-45-00.gh-issue-144777.R97q0a.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix data races in :class:`io.IncrementalNewlineDecoder` in the :term:`free-threaded build`.
22 changes: 19 additions & 3 deletions Modules/_io/clinic/textio.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 8 additions & 4 deletions Modules/_io/textio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself,
}

/*[clinic input]
@critical_section
_io.IncrementalNewlineDecoder.decode
input: object
final: bool = False
Expand All @@ -527,18 +528,19 @@ _io.IncrementalNewlineDecoder.decode
static PyObject *
_io_IncrementalNewlineDecoder_decode_impl(nldecoder_object *self,
PyObject *input, int final)
/*[clinic end generated code: output=0d486755bb37a66e input=90e223c70322c5cd]*/
/*[clinic end generated code: output=0d486755bb37a66e input=9475d16a73168504]*/
{
return _PyIncrementalNewlineDecoder_decode((PyObject *) self, input, final);
}

/*[clinic input]
@critical_section
_io.IncrementalNewlineDecoder.getstate
[clinic start generated code]*/

static PyObject *
_io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self)
/*[clinic end generated code: output=f0d2c9c136f4e0d0 input=f8ff101825e32e7f]*/
/*[clinic end generated code: output=f0d2c9c136f4e0d0 input=dc3e1f27aa850f12]*/
{
PyObject *buffer;
unsigned long long flag;
Expand Down Expand Up @@ -576,6 +578,7 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self)
}

/*[clinic input]
@critical_section
_io.IncrementalNewlineDecoder.setstate
state: object
/
Expand All @@ -584,7 +587,7 @@ _io.IncrementalNewlineDecoder.setstate
static PyObject *
_io_IncrementalNewlineDecoder_setstate_impl(nldecoder_object *self,
PyObject *state)
/*[clinic end generated code: output=09135cb6e78a1dc8 input=c53fb505a76dbbe2]*/
/*[clinic end generated code: output=09135cb6e78a1dc8 input=275fd3982d2b08cb]*/
{
PyObject *buffer;
unsigned long long flag;
Expand Down Expand Up @@ -614,12 +617,13 @@ _io_IncrementalNewlineDecoder_setstate_impl(nldecoder_object *self,
}

/*[clinic input]
@critical_section
_io.IncrementalNewlineDecoder.reset
[clinic start generated code]*/

static PyObject *
_io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self)
/*[clinic end generated code: output=32fa40c7462aa8ff input=728678ddaea776df]*/
/*[clinic end generated code: output=32fa40c7462aa8ff input=31bd8ae4e36cec83]*/
{
CHECK_INITIALIZED_DECODER(self);

Expand Down
Loading