diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index f691b08c9e9..016bfb61b1e 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -839,8 +839,6 @@ def test_send_type_error(self): with self.assertRaises(TypeError): conn.request('POST', 'test', conn) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_chunked(self): expected = chunked_expected sock = FakeSocket(chunked_start + last_chunk + chunked_end) @@ -873,8 +871,6 @@ def test_chunked(self): finally: resp.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_readinto_chunked(self): expected = chunked_expected diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index c9ab155e17a..a02e6cf298f 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1039,8 +1039,6 @@ def test_destructor(self): super().test_destructor(self) class PyIOTest(IOTest): - # TODO: RUSTPYTHON, can't resize b/c of existing exports - @unittest.expectedFailure def test_optional_abilities(self): super().test_optional_abilities() @@ -2630,8 +2628,7 @@ def test_non_text_encoding_codecs_are_rejected(self): with self.assertRaisesRegex(LookupError, "is not a text encoding"): self.TextIOWrapper(b, encoding="hex") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip('TODO: RUSTPYTHON') def test_detach(self): r = self.BytesIO() b = self.BufferedWriter(r) @@ -2693,8 +2690,7 @@ def test_line_buffering(self): t.write("A\rB") self.assertEqual(r.getvalue(), b"XY\nZA\rB") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip('TODO: RUSTPYTHON') def test_reconfigure_line_buffering(self): r = self.BytesIO() b = self.BufferedWriter(r, 1000) @@ -3924,16 +3920,12 @@ class PyTextIOWrapperTest(TextIOWrapperTest): def test_newlines(self): super().test_newlines() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_line_buffering(self): super().test_line_buffering() def test_seeking_too(self): super().test_seeking_too() - # TODO: RUSTPYTHON, can't resize b/c of existing exports - @unittest.expectedFailure def test_bufio_write_through(self): super().test_bufio_write_through() diff --git a/vm/src/buffer.rs b/vm/src/buffer.rs index a91f67d44dd..01ee80d6c33 100644 --- a/vm/src/buffer.rs +++ b/vm/src/buffer.rs @@ -3,12 +3,10 @@ use crate::common::borrow::{BorrowedValue, BorrowedValueMut}; use crate::common::rc::PyRc; use crate::PyThreadingConstraint; -use crate::{PyObjectRef, PyResult, TypeProtocol}; -use crate::{TryFromBorrowedObject, VirtualMachine}; -use std::{borrow::Cow, fmt::Debug, ops::Deref}; +use crate::{PyObjectRef, PyResult, TryFromBorrowedObject, TypeProtocol, VirtualMachine}; +use std::{borrow::Cow, fmt::Debug}; -pub trait PyBuffer: Debug + PyThreadingConstraint { - fn get_options(&self) -> &BufferOptions; +pub trait PyBufferInternal: Debug + PyThreadingConstraint { /// Get the full inner buffer of this memory. You probably want [`as_contiguous()`], as /// `obj_bytes` doesn't take into account the range a memoryview might operate on, among other /// footguns. @@ -18,49 +16,84 @@ pub trait PyBuffer: Debug + PyThreadingConstraint { /// might operate on, among other footguns. fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]>; fn release(&self); + // not included in PyBuffer protocol itself + fn retain(&self); +} + +#[derive(Debug)] +pub struct PyBuffer { + pub obj: PyObjectRef, + pub options: BufferOptions, + pub(crate) internal: PyRc, +} - fn as_contiguous(&self) -> Option> { - if !self.get_options().contiguous { +impl PyBuffer { + pub fn new( + obj: PyObjectRef, + buffer: impl PyBufferInternal + 'static, + options: BufferOptions, + ) -> Self { + buffer.retain(); + Self { + obj, + options, + internal: PyRc::new(buffer), + } + } + pub fn as_contiguous(&self) -> Option> { + if !self.options.contiguous { return None; } - Some(self.obj_bytes()) + Some(self.internal.obj_bytes()) } - fn as_contiguous_mut(&self) -> Option> { - if !self.get_options().contiguous { + pub fn as_contiguous_mut(&self) -> Option> { + if !self.options.contiguous { return None; } - Some(self.obj_bytes_mut()) + Some(self.internal.obj_bytes_mut()) } - fn to_contiguous(&self) -> Vec { - self.obj_bytes().to_vec() + pub fn to_contiguous(&self) -> Vec { + self.internal.obj_bytes().to_vec() + } + + pub fn clone_with_options(&self, options: BufferOptions) -> Self { + self.internal.retain(); + Self { + obj: self.obj.clone(), + options, + internal: self.internal.clone(), + } } } #[derive(Debug, Clone)] pub struct BufferOptions { - pub readonly: bool, + // buf pub len: usize, + pub readonly: bool, pub itemsize: usize, - pub contiguous: bool, pub format: Cow<'static, str>, - // TODO: support multiple dimension array - pub ndim: usize, + pub ndim: usize, // TODO: support multiple dimension array pub shape: Vec, pub strides: Vec, + // suboffsets + + // RustPython fields + pub contiguous: bool, } impl BufferOptions { pub const DEFAULT: Self = BufferOptions { - readonly: true, len: 0, + readonly: true, itemsize: 1, - contiguous: true, format: Cow::Borrowed("B"), ndim: 1, shape: Vec::new(), strides: Vec::new(), + contiguous: true, }; } @@ -70,15 +103,12 @@ impl Default for BufferOptions { } } -#[derive(Debug)] -pub struct PyBufferRef(Box); - -impl TryFromBorrowedObject for PyBufferRef { +impl TryFromBorrowedObject for PyBuffer { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { let obj_cls = obj.class(); for cls in obj_cls.iter_mro() { if let Some(f) = cls.slots.as_buffer.as_ref() { - return f(obj, vm).map(|x| PyBufferRef(x)); + return f(obj, vm); } } Err(vm.new_type_error(format!( @@ -88,76 +118,18 @@ impl TryFromBorrowedObject for PyBufferRef { } } -impl Drop for PyBufferRef { - fn drop(&mut self) { - self.0.release(); - } -} - -impl Deref for PyBufferRef { - type Target = dyn PyBuffer; - - fn deref(&self) -> &Self::Target { - self.0.deref() - } -} - -impl PyBufferRef { - pub fn new(buffer: impl PyBuffer + 'static) -> Self { - Self(Box::new(buffer)) - } - - pub fn into_rcbuf(self) -> RcBuffer { - // move self.0 out of self; PyBufferRef impls Drop so it's tricky - let this = std::mem::ManuallyDrop::new(self); - let buf_box = unsafe { std::ptr::read(&this.0) }; - RcBuffer(buf_box.into()) - } -} - -impl From> for PyBufferRef { - fn from(buffer: Box) -> Self { - PyBufferRef(buffer) - } -} - -#[derive(Debug, Clone)] -pub struct RcBuffer(PyRc); -impl Deref for RcBuffer { - type Target = dyn PyBuffer; - fn deref(&self) -> &Self::Target { - self.0.deref() - } -} - -impl Drop for RcBuffer { +// What we actually want to implement is: +// impl Drop for T where T: PyBufferInternal +// but it is not supported by Rust +impl Drop for PyBuffer { fn drop(&mut self) { - // check if this is the last rc before the inner buffer gets dropped - if let Some(buf) = PyRc::get_mut(&mut self.0) { - buf.release() - } + self.internal.release(); } } -impl PyBuffer for RcBuffer { - fn get_options(&self) -> &BufferOptions { - self.0.get_options() - } - fn obj_bytes(&self) -> BorrowedValue<[u8]> { - self.0.obj_bytes() - } - fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - self.0.obj_bytes_mut() - } - fn release(&self) {} - fn as_contiguous(&self) -> Option> { - self.0.as_contiguous() - } - fn as_contiguous_mut(&self) -> Option> { - self.0.as_contiguous_mut() - } - fn to_contiguous(&self) -> Vec { - self.0.to_contiguous() +impl Clone for PyBuffer { + fn clone(&self) -> Self { + self.clone_with_options(self.options.clone()) } } diff --git a/vm/src/builtins/bytearray.rs b/vm/src/builtins/bytearray.rs index 2c7376ef14f..4b5887a31b0 100644 --- a/vm/src/builtins/bytearray.rs +++ b/vm/src/builtins/bytearray.rs @@ -6,7 +6,7 @@ use super::pystr::PyStrRef; use super::pytype::PyTypeRef; use super::tuple::PyTupleRef; use crate::anystr::{self, AnyStr}; -use crate::buffer::{BufferOptions, PyBuffer, ResizeGuard}; +use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal, ResizeGuard}; use crate::bytesinner::{ bytes_decode, bytes_from_object, value_from_object, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, @@ -670,41 +670,35 @@ impl Comparable for PyByteArray { } impl AsBuffer for PyByteArray { - fn get_buffer(zelf: &PyRef, _vm: &VirtualMachine) -> PyResult> { - zelf.exports.fetch_add(1); - let buf = ByteArrayBuffer { - bytearray: zelf.clone(), - options: BufferOptions { + fn get_buffer(zelf: &PyRef, _vm: &VirtualMachine) -> PyResult { + let buffer = PyBuffer::new( + zelf.as_object().clone(), + zelf.clone(), + BufferOptions { readonly: false, len: zelf.len(), ..Default::default() }, - }; - Ok(Box::new(buf)) + ); + Ok(buffer) } } -#[derive(Debug)] -struct ByteArrayBuffer { - bytearray: PyByteArrayRef, - options: BufferOptions, -} - -impl PyBuffer for ByteArrayBuffer { +impl PyBufferInternal for PyRef { fn obj_bytes(&self) -> BorrowedValue<[u8]> { - self.bytearray.borrow_buf().into() + self.borrow_buf().into() } fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - PyRwLockWriteGuard::map(self.bytearray.inner_mut(), |inner| &mut *inner.elements).into() + PyRwLockWriteGuard::map(self.inner_mut(), |inner| &mut *inner.elements).into() } fn release(&self) { - self.bytearray.exports.fetch_sub(1); + self.exports.fetch_sub(1); } - fn get_options(&self) -> &BufferOptions { - &self.options + fn retain(&self) { + self.exports.fetch_add(1); } } diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index 15a9d3a5e59..f41e55053fa 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -3,7 +3,7 @@ use super::int::PyIntRef; use super::pystr::PyStrRef; use super::pytype::PyTypeRef; use crate::anystr::{self, AnyStr}; -use crate::buffer::{BufferOptions, PyBuffer}; +use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal}; use crate::builtins::tuple::PyTupleRef; use crate::bytesinner::{ bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, @@ -519,27 +519,22 @@ impl PyBytes { } impl AsBuffer for PyBytes { - fn get_buffer(zelf: &PyRef, _vm: &VirtualMachine) -> PyResult> { - let buf = BytesBuffer { - bytes: zelf.clone(), - options: BufferOptions { + fn get_buffer(zelf: &PyRef, _vm: &VirtualMachine) -> PyResult { + let buf = PyBuffer::new( + zelf.as_object().clone(), + zelf.clone(), + BufferOptions { len: zelf.len(), ..Default::default() }, - }; - Ok(Box::new(buf)) + ); + Ok(buf) } } -#[derive(Debug)] -struct BytesBuffer { - bytes: PyBytesRef, - options: BufferOptions, -} - -impl PyBuffer for BytesBuffer { +impl PyBufferInternal for PyRef { fn obj_bytes(&self) -> BorrowedValue<[u8]> { - self.bytes.as_bytes().into() + self.as_bytes().into() } fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { @@ -547,10 +542,7 @@ impl PyBuffer for BytesBuffer { } fn release(&self) {} - - fn get_options(&self) -> &BufferOptions { - &self.options - } + fn retain(&self) {} } impl Hashable for PyBytes { diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index c295e22c1bf..bd50cf28d6f 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -1,13 +1,16 @@ -use crate::buffer::{BufferOptions, PyBuffer, PyBufferRef}; +use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal}; use crate::builtins::bytes::{PyBytes, PyBytesRef}; use crate::builtins::list::{PyList, PyListRef}; use crate::builtins::pystr::{PyStr, PyStrRef}; use crate::builtins::pytype::PyTypeRef; use crate::builtins::slice::PySliceRef; use crate::bytesinner::bytes_to_hex; -use crate::common::borrow::{BorrowedValue, BorrowedValueMut}; -use crate::common::hash::PyHash; -use crate::common::lock::OnceCell; +use crate::common::{ + borrow::{BorrowedValue, BorrowedValueMut}, + hash::PyHash, + lock::OnceCell, + rc::PyRc, +}; use crate::function::{FuncArgs, OptionalArg}; use crate::sliceable::{convert_slice, saturate_range, wrap_index, SequenceIndex}; use crate::slots::{AsBuffer, Comparable, Hashable, PyComparisonOp, SlotConstructor}; @@ -15,14 +18,14 @@ use crate::stdlib::pystruct::_struct::FormatSpec; use crate::utils::Either; use crate::{ IdProtocol, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, PyRef, - PyResult, PyValue, TypeProtocol, + PyResult, PyValue, TryFromBorrowedObject, TryFromObject, TypeProtocol, VirtualMachine, }; -use crate::{TryFromBorrowedObject, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; use num_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; use std::fmt::Debug; +use std::ops::Deref; #[derive(FromArgs)] pub struct PyMemoryViewNewArgs { @@ -33,9 +36,7 @@ pub struct PyMemoryViewNewArgs { #[pyclass(module = false, name = "memoryview")] #[derive(Debug)] pub struct PyMemoryView { - obj: PyObjectRef, - buffer: PyBufferRef, - options: BufferOptions, + buffer: PyBuffer, pub(crate) released: AtomicCell, // start should always less or equal to the stop // start and stop pointing to the memory index not slice index @@ -44,7 +45,6 @@ pub struct PyMemoryView { stop: usize, // step can be negative, means read the memory from stop-1 to start step: isize, - exports: AtomicCell, format_spec: FormatSpec, hash: OnceCell, } @@ -53,84 +53,94 @@ impl SlotConstructor for PyMemoryView { type Args = PyMemoryViewNewArgs; fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let buffer = PyBufferRef::try_from_borrowed_object(vm, &args.object)?; - let zelf = PyMemoryView::from_buffer(args.object, buffer, vm)?; + let buffer = PyBuffer::try_from_borrowed_object(vm, &args.object)?; + let zelf = PyMemoryView::from_buffer(buffer, vm)?; zelf.into_pyresult_with_type(vm, cls) } } -type PyMemoryViewRef = PyRef; - #[pyimpl(with(Hashable, Comparable, AsBuffer, SlotConstructor))] impl PyMemoryView { fn parse_format(format: &str, vm: &VirtualMachine) -> PyResult { FormatSpec::parse(format, vm) } - pub fn from_buffer( - obj: PyObjectRef, - buffer: PyBufferRef, - vm: &VirtualMachine, - ) -> PyResult { + pub fn from_buffer(buffer: PyBuffer, vm: &VirtualMachine) -> PyResult { // when we get a buffer means the buffered object is size locked // so we can assume the buffer's options will never change as long // as memoryview is still alive - let options = buffer.get_options().clone(); - let len = options.len; - let itemsize = options.itemsize; + let options = &buffer.options; + let stop = options.len * options.itemsize; let format_spec = Self::parse_format(&options.format, vm)?; Ok(PyMemoryView { - obj, buffer, - options, released: AtomicCell::new(false), start: 0, - stop: len * itemsize, + stop, step: 1, - exports: AtomicCell::new(0), format_spec, hash: OnceCell::new(), }) } pub fn from_buffer_range( - obj: PyObjectRef, - buffer: PyBufferRef, + mut buffer: PyBuffer, range: std::ops::Range, vm: &VirtualMachine, ) -> PyResult { - let options = buffer.get_options().clone(); - let itemsize = options.itemsize; - let format_spec = Self::parse_format(&options.format, vm)?; + let itemsize = buffer.options.itemsize; + let format_spec = Self::parse_format(&buffer.options.format, vm)?; + buffer.options.len = range.len(); Ok(PyMemoryView { - obj, buffer, - options, released: AtomicCell::new(false), start: range.start * itemsize, stop: range.end * itemsize, step: 1, - exports: AtomicCell::new(0), format_spec, hash: OnceCell::new(), }) } + fn as_contiguous(&self) -> Option> { + if !self.buffer.options.contiguous { + return None; + } + Some(self.obj_bytes()) + } + + fn as_contiguous_mut(&self) -> Option> { + if !self.buffer.options.contiguous { + return None; + } + Some(self.obj_bytes_mut()) + } + pub fn try_bytes(&self, vm: &VirtualMachine, f: F) -> PyResult where F: FnOnce(&[u8]) -> R, { self.try_not_released(vm)?; - self.buffer.as_contiguous().map(|x| f(&*x)).ok_or_else(|| { + self.as_contiguous().map(|x| f(&*x)).ok_or_else(|| { vm.new_type_error("non-contiguous memoryview is not a bytes-like object".to_owned()) }) } #[pymethod] fn release(&self) { + self._release(); + } + + fn _release(&self) { // avoid double release - if self.released.compare_exchange(false, true).is_ok() && self.exports.load() == 0 { - self.buffer.release(); + if self.released.compare_exchange(false, true).is_ok() { + unsafe { + // SAFETY: this branch is only once accessible form _release and guarded by AtomicCell released + let buffer: &std::cell::UnsafeCell = std::mem::transmute(&self.buffer); + let buffer = &mut *buffer.get(); + let internal = std::mem::replace(&mut buffer.internal, PyRc::new(Released)); + internal.release(); + } } } @@ -144,35 +154,37 @@ impl PyMemoryView { #[pyproperty] fn obj(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.obj.clone()) + self.try_not_released(vm).map(|_| self.buffer.obj.clone()) } #[pyproperty] fn nbytes(&self, vm: &VirtualMachine) -> PyResult { self.try_not_released(vm) - .map(|_| self.options.len * self.options.itemsize) + .map(|_| self.buffer.options.len * self.buffer.options.itemsize) } #[pyproperty] fn readonly(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.options.readonly) + self.try_not_released(vm) + .map(|_| self.buffer.options.readonly) } #[pyproperty] fn itemsize(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.options.itemsize) + self.try_not_released(vm) + .map(|_| self.buffer.options.itemsize) } #[pyproperty] fn ndim(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.options.ndim) + self.try_not_released(vm).map(|_| self.buffer.options.ndim) } // TODO #[pyproperty] fn shape(&self, vm: &VirtualMachine) -> PyResult { self.try_not_released(vm) - .map(|_| (self.options.len,).into_pyobject(vm)) + .map(|_| (self.buffer.options.len,).into_pyobject(vm)) } // TODO @@ -184,7 +196,7 @@ impl PyMemoryView { #[pyproperty] fn format(&self, vm: &VirtualMachine) -> PyResult { self.try_not_released(vm) - .map(|_| PyStr::from(&self.options.format)) + .map(|_| PyStr::from(&self.buffer.options.format)) } #[pymethod(magic)] @@ -194,18 +206,18 @@ impl PyMemoryView { #[pymethod(magic)] fn exit(&self, _args: FuncArgs) { - self.release(); + self._release(); } // translate the slice index to memory index fn get_pos(&self, i: isize) -> Option { - let len = self.options.len; - let itemsize = self.options.itemsize; + let len = self.buffer.options.len; + let itemsize = self.buffer.options.itemsize; let i = wrap_index(i, len)?; let i = if self.step < 0 { - self.stop - itemsize - (-self.step as usize) * i * itemsize + (len - 1 + (self.step as usize) * i) * itemsize } else { - self.start + self.step as usize * i * itemsize + self.step as usize * i * itemsize }; Some(i) } @@ -214,15 +226,14 @@ impl PyMemoryView { let i = zelf .get_pos(i) .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; - zelf.format_spec - .unpack(&zelf.obj_bytes()[i..i + zelf.options.itemsize], vm) - .map(|x| { - if x.len() == 1 { - x.fast_getitem(0) - } else { - x.into_object() - } - }) + let bytes = &zelf.obj_bytes()[i..i + zelf.buffer.options.itemsize]; + zelf.format_spec.unpack(bytes, vm).map(|x| { + if x.len() == 1 { + x.fast_getitem(0) + } else { + x.into_object() + } + }) } fn getitem_by_slice(zelf: PyRef, slice: PySliceRef, vm: &VirtualMachine) -> PyResult { @@ -234,13 +245,9 @@ impl PyMemoryView { return Err(vm.new_value_error("slice step cannot be zero".to_owned())); } let newstep: BigInt = step.clone() * zelf.step; - let len = zelf.options.len; - let itemsize = zelf.options.itemsize; + let len = zelf.buffer.options.len; + let itemsize = zelf.buffer.options.itemsize; - let obj = zelf.obj.clone(); - let buffer = PyBufferRef::new(zelf.clone()); - zelf.exports.fetch_add(1); - let options = zelf.options.clone(); let format_spec = zelf.format_spec.clone(); if newstep == BigInt::one() { @@ -254,18 +261,15 @@ impl PyMemoryView { let start = zelf.start + range.start * itemsize; let stop = zelf.start + range.end * itemsize; return Ok(PyMemoryView { - obj, - buffer, - options: BufferOptions { + buffer: zelf.buffer.clone_with_options(BufferOptions { len: newlen, contiguous: true, - ..options - }, + ..zelf.buffer.options.clone() + }), released: AtomicCell::new(false), start, stop, step: 1, - exports: AtomicCell::new(0), format_spec, hash: OnceCell::new(), } @@ -302,18 +306,15 @@ impl PyMemoryView { + 1 } else { return Ok(PyMemoryView { - obj, - buffer, - options: BufferOptions { + buffer: zelf.buffer.clone_with_options(BufferOptions { len: 0, contiguous: true, - ..options - }, + ..zelf.buffer.options.clone() + }), released: AtomicCell::new(false), start: range.end, stop: range.end, step: 1, - exports: AtomicCell::new(0), format_spec, hash: OnceCell::new(), } @@ -333,19 +334,17 @@ impl PyMemoryView { (start, stop) }; + let options = BufferOptions { + len: newlen, + contiguous: false, + ..zelf.buffer.options.clone() + }; Ok(PyMemoryView { - obj, - buffer, - options: BufferOptions { - len: newlen, - contiguous: false, - ..options - }, + buffer: zelf.buffer.clone_with_options(options), released: AtomicCell::new(false), start, stop, step: newstep, - exports: AtomicCell::new(0), format_spec, hash: OnceCell::new(), } @@ -370,7 +369,7 @@ impl PyMemoryView { let i = zelf .get_pos(i) .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; - let itemsize = zelf.options.itemsize; + let itemsize = zelf.buffer.options.itemsize; let data = zelf.format_spec.pack(vec![value], vm)?; zelf.obj_bytes_mut()[i..i + itemsize].copy_from_slice(&data); Ok(()) @@ -382,15 +381,15 @@ impl PyMemoryView { items: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - let items = PyBufferRef::try_from_object(vm, items)?; - let options = items.get_options(); + let items = PyBuffer::try_from_object(vm, items)?; + let options = &items.options; let len = options.len; let itemsize = options.itemsize; - if itemsize != zelf.options.itemsize { + if itemsize != zelf.buffer.options.itemsize { return Err(vm.new_type_error(format!( "memoryview: invalid type for format '{}'", - zelf.options.format + zelf.buffer.options.format ))); } @@ -400,11 +399,11 @@ impl PyMemoryView { )) }; - if options.format != zelf.options.format { + if options.format != zelf.buffer.options.format { return diff_err(); } - let (range, step, is_negative_step) = convert_slice(&slice, zelf.options.len, vm)?; + let (range, step, is_negative_step) = convert_slice(&slice, zelf.buffer.options.len, vm)?; let bytes = items.to_contiguous(); assert_eq!(bytes.len(), len * itemsize); @@ -475,7 +474,7 @@ impl PyMemoryView { vm: &VirtualMachine, ) -> PyResult<()> { zelf.try_not_released(vm)?; - if zelf.options.readonly { + if zelf.buffer.options.readonly { return Err(vm.new_type_error("cannot modify read-only memory".to_owned())); } match needle { @@ -486,7 +485,7 @@ impl PyMemoryView { #[pymethod(magic)] fn len(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.options.len) + self.try_not_released(vm).map(|_| self.buffer.options.len) } fn to_bytes_vec(zelf: &PyRef) -> Vec { @@ -494,8 +493,8 @@ impl PyMemoryView { bytes.to_vec() } else { let bytes = &*zelf.obj_bytes(); - let len = zelf.options.len; - let itemsize = zelf.options.itemsize; + let len = zelf.buffer.options.len; + let itemsize = zelf.buffer.options.itemsize; (0..len) .map(|i| zelf.get_pos(i as isize).unwrap()) .flat_map(|i| bytes[i..i + itemsize].to_vec()) @@ -514,9 +513,15 @@ impl PyMemoryView { zelf.try_not_released(vm)?; let bytes = &*zelf.obj_bytes(); - let elements: Vec = (0..zelf.options.len) + let elements: Vec = (0..zelf.buffer.options.len) .map(|i| zelf.get_pos(i as isize).unwrap()) - .map(|i| format_unpack(&zelf.format_spec, &bytes[i..i + zelf.options.itemsize], vm)) + .map(|i| { + format_unpack( + &zelf.format_spec, + &bytes[i..i + zelf.buffer.options.itemsize], + vm, + ) + }) .try_collect()?; Ok(PyList::from(elements).into_ref(vm)) @@ -525,17 +530,13 @@ impl PyMemoryView { #[pymethod] fn toreadonly(zelf: PyRef, vm: &VirtualMachine) -> PyResult> { zelf.try_not_released(vm)?; - let buffer = PyBufferRef::new(zelf.clone()); - zelf.exports.fetch_add(1); + let buffer = zelf.buffer.clone_with_options(BufferOptions { + readonly: true, + ..zelf.buffer.options.clone() + }); Ok(PyMemoryView { - obj: zelf.obj.clone(), buffer, - options: BufferOptions { - readonly: true, - ..zelf.options.clone() - }, released: AtomicCell::new(false), - exports: AtomicCell::new(0), format_spec: zelf.format_spec.clone(), hash: OnceCell::new(), ..*zelf @@ -568,7 +569,7 @@ impl PyMemoryView { &*guard } None => { - vec = zelf.to_contiguous(); + vec = Self::to_bytes_vec(&zelf); vec.as_slice() } }; @@ -580,7 +581,7 @@ impl PyMemoryView { #[pymethod] fn cast(zelf: PyRef, format: PyStrRef, vm: &VirtualMachine) -> PyResult> { zelf.try_not_released(vm)?; - if !zelf.options.contiguous { + if !zelf.buffer.options.contiguous { return Err(vm.new_type_error( "memoryview: casts are restricted to C-contiguous views".to_owned(), )); @@ -588,7 +589,7 @@ impl PyMemoryView { let format_spec = Self::parse_format(format.as_str(), vm)?; let itemsize = format_spec.size(); - let bytelen = zelf.options.len * zelf.options.itemsize; + let bytelen = zelf.buffer.options.len * zelf.buffer.options.itemsize; if bytelen % itemsize != 0 { return Err( @@ -596,21 +597,15 @@ impl PyMemoryView { ); } - let buffer = PyBufferRef::new(zelf.clone()); - zelf.exports.fetch_add(1); - + let buffer = zelf.buffer.clone_with_options(BufferOptions { + itemsize, + len: bytelen / itemsize, + format: format.to_string().into(), + ..zelf.buffer.options.clone() + }); Ok(PyMemoryView { - obj: zelf.obj.clone(), buffer, - options: BufferOptions { - itemsize, - len: bytelen / itemsize, - format: format.to_string().into(), - ..zelf.options.clone() - }, released: AtomicCell::new(false), - stop: zelf.stop + itemsize - zelf.options.itemsize, - exports: AtomicCell::new(0), format_spec, hash: OnceCell::new(), ..*zelf @@ -626,13 +621,13 @@ impl PyMemoryView { return Ok(false); } - let other = match PyBufferRef::try_from_borrowed_object(vm, other) { + let other = match PyBuffer::try_from_borrowed_object(vm, other) { Ok(buf) => buf, Err(_) => return Ok(false), }; - let a_options = &zelf.options; - let b_options = other.get_options(); + let a_options = &zelf.buffer.options; + let b_options = &other.options; if a_options.len != b_options.len || a_options.ndim != b_options.ndim @@ -649,7 +644,7 @@ impl PyMemoryView { &*a_guard } None => { - a_vec = zelf.to_contiguous(); + a_vec = Self::to_bytes_vec(zelf); a_vec.as_slice() } }; @@ -687,62 +682,64 @@ impl PyMemoryView { } } -impl Drop for PyMemoryView { - fn drop(&mut self) { - self.release(); - } -} - impl AsBuffer for PyMemoryView { - fn get_buffer(zelf: &PyRef, vm: &VirtualMachine) -> PyResult> { + fn get_buffer(zelf: &PyRef, vm: &VirtualMachine) -> PyResult { if zelf.released.load() { Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned())) } else { - Ok(Box::new(zelf.clone())) + Ok(PyBuffer::new( + zelf.as_object().clone(), + zelf.clone(), + zelf.buffer.options.clone(), + )) } } } -impl PyBuffer for PyMemoryViewRef { - fn get_options(&self) -> &BufferOptions { - &self.options - } - +#[derive(Debug)] +struct Released; +impl PyBufferInternal for Released { fn obj_bytes(&self) -> BorrowedValue<[u8]> { - self.buffer.obj_bytes() + panic!(); } fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - self.buffer.obj_bytes_mut() + panic!(); } - fn release(&self) { - if self.exports.fetch_sub(1) == 1 && !self.released.load() { - self.buffer.release(); - } - } + fn release(&self) {} - fn as_contiguous(&self) -> Option> { - if !self.options.contiguous { - return None; - } - Some(BorrowedValue::map(self.obj_bytes(), |x| { + fn retain(&self) {} +} + +impl PyBufferInternal for PyMemoryView { + // NOTE: This impl maybe is anti-pattern. Only used for internal usage. + fn obj_bytes(&self) -> BorrowedValue<[u8]> { + BorrowedValue::map(self.buffer.internal.obj_bytes(), |x| { &x[self.start..self.stop] - })) + }) } - fn as_contiguous_mut(&self) -> Option> { - if !self.options.contiguous { - return None; - } - Some(BorrowedValueMut::map(self.obj_bytes_mut(), |x| { + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + BorrowedValueMut::map(self.buffer.internal.obj_bytes_mut(), |x| { &mut x[self.start..self.stop] - })) + }) } - fn to_contiguous(&self) -> Vec { - PyMemoryView::to_bytes_vec(self) + fn release(&self) {} + + fn retain(&self) {} +} + +impl PyBufferInternal for PyRef { + fn obj_bytes(&self) -> BorrowedValue<[u8]> { + self.deref().obj_bytes() + } + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + self.deref().obj_bytes_mut() } + fn release(&self) {} + fn retain(&self) {} } impl Comparable for PyMemoryView { @@ -772,7 +769,7 @@ impl Hashable for PyMemoryView { zelf.hash .get_or_try_init(|| { zelf.try_not_released(vm)?; - if !zelf.options.readonly { + if !zelf.buffer.options.readonly { return Err( vm.new_value_error("cannot hash writable memoryview object".to_owned()) ); @@ -785,7 +782,7 @@ impl Hashable for PyMemoryView { &*guard } None => { - vec = zelf.to_contiguous(); + vec = Self::to_bytes_vec(zelf); vec.as_slice() } }; diff --git a/vm/src/byteslike.rs b/vm/src/byteslike.rs index cefb9881e47..b5caa945b74 100644 --- a/vm/src/byteslike.rs +++ b/vm/src/byteslike.rs @@ -1,4 +1,4 @@ -use crate::buffer::PyBufferRef; +use crate::buffer::PyBuffer; use crate::builtins::PyStrRef; use crate::common::borrow::{BorrowedValue, BorrowedValueMut}; use crate::vm::VirtualMachine; @@ -8,11 +8,11 @@ use crate::{PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject}; /// any bytes-like object. Like the `y*` format code for `PyArg_Parse` in CPython. #[derive(Debug)] -pub struct ArgBytesLike(PyBufferRef); +pub struct ArgBytesLike(PyBuffer); /// A memory buffer, read-write access. Like the `w*` format code for `PyArg_Parse` in CPython. #[derive(Debug)] -pub struct ArgMemoryBuffer(PyBufferRef); +pub struct ArgMemoryBuffer(PyBuffer); impl ArgBytesLike { pub fn with_ref(&self, f: F) -> R @@ -54,15 +54,15 @@ impl ArgMemoryBuffer { impl ArgBytesLike { pub fn new(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { - let buffer = PyBufferRef::try_from_borrowed_object(vm, obj)?; - if buffer.get_options().contiguous { + let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; + if buffer.options.contiguous { Ok(Self(buffer)) } else { Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned())) } } - pub fn into_buffer(self) -> PyBufferRef { + pub fn into_buffer(self) -> PyBuffer { self.0 } @@ -82,7 +82,7 @@ pub fn try_bytes_like( obj: &PyObjectRef, f: impl FnOnce(&[u8]) -> R, ) -> PyResult { - let buffer = PyBufferRef::try_from_borrowed_object(vm, obj)?; + let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; buffer.as_contiguous().map(|x| f(&*x)).ok_or_else(|| { vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned()) }) @@ -93,7 +93,7 @@ pub fn try_rw_bytes_like( obj: &PyObjectRef, f: impl FnOnce(&mut [u8]) -> R, ) -> PyResult { - let buffer = PyBufferRef::try_from_borrowed_object(vm, obj)?; + let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; buffer .as_contiguous_mut() .map(|mut x| f(&mut *x)) @@ -102,18 +102,17 @@ pub fn try_rw_bytes_like( impl ArgMemoryBuffer { pub fn new(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { - let buffer = PyBufferRef::try_from_borrowed_object(vm, obj)?; - let options = buffer.get_options(); - if !options.contiguous { + let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; + if !buffer.options.contiguous { Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned())) - } else if options.readonly { + } else if buffer.options.readonly { Err(vm.new_type_error("buffer is not a read-write bytes-like object".to_owned())) } else { Ok(Self(buffer)) } } - pub fn into_buffer(self) -> PyBufferRef { + pub fn into_buffer(self) -> PyBuffer { self.0 } diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 04be329e53c..d7744a0c1ae 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -1,4 +1,4 @@ -use crate::buffer::PyBufferRef; +use crate::buffer::PyBuffer; /// Implementation of Printf-Style string formatting /// [https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting] use crate::builtins::float::{try_bigint, IntoPyFloat, PyFloat}; @@ -366,7 +366,7 @@ impl CFormatSpec { Ok(s.into_bytes()) } CFormatPreconversor::Str | CFormatPreconversor::Bytes => { - if let Ok(buffer) = PyBufferRef::try_from_borrowed_object(vm, &obj) { + if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &obj) { let guard; let vec; let bytes = match buffer.as_contiguous() { diff --git a/vm/src/slots.rs b/vm/src/slots.rs index e589051c8dd..8f5a126fe4c 100644 --- a/vm/src/slots.rs +++ b/vm/src/slots.rs @@ -61,7 +61,7 @@ pub(crate) type RichCompareFunc = fn( pub(crate) type GetattroFunc = fn(PyObjectRef, PyStrRef, &VirtualMachine) -> PyResult; pub(crate) type SetattroFunc = fn(&PyObjectRef, PyStrRef, Option, &VirtualMachine) -> PyResult<()>; -pub(crate) type BufferFunc = fn(&PyObjectRef, &VirtualMachine) -> PyResult>; +pub(crate) type BufferFunc = fn(&PyObjectRef, &VirtualMachine) -> PyResult; pub(crate) type IterFunc = fn(PyObjectRef, &VirtualMachine) -> PyResult; pub(crate) type IterNextFunc = fn(&PyObjectRef, &VirtualMachine) -> PyResult; @@ -510,16 +510,16 @@ pub trait SlotSetattro: PyValue { #[pyimpl] pub trait AsBuffer: PyValue { + // TODO: `flags` parameter #[pyslot] - fn tp_as_buffer(zelf: &PyObjectRef, vm: &VirtualMachine) -> PyResult> { - if let Some(zelf) = zelf.downcast_ref() { - Self::get_buffer(zelf, vm) - } else { - Err(vm.new_type_error("unexpected payload for get_buffer".to_owned())) - } + fn as_buffer(zelf: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = zelf + .downcast_ref() + .ok_or_else(|| vm.new_type_error("unexpected payload for get_buffer".to_owned()))?; + Self::get_buffer(zelf, vm) } - fn get_buffer(zelf: &PyRef, vm: &VirtualMachine) -> PyResult>; + fn get_buffer(zelf: &PyRef, vm: &VirtualMachine) -> PyResult; } #[pyimpl] diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index b430a3687ec..4127b5c42bb 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -9,7 +9,7 @@ pub type wchar_t = u32; #[pymodule(name = "array")] mod array { - use crate::buffer::{BufferOptions, PyBuffer, ResizeGuard}; + use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal, ResizeGuard}; use crate::builtins::float::IntoPyFloat; use crate::builtins::list::{PyList, PyListRef}; use crate::builtins::pystr::{PyStr, PyStrRef}; @@ -1088,44 +1088,38 @@ mod array { } impl AsBuffer for PyArray { - fn get_buffer(zelf: &PyRef, _vm: &VirtualMachine) -> PyResult> { - zelf.exports.fetch_add(1); + fn get_buffer(zelf: &PyRef, _vm: &VirtualMachine) -> PyResult { let array = zelf.read(); - let buf = ArrayBuffer { - array: zelf.clone(), - options: BufferOptions { + let buf = PyBuffer::new( + zelf.as_object().clone(), + zelf.clone(), + BufferOptions { readonly: false, len: array.len(), itemsize: array.itemsize(), format: array.typecode_str().into(), ..Default::default() }, - }; - Ok(Box::new(buf)) + ); + Ok(buf) } } - #[derive(Debug)] - struct ArrayBuffer { - array: PyArrayRef, - options: BufferOptions, - } - - impl PyBuffer for ArrayBuffer { + impl PyBufferInternal for PyRef { fn obj_bytes(&self) -> BorrowedValue<[u8]> { - self.array.get_bytes().into() + self.get_bytes().into() } fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - self.array.get_bytes_mut().into() + self.get_bytes_mut().into() } fn release(&self) { - self.array.exports.fetch_sub(1); + self.exports.fetch_sub(1); } - fn get_options(&self) -> &BufferOptions { - &self.options + fn retain(&self) { + self.exports.fetch_add(1); } } diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 3d6d9b9cdaf..456160aa903 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -77,7 +77,7 @@ mod _io { use std::io::{self, prelude::*, Cursor, SeekFrom}; use std::ops::Range; - use crate::buffer::{BufferOptions, PyBuffer, PyBufferRef, ResizeGuard}; + use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal, ResizeGuard}; use crate::builtins::memory::PyMemoryView; use crate::builtins::{ bytes::{PyBytes, PyBytesRef}, @@ -858,14 +858,13 @@ mod _io { /// None means non-blocking failed fn raw_write( &mut self, - buf: Option, + buf: Option, buf_range: Range, vm: &VirtualMachine, ) -> PyResult> { let len = buf_range.len(); let res = if let Some(buf) = buf { - let memobj = PyMemoryView::from_buffer_range(vm.ctx.none(), buf, buf_range, vm)? - .into_pyobject(vm); + let memobj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?.into_pyobject(vm); // TODO: loop if write() raises an interrupt vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj,))? @@ -879,17 +878,20 @@ mod _io { let writebuf = PyRc::new(BufferedRawBuffer { data: std::mem::take(&mut self.buffer).into(), range: buf_range, - options, }); + let raw = self.raw.as_ref().unwrap(); let memobj = PyMemoryView::from_buffer( - vm.ctx.none(), - PyBufferRef::new(writebuf.clone()), + PyBuffer { + obj: raw.clone(), + options, + internal: writebuf.clone(), + }, vm, )? .into_ref(vm); // TODO: loop if write() raises an interrupt - let res = vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj.clone(),)); + let res = vm.call_method(raw, "write", (memobj.clone(),)); memobj.released.store(true); self.buffer = std::mem::take(&mut writebuf.data.lock()); @@ -947,10 +949,9 @@ mod _io { let mut remaining = buf_len; let mut written = 0; - let rcbuf = obj.into_buffer().into_rcbuf(); + let buffer = obj.into_buffer(); while remaining > self.buffer.len() { - let res = - self.raw_write(Some(PyBufferRef::new(rcbuf.clone())), written..buf_len, vm)?; + let res = self.raw_write(Some(buffer.clone()), written..buf_len, vm)?; match res { Some(n) => { written += n; @@ -965,7 +966,7 @@ mod _io { // raw file is non-blocking if remaining > self.buffer.len() { // can't buffer everything, buffer what we can and error - let buf = rcbuf.as_contiguous().unwrap(); + let buf = buffer.as_contiguous().unwrap(); let buffer_len = self.buffer.len(); self.buffer.copy_from_slice(&buf[written..][..buffer_len]); self.raw_pos = 0; @@ -988,7 +989,7 @@ mod _io { self.reset_read(); } if remaining > 0 { - let buf = rcbuf.as_contiguous().unwrap(); + let buf = buffer.as_contiguous().unwrap(); self.buffer[..remaining].copy_from_slice(&buf[written..][..remaining]); written += remaining; } @@ -1098,7 +1099,7 @@ mod _io { fn raw_read( &mut self, - v: Either>, PyBufferRef>, + v: Either>, PyBuffer>, buf_range: Range, vm: &VirtualMachine, ) -> PyResult> { @@ -1116,11 +1117,13 @@ mod _io { let readbuf = PyRc::new(BufferedRawBuffer { data: std::mem::take(v).into(), range: buf_range, - options, }); let memobj = PyMemoryView::from_buffer( - vm.ctx.none(), - PyBufferRef::new(readbuf.clone()), + PyBuffer { + obj: vm.ctx.none(), + options, + internal: readbuf.clone(), + }, vm, )? .into_ref(vm); @@ -1135,8 +1138,7 @@ mod _io { res? } Either::B(buf) => { - let memobj = - PyMemoryView::from_buffer_range(vm.ctx.none(), buf, buf_range, vm)?; + let memobj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?; // TODO: loop if readinto() raises an interrupt vm.call_method(self.raw.as_ref().unwrap(), "readinto", (memobj,))? } @@ -1244,7 +1246,7 @@ mod _io { fn readinto_generic( &mut self, - buf: PyBufferRef, + buf: PyBuffer, readinto1: bool, vm: &VirtualMachine, ) -> PyResult> { @@ -1272,17 +1274,15 @@ mod _io { self.reset_read(); self.pos = 0; - let rcbuf = buf.into_rcbuf(); let mut remaining = buf_len - written; while remaining > 0 { let n = if remaining as usize > self.buffer.len() { - let buf = PyBufferRef::new(rcbuf.clone()); - self.raw_read(Either::B(buf), written..written + remaining, vm)? + self.raw_read(Either::B(buf.clone()), written..written + remaining, vm)? } else if !(readinto1 && written != 0) { let n = self.fill_buffer(vm)?; if let Some(n) = n.filter(|&n| n > 0) { let n = std::cmp::min(n, remaining); - rcbuf.as_contiguous_mut().unwrap()[written..][..n] + buf.as_contiguous_mut().unwrap()[written..][..n] .copy_from_slice(&self.buffer[self.pos as usize..][..n]); self.pos += n as Offset; written += n; @@ -1319,13 +1319,8 @@ mod _io { struct BufferedRawBuffer { data: PyMutex>, range: Range, - options: BufferOptions, } - impl PyBuffer for PyRc { - fn get_options(&self) -> &BufferOptions { - &self.options - } - + impl PyBufferInternal for BufferedRawBuffer { fn obj_bytes(&self) -> BorrowedValue<[u8]> { BorrowedValue::map(self.data.lock().into(), |data| &data[self.range.clone()]) } @@ -1337,6 +1332,7 @@ mod _io { } fn release(&self) {} + fn retain(&self) {} } pub fn get_offset(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -3352,47 +3348,33 @@ mod _io { #[pymethod] fn getbuffer(self, vm: &VirtualMachine) -> PyResult { - self.exports.fetch_add(1); - let buffer = PyBufferRef::new(BytesIOBuffer { - bytesio: self.clone(), - options: BufferOptions { - readonly: false, - len: self.buffer.read().cursor.get_ref().len(), - ..Default::default() - }, - }); - let view = PyMemoryView::from_buffer(self.into_object(), buffer, vm)?; + let options = BufferOptions { + readonly: false, + len: self.buffer.read().cursor.get_ref().len(), + ..Default::default() + }; + let buffer = PyBuffer::new(self.as_object().clone(), self, options); + let view = PyMemoryView::from_buffer(buffer, vm)?; Ok(view) } } - #[derive(Debug)] - struct BytesIOBuffer { - bytesio: BytesIORef, - options: BufferOptions, - } - - impl PyBuffer for BytesIOBuffer { - fn get_options(&self) -> &BufferOptions { - &self.options - } - + impl PyBufferInternal for PyRef { fn obj_bytes(&self) -> BorrowedValue<[u8]> { - PyRwLockReadGuard::map(self.bytesio.buffer.read(), |x| { - x.cursor.get_ref().as_slice() - }) - .into() + PyRwLockReadGuard::map(self.buffer.read(), |x| x.cursor.get_ref().as_slice()).into() } fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - PyRwLockWriteGuard::map(self.bytesio.buffer.write(), |x| { - x.cursor.get_mut().as_mut_slice() - }) - .into() + PyRwLockWriteGuard::map(self.buffer.write(), |x| x.cursor.get_mut().as_mut_slice()) + .into() } fn release(&self) { - self.bytesio.exports.fetch_sub(1); + self.exports.fetch_sub(1); + } + + fn retain(&self) { + self.exports.fetch_add(1); } } diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs index 0a765b8f953..016965458a5 100644 --- a/vm/src/stdlib/marshal.rs +++ b/vm/src/stdlib/marshal.rs @@ -22,14 +22,14 @@ mod decl { #[pyfunction] fn loads(code_bytes: ArgBytesLike, vm: &VirtualMachine) -> PyResult { - let code = - bytecode::CodeObject::from_bytes(&*code_bytes.borrow_buf()).map_err(|e| match e { - bytecode::CodeDeserializeError::Eof => vm.new_exception_msg( - vm.ctx.exceptions.eof_error.clone(), - "end of file while deserializing bytecode".to_owned(), - ), - _ => vm.new_value_error("Couldn't deserialize python bytecode".to_owned()), - })?; + let buf = &*code_bytes.borrow_buf(); + let code = bytecode::CodeObject::from_bytes(buf).map_err(|e| match e { + bytecode::CodeDeserializeError::Eof => vm.new_exception_msg( + vm.ctx.exceptions.eof_error.clone(), + "end of file while deserializing bytecode".to_owned(), + ), + _ => vm.new_value_error("Couldn't deserialize python bytecode".to_owned()), + })?; Ok(PyCode { code: vm.map_codeobj(code), }) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 46ca7e7c1e0..b06adb4fed6 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -14,7 +14,7 @@ use num_bigint::BigInt; use strum_macros::EnumString; use super::errno::errors; -use crate::buffer::PyBufferRef; +use crate::buffer::PyBuffer; use crate::builtins::bytes::{PyBytes, PyBytesRef}; use crate::builtins::dict::PyDictRef; use crate::builtins::int; @@ -213,8 +213,8 @@ pub(crate) fn fspath( impl TryFromObject for PyPathLike { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { // path_converter in CPython - let obj = match PyBufferRef::try_from_borrowed_object(vm, &obj) { - Ok(buffer) => PyBytes::from(Vec::from(&*buffer.obj_bytes())).into_pyobject(vm), + let obj = match PyBuffer::try_from_borrowed_object(vm, &obj) { + Ok(buffer) => PyBytes::from(buffer.internal.obj_bytes().to_vec()).into_pyobject(vm), Err(_) => obj, }; let path = fspath(obj, true, vm)?; diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index 382d7ce3610..44d0f1278cc 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -2,7 +2,7 @@ pub(crate) use _sre::make_module; #[pymodule] mod _sre { - use crate::buffer::PyBufferRef; + use crate::buffer::PyBuffer; use crate::builtins::list::PyListRef; use crate::builtins::tuple::PyTupleRef; use crate::builtins::{ @@ -152,7 +152,7 @@ mod _sre { let vec; let s; let str_drive = if self.isbytes { - buffer = PyBufferRef::try_from_borrowed_object(vm, &string)?; + buffer = PyBuffer::try_from_borrowed_object(vm, &string)?; let bytes = match buffer.as_contiguous() { Some(bytes) => { guard = bytes;