From 7efed05a8128f1dab8f0c860be13dce579a6ffc9 Mon Sep 17 00:00:00 2001 From: gf712 Date: Mon, 6 Jul 2026 14:27:58 +0100 Subject: [PATCH] runtime: add support for and implement stdin --- src/interpreter/Interpreter.cpp | 21 ++- src/runtime/PyMemoryView.cpp | 7 + src/runtime/PyMemoryView.hpp | 1 + src/runtime/modules/BuiltinsModule.cpp | 16 ++ src/runtime/modules/IOModule.cpp | 200 ++++++++++++++++++++++--- 5 files changed, 224 insertions(+), 21 deletions(-) diff --git a/src/interpreter/Interpreter.cpp b/src/interpreter/Interpreter.cpp index fea3bbdc..54b7d8f4 100644 --- a/src/interpreter/Interpreter.cpp +++ b/src/interpreter/Interpreter.cpp @@ -24,7 +24,9 @@ #include "executable/bytecode/BytecodeProgram.hpp" #include "utilities.hpp" +#include #include +#include namespace fs = std::filesystem; using namespace py; @@ -186,10 +188,23 @@ void Interpreter::internal_setup(const std::string &name, std::get(io_module->symbol_table()->map().at(String{ "FileIO" })); auto *buffered_writer = std::get(io_module->symbol_table()->map().at(String{ "BufferedWriter" })); + auto *buffered_reader = + std::get(io_module->symbol_table()->map().at(String{ "BufferedReader" })); auto *text_io_wrapper = std::get(io_module->symbol_table()->map().at(String{ "TextIOWrapper" })); + auto py_stdin = + file_io->call(PyTuple::create(Number{ STDIN_FILENO }, String{ "rb" }).unwrap(), nullptr) + .and_then([buffered_reader](PyObject *stdin) { + return buffered_reader->call(PyTuple::create(stdin).unwrap(), nullptr); + }) + .and_then([text_io_wrapper](PyObject *stdin_buffer_writer) { + return text_io_wrapper->call( + PyTuple::create(stdin_buffer_writer).unwrap(), nullptr); + }); + ASSERT(py_stdin.is_ok()); auto py_stdout = - file_io->call(PyTuple::create(Number{ 1 }, String{ "wb" }).unwrap(), nullptr) + file_io->call( + PyTuple::create(Number{ STDOUT_FILENO }, String{ "wb" }).unwrap(), nullptr) .and_then([buffered_writer](PyObject *stdout) { return buffered_writer->call(PyTuple::create(stdout).unwrap(), nullptr); }) @@ -199,7 +214,8 @@ void Interpreter::internal_setup(const std::string &name, }); ASSERT(py_stdout.is_ok()); auto py_stderr = - file_io->call(PyTuple::create(Number{ 2 }, String{ "wb" }).unwrap(), nullptr) + file_io->call( + PyTuple::create(Number{ STDERR_FILENO }, String{ "wb" }).unwrap(), nullptr) .and_then([buffered_writer](PyObject *stderr) { return buffered_writer->call(PyTuple::create(stderr).unwrap(), nullptr); }) @@ -208,6 +224,7 @@ void Interpreter::internal_setup(const std::string &name, PyTuple::create(stderr_buffer_writer).unwrap(), nullptr); }); ASSERT(py_stderr.is_ok()); + sys->add_symbol(PyString::create("stdin").unwrap(), py_stdin.unwrap()); sys->add_symbol(PyString::create("stdout").unwrap(), py_stdout.unwrap()); sys->add_symbol(PyString::create("stderr").unwrap(), py_stderr.unwrap()); } diff --git a/src/runtime/PyMemoryView.cpp b/src/runtime/PyMemoryView.cpp index 5c939d9a..1b280b6f 100644 --- a/src/runtime/PyMemoryView.cpp +++ b/src/runtime/PyMemoryView.cpp @@ -246,6 +246,13 @@ PyResult PyMemoryView::create(PyObject *object) type_error("memoryview: a bytes-like object is required, not {}", object->type()->name())); } +PyResult PyMemoryView::create(PyBuffer buffer) +{ + auto obj = VirtualMachine::the().heap().allocate(std::move(buffer)); + if (!obj) { return Err(memory_error(sizeof(PyMemoryView))); } + return Ok(obj); +} + PyResult PyMemoryView::__new__(const PyType *, PyTuple *args, PyDict *kwargs) { auto result = PyArgsParser::unpack_tuple(args, diff --git a/src/runtime/PyMemoryView.hpp b/src/runtime/PyMemoryView.hpp index b5d21871..1d5c6972 100644 --- a/src/runtime/PyMemoryView.hpp +++ b/src/runtime/PyMemoryView.hpp @@ -24,6 +24,7 @@ class PyMemoryView : public PyBaseObject public: static PyResult create(PyObject *object); + static PyResult create(PyBuffer buffer); static PyResult __new__(const PyType *type, PyTuple *args, PyDict *kwargs); PyResult __repr__() const; diff --git a/src/runtime/modules/BuiltinsModule.cpp b/src/runtime/modules/BuiltinsModule.cpp index 05232d6e..b151b139 100644 --- a/src/runtime/modules/BuiltinsModule.cpp +++ b/src/runtime/modules/BuiltinsModule.cpp @@ -80,10 +80,14 @@ PyResult print(const PyTuple *args, const PyDict *kwargs, Interprete .unwrap(); // sys.stdout may be None when FILE* stdout isn't connected if (!file || file == py_none()) { return Ok(py_none()); } + // TODO: flush should be false, but for now we keep it as true, since there is no flush at + // interpreter shutdown bool flush = true; if (kwargs) { static const Value separator_keyword = String{ "sep" }; static const Value end_keyword = String{ "end" }; + static const Value file_keyword = String{ "file" }; + static const Value flush_keyword = String{ "flush" }; if (auto it = kwargs->map().find(separator_keyword); it != kwargs->map().end()) { auto maybe_str = it->second; @@ -107,6 +111,18 @@ PyResult print(const PyTuple *args, const PyDict *kwargs, Interprete } end = PyString::create(std::get(maybe_str).s).unwrap(); } + if (auto it = kwargs->map().find(file_keyword); it != kwargs->map().end()) { + auto file_ = PyObject::from(it->second); + if (file_.is_err()) { return file_; } + file = file_.unwrap(); + } + if (auto it = kwargs->map().find(flush_keyword); it != kwargs->map().end()) { + auto flush_ = PyObject::from(it->second); + if (flush_.is_err()) { return flush_; } + const auto truthy = flush_.unwrap()->true_(); + if (truthy.is_err()) { return Err(truthy.unwrap_err()); } + flush = truthy.unwrap(); + } } auto file_write_ = file->get_method(PyString::create("write").unwrap()); if (file_write_.is_err()) { return file_write_; } diff --git a/src/runtime/modules/IOModule.cpp b/src/runtime/modules/IOModule.cpp index cdc01d17..a28358cb 100644 --- a/src/runtime/modules/IOModule.cpp +++ b/src/runtime/modules/IOModule.cpp @@ -22,8 +22,10 @@ #include "runtime/types/builtin.hpp" #include "utilities.hpp" #include "vm/VM.hpp" +#include #include #include +#include #include #include @@ -557,7 +559,7 @@ class RawIOBase : public IOBase }); } - PyResult readinto() const + PyResult readinto(PyObject *) const { return Err(not_implemented_error("_RawIOBase.readinto")); } @@ -984,6 +986,24 @@ struct Buffered } } + PyResult read1(int64_t n) + { + if (auto err = check_initialized(); err.is_err()) return Err(err.unwrap_err()); + + if (n < -1) { return Err(value_error("read length must be non-negative or -1")); } + + if (is_closed()) { return Err(value_error("read of closed file")); } + + const auto have = readahead(); + if (have > 0) { return static_cast(this)->readfast(n); } + Bytes b; + b.b.resize(n); + auto result = static_cast(this)->raw_read(std::span{ b.b.begin(), b.b.end() }); + if (result.is_err()) { return Err(result.unwrap_err()); } + b.b.resize(result.unwrap()); + return PyBytes::create(std::move(b)); + } + void visit_graph_buffered(Cell::Visitor &visitor) { if (raw) { visitor.visit(*raw); } @@ -1137,9 +1157,62 @@ class BufferedReader return Ok(py_none()); } - PyResult readgeneric(size_t) + PyResult raw_read(std::span data) { - return Err(not_implemented_error("BufferedReader.readgeneric")); + PyBuffer buffer{ + .buf = std::make_unique>(data.data()), + .obj = nullptr, + .len = static_cast(data.size()), + .itemsize = 1, + .readonly = false, + .ndim = 1, + .format = "B", + }; + auto memobj = PyMemoryView::create(std::move(buffer)).unwrap(); + return raw->get_method(PyString::create("readinto").unwrap()) + .and_then([memobj](PyObject *read_into) { + return read_into->call(PyTuple::create(memobj).unwrap(), nullptr); + }) + .and_then([](PyObject *n) -> PyResult { + if (!as(n)) { + return Err(value_error("expected 'int' got '{}'", n->type()->name())); + } + return Ok(as(n)->as_size_t()); + }); + } + + PyResult readgeneric(size_t n) + { + if (auto result = readfast(n); result.is_ok() && result.unwrap() != py_none()) { + return result; + } + Bytes bytes; + bytes.b.resize(n); + const auto current_size = readahead(); + auto remaining = n; + size_t written = 0; + if (current_size > 0) { + buffer->sgetn(bit_cast(bytes.b.data()), current_size); + remaining -= current_size; + written += current_size; + } + + // TODO: read and writable buffer? + // if (auto result = flush_and_rewind(); result.is_err()) { return result; } + + while (remaining > 0) { + std::span view{ bytes.b.data() + written, remaining }; + if (auto result = raw_read(view); result.is_err()) { + return Err(result.unwrap_err()); + } else { + if (result.unwrap() == 0) { break; } + ASSERT(result.unwrap() <= remaining); + remaining -= result.unwrap(); + written += result.unwrap(); + } + } + bytes.b.resize(written); + return PyBytes::create(std::move(bytes)); } PyResult __repr__() const @@ -1194,9 +1267,9 @@ class BufferedReader ASSERT(!kwargs || kwargs->map().empty()); int64_t n = -1; if (args && args->elements().size() > 1) { - return Err(value_error( - "BaseIO.readlines expected at most one argument (got {})", - args->elements().size())); + return Err( + value_error("read expected at most one argument (got {})", + args->elements().size())); } else if (args && args->elements().size() == 1) { auto arg0 = PyObject::from(args->elements()[0]); if (arg0.is_err()) return arg0; @@ -1211,9 +1284,31 @@ class BufferedReader } return static_cast *>(self)->read(n); }) + .def("read1", + [](BufferedReader *self, + PyTuple *args, + PyDict *kwargs) -> PyResult { + ASSERT(!kwargs || kwargs->map().empty()); + int64_t n = -1; + if (args && args->elements().size() > 1) { + return Err(value_error("read1 at most one argument (got {})", + args->elements().size())); + } else if (args && args->elements().size() == 1) { + auto arg0 = PyObject::from(args->elements()[0]); + if (arg0.is_err()) return arg0; + if (!as(arg0.unwrap()) && arg0.unwrap() != py_none()) { + return Err( + type_error("argument should be integer or None, not '{}'", + arg0.unwrap()->type()->name())); + } + if (arg0.unwrap() != py_none()) { + n = as(arg0.unwrap())->as_i64(); + } + } + return static_cast *>(self)->read1(n); + }) .def("readall", &BufferedReader::readall) // .def("peek", &BufferedReader::peek) - // .def("read1", &BufferedReader::read1) // .def("readinto", &BufferedReader::readinto) // .def("readinto1", &BufferedReader::readinto1) // .def("readline", &BufferedReader::readline) @@ -2172,6 +2267,27 @@ class FileIO : public RawIOBase return PyInteger::create(written); } + PyResult readinto(PyObject *bytes) + { + PyBuffer buffer; + if (auto result = bytes->get_buffer(buffer, 1); result.is_err()) { + return Err(result.unwrap_err()); + } + if (!buffer.is_ccontiguous()) { + return Err(type_error( + "readinto() argument must be contiguous buffer, not {}", bytes->type()->name())); + } + auto *data = static_cast(buffer.buf->get_buffer()); + // prime the buffer + m_filebuffer.sgetc(); + const auto to_read = std::min(buffer.len, m_filebuffer.in_avail()); + if (to_read > 0) { + const auto read = m_filebuffer.sgetn(data, to_read); + return PyInteger::create(read); + } + return PyInteger::create(0); + } + static PyType *register_type(PyModule *module) { if (!s_io_fileio) { @@ -2180,6 +2296,7 @@ class FileIO : public RawIOBase .def("write", &FileIO::write) .def("close", &FileIO::close) .def("flush", &FileIO::flush) + .def("readinto", &FileIO::readinto) .finalize(); } module->add_symbol(PyString::create("FileIO").unwrap(), s_io_fileio); @@ -2828,9 +2945,11 @@ class TextIOWrapper : public TextIOBase bool m_write_through; std::optional m_buffer_bytes; - size_t m_position; + size_t m_position{ 0 }; std::optional m_pending_bytes;// bytes to be written - size_t m_pending_bytes_count; + size_t m_pending_bytes_count{ 0 }; + + size_t m_chunk_size{ 8192 }; TextIOWrapper(PyType *type) : TextIOBase(type) {} @@ -2963,12 +3082,44 @@ class TextIOWrapper : public TextIOBase write_through == py_true()); } + PyResult read_chunk(size_t size_hint) + { + // TODO: handle decoding + const auto chunk_size = std::max(size_hint, m_chunk_size); + auto input_chunk = + this->m_buffer->get_method(PyString::create("read1").unwrap()) + .and_then([chunk_size](auto *read) { + return read->call(PyTuple::create(Number{ chunk_size }).unwrap(), nullptr); + }); + if (input_chunk.is_err()) { return Err(input_chunk.unwrap_err()); } + PyBuffer out; + if (auto result = input_chunk.unwrap()->get_buffer(out, 0); result.is_err()) { + return Err(result.unwrap_err()); + } + + if (out.len == 0) { return Ok(false); } + + auto *chunk_bytes = static_cast(out.buf->get_buffer()); + if (m_buffer_bytes.has_value()) { + m_buffer_bytes->b.insert(m_buffer_bytes->b.end(), chunk_bytes, chunk_bytes + out.len); + } else { + m_buffer_bytes = Bytes{ .b = { chunk_bytes, chunk_bytes + out.len } }; + } + return Ok(true); + } + PyResult readline(PyTuple *args, PyDict *kwargs) { - ASSERT(!args || args->size() == 0); - ASSERT(!kwargs || kwargs->map().empty()); + auto result = PyArgsParser::unpack_tuple(args, + kwargs, + "readline", + std::integral_constant{}, + std::integral_constant{}, + std::numeric_limits::max()); + if (result.is_err()) { return Err(result.unwrap_err()); } + auto [limit] = result.unwrap(); - if (auto err = setup_buffer(); err.is_err()) { return Err(err.unwrap_err()); } + if (auto result = writeflush(); result.is_err()) { return Err(result.unwrap_err()); } // Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\' auto is_line_end = [](std::string_view remaining) { @@ -2979,22 +3130,32 @@ class TextIOWrapper : public TextIOBase } return 0; }; + while (!m_buffer_bytes.has_value() || m_buffer_bytes->b.empty()) { + auto chunk = read_chunk(0); + if (chunk.is_err()) { return Err(chunk.unwrap_err()); } + if (chunk.unwrap() == 0) { break; } + } std::string_view remaining{ bit_cast(m_buffer_bytes->b.data()) + m_position, m_buffer_bytes->b.size() - m_position }; // TODO: use utf8 codepoints - std::string line; - while (!remaining.empty()) { + const char *start = remaining.data(); + size_t len = 0; + limit = std::min(limit, remaining.size()); + for (; len < limit; ++len) { if (auto chars = is_line_end(remaining)) { - m_position += chars; - line.insert(line.end(), remaining.begin(), remaining.begin() + chars); - return PyString::create(line); + // TODO: what if len becomes greater than limit due to \r\n? + len += chars; + break; } - line.push_back(remaining[0]); remaining = remaining.substr(1); - m_position++; } + std::string line{ start, start + len }; + std::rotate( + m_buffer_bytes->b.begin(), m_buffer_bytes->b.begin() + len, m_buffer_bytes->b.end()); + m_position = 0; + m_buffer_bytes->b.resize(m_buffer_bytes->b.size() - line.size()); return PyString::create(std::move(line)); } @@ -3030,6 +3191,7 @@ class TextIOWrapper : public TextIOBase return Ok(0); } + PyResult write(PyTuple *args, PyDict *kwargs) { ASSERT(!kwargs || kwargs->map().empty());