diff --git a/.gitignore b/.gitignore index 4041f03..ab9464c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ *.egg-info/ .coverage +/bazel-* diff --git a/BUILD b/BUILD new file mode 100644 index 0000000..ae821f1 --- /dev/null +++ b/BUILD @@ -0,0 +1,2 @@ +package(default_visibility = ["//visibility:public"]) + diff --git a/README.md b/README.md index 15be0b9..2a836ba 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Python Cloud Debugger Agent Google [Cloud Debugger](https://cloud.google.com/debugger/) for -Python 3.6, Python 3.7, Python 3.8 and Python 3.9. +Python 3.6, Python 3.7, Python 3.8, Python 3.9, and Python 3.10. ## Overview @@ -28,7 +28,7 @@ tested on Debian Linux, but it should work on other distributions as well. Cloud Debugger consists of 3 primary components: 1. The Python debugger agent (this repo implements one for CPython 3.6, - 3.7, 3.8 and 3.9). + 3.7, 3.8, 3.9, and 3.10). 2. Cloud Debugger service storing and managing snapshots/logpoints. Explore the APIs using [APIs Explorer](https://cloud.google.com/debugger/api/reference/rest/). @@ -224,14 +224,16 @@ Alternatively, you can pass the `--noreload` flag when running the Django using the `--noreload` flag disables the autoreload feature in Django, which means local changes to files will not be automatically picked up by Django. -### Experimental Firebase Realtime Database Backend +### Snapshot Debugger - Firebase Realtime Database Backend -This functionality is available for release 3.0 onward of this agent. +This functionality is available for release 3.0 onward of this agent and +provides support for the Snapshot Debugger, which is being provided as a +replacement for the deprecated Cloud Debugger service. The agent can be configured to use Firebase Realtime Database as a backend -instead of the deprecated Cloud Debugger service. If the Firebase backend is -used, breakpoints can be viewed and set using the Snapshot Debugger CLI instead -of the Cloud Console. +instead of the Cloud Debugger service. If the Firebase backend is used, +breakpoints can be viewed and set using the Snapshot Debugger CLI instead of the +Cloud Console. To use the Firebase backend, set the flag when enabling the agent: @@ -258,7 +260,8 @@ except ImportError: pass ``` -See https://github.com/GoogleCloudPlatform/snapshot-debugger for more details. +See https://github.com/GoogleCloudPlatform/snapshot-debugger and +https://cloud.google.com/debugger/docs/deprecations for more details. ## Flag Reference @@ -312,6 +315,9 @@ The following instructions are intended to help with modifying the codebase. Run the `build_and_test.sh` script from the root of the repository to build and run the unit tests using the locally installed version of Python. +Run `bazel test tests/cpp:all` from the root of the repository to run unit +tests against the C++ portion of the codebase. + #### Local development You may want to run an agent with local changes in an application in order to diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..55013f2 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,50 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "bazel_skylib", + sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + ], +) +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") +bazel_skylib_workspace() + +http_archive( + name = "com_github_gflags_gflags", + sha256 = "34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf", + strip_prefix = "gflags-2.2.2", + urls = ["https://github.com/gflags/gflags/archive/v2.2.2.tar.gz"], +) + +http_archive( + name = "com_github_google_glog", + sha256 = "21bc744fb7f2fa701ee8db339ded7dce4f975d0d55837a97be7d46e8382dea5a", + strip_prefix = "glog-0.5.0", + urls = ["https://github.com/google/glog/archive/v0.5.0.zip"], +) + +# Pinning to 1.12.1, the last release that supports C++11 +http_archive( + name = "com_google_googletest", + urls = ["https://github.com/google/googletest/archive/58d77fa8070e8cec2dc1ed015d66b454c8d78850.tar.gz"], + strip_prefix = "googletest-58d77fa8070e8cec2dc1ed015d66b454c8d78850", +) + +# Used to build against Python.h +http_archive( + name = "pybind11_bazel", + strip_prefix = "pybind11_bazel-faf56fb3df11287f26dbc66fdedf60a2fc2c6631", + urls = ["https://github.com/pybind/pybind11_bazel/archive/faf56fb3df11287f26dbc66fdedf60a2fc2c6631.zip"], +) + +http_archive( + name = "pybind11", + build_file = "@pybind11_bazel//:pybind11.BUILD", + strip_prefix = "pybind11-2.9.2", + urls = ["https://github.com/pybind/pybind11/archive/v2.9.2.tar.gz"], +) +load("@pybind11_bazel//:python_configure.bzl", "python_configure") +python_configure(name = "local_config_python")#, python_interpreter_target = interpreter) + diff --git a/build_and_test.sh b/build_and_test.sh index 8035cce..8e742ea 100755 --- a/build_and_test.sh +++ b/build_and_test.sh @@ -8,5 +8,5 @@ python3 -m venv /tmp/cdbg-venv source /tmp/cdbg-venv/bin/activate pip3 install -r requirements_dev.txt pip3 install src/dist/* --force-reinstall -python3 -m pytest tests +python3 -m pytest tests/py deactivate diff --git a/src/build-wheels.sh b/src/build-wheels.sh index 2d6e92d..1e4a0c6 100755 --- a/src/build-wheels.sh +++ b/src/build-wheels.sh @@ -3,7 +3,7 @@ GFLAGS_URL=https://github.com/gflags/gflags/archive/v2.2.2.tar.gz GLOG_URL=https://github.com/google/glog/archive/v0.4.0.tar.gz -SUPPORTED_VERSIONS=(cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39) +SUPPORTED_VERSIONS=(cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310) ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") >/dev/null; /bin/pwd -P) @@ -72,7 +72,7 @@ for PY_VERSION in ${SUPPORTED_VERSIONS[@]}; do echo "Running tests" "/opt/python/${PY_VERSION}/bin/pip" install google-python-cloud-debugger --no-index -f /io/dist - "/opt/python/${PY_VERSION}/bin/pytest" /io/tests + "/opt/python/${PY_VERSION}/bin/pytest" /io/tests/py done popd diff --git a/src/googleclouddebugger/BUILD b/src/googleclouddebugger/BUILD new file mode 100644 index 0000000..c0d6ae7 --- /dev/null +++ b/src/googleclouddebugger/BUILD @@ -0,0 +1,103 @@ +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "common", + hdrs = ["common.h"], + deps = [ + "@com_github_google_glog//:glog", + "@local_config_python//:python_headers", + ], +) + +cc_library( + name = "nullable", + hdrs = ["nullable.h"], + deps = [ + ":common", + ], +) + +cc_library( + name = "python_util", + srcs = ["python_util.cc"], + hdrs = ["python_util.h"], + deps = [ + ":common", + ":nullable", + "//src/third_party:pylinetable", + ], +) + + +cc_library( + name = "python_callback", + srcs = ["python_callback.cc"], + hdrs = ["python_callback.h"], + deps = [ + ":common", + ":python_util", + ], +) + +cc_library( + name = "leaky_bucket", + srcs = ["leaky_bucket.cc"], + hdrs = ["leaky_bucket.h"], + deps = [ + ":common", + ], +) + +cc_library( + name = "rate_limit", + srcs = ["rate_limit.cc"], + hdrs = ["rate_limit.h"], + deps = [ + ":common", + ":leaky_bucket", + ], +) + +cc_library( + name = "bytecode_manipulator", + srcs = ["bytecode_manipulator.cc"], + hdrs = ["bytecode_manipulator.h"], + deps = [ + ":common", + ], +) + +cc_library( + name = "bytecode_breakpoint", + srcs = ["bytecode_breakpoint.cc"], + hdrs = ["bytecode_breakpoint.h"], + deps = [ + ":bytecode_manipulator", + ":common", + ":python_callback", + ":python_util", + ], +) + +cc_library( + name = "immutability_tracer", + srcs = ["immutability_tracer.cc"], + hdrs = ["immutability_tracer.h"], + deps = [ + ":common", + ":python_util", + ], +) + +cc_library( + name = "conditional_breakpoint", + srcs = ["conditional_breakpoint.cc"], + hdrs = ["conditional_breakpoint.h"], + deps = [ + ":common", + ":immutability_tracer", + ":python_util", + ":rate_limit", + ":leaky_bucket", + ], +) diff --git a/src/googleclouddebugger/bytecode_breakpoint.cc b/src/googleclouddebugger/bytecode_breakpoint.cc index 8b782d7..dd1af6e 100644 --- a/src/googleclouddebugger/bytecode_breakpoint.cc +++ b/src/googleclouddebugger/bytecode_breakpoint.cc @@ -82,7 +82,7 @@ int BytecodeBreakpoint::CreateBreakpoint( // table in case "code_object" is already patched with another breakpoint. CodeObjectLinesEnumerator lines_enumerator( code_object->co_firstlineno, - code_object_breakpoints->original_lnotab.get()); + code_object_breakpoints->original_linedata.get()); while (lines_enumerator.line_number() != line) { if (!lines_enumerator.Next()) { LOG(ERROR) << "Line " << line << " not found in " @@ -237,8 +237,14 @@ BytecodeBreakpoint::PreparePatchCodeObject( return nullptr; // Probably a built-in method or uninitialized code object. } - data->original_lnotab = + // Store the original (unmodified) line data. +#if PY_VERSION_HEX < 0x030A0000 + data->original_linedata = ScopedPyObject::NewReference(code_object.get()->co_lnotab); +#else + data->original_linedata = + ScopedPyObject::NewReference(code_object.get()->co_linetable); +#endif patches_[code_object] = data.get(); return data.release(); @@ -262,29 +268,38 @@ void BytecodeBreakpoint::PatchCodeObject(CodeObjectBreakpoints* code) { << " from patched " << code->zombie_refs.back().get(); Py_INCREF(code_object->co_code); + // Restore the original line data to the code object. +#if PY_VERSION_HEX < 0x030A0000 if (code_object->co_lnotab != nullptr) { code->zombie_refs.push_back(ScopedPyObject(code_object->co_lnotab)); } - code_object->co_lnotab = code->original_lnotab.get(); + code_object->co_lnotab = code->original_linedata.get(); Py_INCREF(code_object->co_lnotab); +#else + if (code_object->co_linetable != nullptr) { + code->zombie_refs.push_back(ScopedPyObject(code_object->co_linetable)); + } + code_object->co_linetable = code->original_linedata.get(); + Py_INCREF(code_object->co_linetable); +#endif return; } std::vector bytecode = PyBytesToByteArray(code->original_code.get()); - bool has_lnotab = false; - std::vector lnotab; - if (!code->original_lnotab.is_null() && - PyBytes_CheckExact(code->original_lnotab.get())) { - has_lnotab = true; - lnotab = PyBytesToByteArray(code->original_lnotab.get()); + bool has_linedata = false; + std::vector linedata; + if (!code->original_linedata.is_null() && + PyBytes_CheckExact(code->original_linedata.get())) { + has_linedata = true; + linedata = PyBytesToByteArray(code->original_linedata.get()); } BytecodeManipulator bytecode_manipulator( std::move(bytecode), - has_lnotab, - std::move(lnotab)); + has_linedata, + std::move(linedata)); // Add callbacks to code object constants and patch the bytecode. std::vector callbacks; @@ -306,17 +321,16 @@ void BytecodeBreakpoint::PatchCodeObject(CodeObjectBreakpoints* code) { callbacks.push_back(breakpoint.hit_callable.get()); -#if PY_MAJOR_VERSION >= 3 // In Python 3, since we allow upgrading of instructions to use // EXTENDED_ARG, the offsets for lines originally calculated might not be // accurate, so we need to recalculate them each insertion. offset_found = false; - if (bytecode_manipulator.has_lnotab()) { - ScopedPyObject lnotab(PyBytes_FromStringAndSize( - reinterpret_cast(bytecode_manipulator.lnotab().data()), - bytecode_manipulator.lnotab().size())); + if (bytecode_manipulator.has_linedata()) { + ScopedPyObject linedata(PyBytes_FromStringAndSize( + reinterpret_cast(bytecode_manipulator.linedata().data()), + bytecode_manipulator.linedata().size())); CodeObjectLinesEnumerator lines_enumerator(code_object->co_firstlineno, - lnotab.release()); + linedata.release()); while (lines_enumerator.line_number() != breakpoint.line) { if (!lines_enumerator.Next()) { break; @@ -325,7 +339,6 @@ void BytecodeBreakpoint::PatchCodeObject(CodeObjectBreakpoints* code) { } offset_found = lines_enumerator.line_number() == breakpoint.line; } -#endif if (!offset_found || !bytecode_manipulator.InjectMethodCall(offset, const_index)) { @@ -355,14 +368,26 @@ void BytecodeBreakpoint::PatchCodeObject(CodeObjectBreakpoints* code) { << " reassigned to " << code_object->co_code << ", original was " << code->original_code.get(); - if (has_lnotab) { + // Update the line data in the code object. +#if PY_VERSION_HEX < 0x030A0000 + if (has_linedata) { code->zombie_refs.push_back(ScopedPyObject(code_object->co_lnotab)); ScopedPyObject lnotab_string(PyBytes_FromStringAndSize( - reinterpret_cast(bytecode_manipulator.lnotab().data()), - bytecode_manipulator.lnotab().size())); + reinterpret_cast(bytecode_manipulator.linedata().data()), + bytecode_manipulator.linedata().size())); DCHECK(!lnotab_string.is_null()); code_object->co_lnotab = lnotab_string.release(); } +#else + if (has_linedata) { + code->zombie_refs.push_back(ScopedPyObject(code_object->co_linetable)); + ScopedPyObject linetable_string(PyBytes_FromStringAndSize( + reinterpret_cast(bytecode_manipulator.linedata().data()), + bytecode_manipulator.linedata().size())); + DCHECK(!linetable_string.is_null()); + code_object->co_linetable = linetable_string.release(); + } +#endif // Invoke error callback after everything else is done. The callback may // decide to remove the breakpoint, which will change "code". diff --git a/src/googleclouddebugger/bytecode_breakpoint.h b/src/googleclouddebugger/bytecode_breakpoint.h index 057766f..5eaa893 100644 --- a/src/googleclouddebugger/bytecode_breakpoint.h +++ b/src/googleclouddebugger/bytecode_breakpoint.h @@ -162,9 +162,10 @@ class BytecodeBreakpoint { // Original value of PyCodeObject::co_code before patching. ScopedPyObject original_code; - // Original value of PythonCode::co_lnotab before patching. - // "lnotab" stands for "line numbers table" in CPython lingo. - ScopedPyObject original_lnotab; + // Original value of PythonCode::co_lnotab or PythonCode::co_linetable + // before patching. This is the line numbers table in CPython <= 3.9 and + // CPython >= 3.10 respectively + ScopedPyObject original_linedata; }; // Loads code object into "patches_" if not there yet. Returns nullptr if diff --git a/src/googleclouddebugger/bytecode_manipulator.cc b/src/googleclouddebugger/bytecode_manipulator.cc index 9ee7e27..3c95edd 100644 --- a/src/googleclouddebugger/bytecode_manipulator.cc +++ b/src/googleclouddebugger/bytecode_manipulator.cc @@ -228,11 +228,11 @@ static std::vector BuildMethodCall(int const_index) { } BytecodeManipulator::BytecodeManipulator(std::vector bytecode, - const bool has_lnotab, - std::vector lnotab) - : has_lnotab_(has_lnotab) { + const bool has_linedata, + std::vector linedata) + : has_linedata_(has_linedata) { data_.bytecode = std::move(bytecode); - data_.lnotab = std::move(lnotab); + data_.linedata = std::move(linedata); strategy_ = STRATEGY_INSERT; // Default strategy. for (auto it = data_.bytecode.begin(); it < data_.bytecode.end(); ) { @@ -296,21 +296,13 @@ struct Insertion { // InsertAndUpdateBranchInstructions. static const int kMaxInsertionIterations = 10; - +#if PY_VERSION_HEX < 0x030A0000 // Updates the line number table for an insertion in the bytecode. -// This is different than what the Python 2 version of InsertMethodCall() does. -// It should be more accurate, but is confined to Python 3 only for safety. -// This handles the case of adding insertion for EXTENDED_ARG better. // Example for inserting 2 bytes at offset 2: -// lnotab: [{2, 1}, {4, 1}] // {offset_delta, line_delta} -// Old algorithm: [{2, 0}, {2, 1}, {4, 1}] -// New algorithm: [{2, 1}, {6, 1}] -// In the old version, trying to get the offset to insert a breakpoint right -// before line 1 would result in an offset of 2, which is inaccurate as the -// instruction before is an EXTENDED_ARG which will now be applied to the first -// instruction inserted instead of its original target. -static void InsertAndUpdateLnotab(int offset, int size, - std::vector* lnotab) { +// lnotab: [{2, 1}, {4, 1}] // {offset_delta, line_delta} +// updated: [{2, 1}, {6, 1}] +static void InsertAndUpdateLineData(int offset, int size, + std::vector* lnotab) { int current_offset = 0; for (auto it = lnotab->begin(); it != lnotab->end(); it += 2) { current_offset += it[0]; @@ -330,6 +322,36 @@ static void InsertAndUpdateLnotab(int offset, int size, } } } +#else +// Updates the line number table for an insertion in the bytecode. +// Example for inserting 2 bytes at offset 2: +// linetable: [{2, 1}, {4, 1}] // {address_end_delta, line_delta} +// updated: [{2, 1}, {6, 1}] +// +// For more information on the linetable format in Python 3.10, see: +// https://github.com/python/cpython/blob/main/Objects/lnotab_notes.txt +static void InsertAndUpdateLineData(int offset, int size, + std::vector* linetable) { + int current_offset = 0; + for (auto it = linetable->begin(); it != linetable->end(); it += 2) { + current_offset += it[0]; + + if (current_offset > offset) { + int remaining_size = it[0] + size; + int remaining_lines = it[1]; + it = linetable->erase(it, it + 2); + while (remaining_size > 0xFE) { // Max address delta is listed as 254. + it = linetable->insert(it, 0xFE) + 1; + it = linetable->insert(it, 0) + 1; + remaining_size -= 0xFE; + } + it = linetable->insert(it, remaining_size) + 1; + it = linetable->insert(it, remaining_lines) + 1; + return; + } + } +} +#endif // Reserves space for instructions to be inserted into the bytecode, and // calculates the new offsets and arguments of branch instructions. @@ -426,8 +448,16 @@ static bool InsertAndUpdateBranchInstructions( } if (need_to_update) { +#if PY_VERSION_HEX < 0x030A0000 + int delta = insertion.size; +#else + // Changed in version 3.10: The argument of jump, exception handling + // and loop instructions is now the instruction offset rather than the + // byte offset. + int delta = insertion.size / 2; +#endif PythonInstruction new_instruction = - PythonInstructionArg(instruction.opcode, arg + insertion.size); + PythonInstructionArg(instruction.opcode, arg + delta); int size_diff = new_instruction.size - instruction.size; if (size_diff > 0) { insertions.push_back(Insertion { size_diff, it->current_offset }); @@ -490,8 +520,8 @@ bool BytecodeManipulator::InsertMethodCall( // Insert the method call. data->bytecode.insert(data->bytecode.begin() + offset, method_call_size, NOP); WriteInstructions(data->bytecode.begin() + offset, method_call_instructions); - if (has_lnotab_) { - InsertAndUpdateLnotab(offset, method_call_size, &data->lnotab); + if (has_linedata_) { + InsertAndUpdateLineData(offset, method_call_size, &data->linedata); } // Write new branch instructions. @@ -503,8 +533,8 @@ bool BytecodeManipulator::InsertMethodCall( int offset = it->current_offset; if (size_diff > 0) { data->bytecode.insert(data->bytecode.begin() + offset, size_diff, NOP); - if (has_lnotab_) { - InsertAndUpdateLnotab(it->current_offset, size_diff, &data->lnotab); + if (has_linedata_) { + InsertAndUpdateLineData(it->current_offset, size_diff, &data->linedata); } } else if (size_diff < 0) { // The Python compiler sometimes prematurely adds EXTENDED_ARG with an diff --git a/src/googleclouddebugger/bytecode_manipulator.h b/src/googleclouddebugger/bytecode_manipulator.h index d3a7de4..31a5e46 100644 --- a/src/googleclouddebugger/bytecode_manipulator.h +++ b/src/googleclouddebugger/bytecode_manipulator.h @@ -71,17 +71,17 @@ namespace cdbg { // 19 JUMP_ABSOLUTE 3 class BytecodeManipulator { public: - BytecodeManipulator(std::vector bytecode, const bool has_lnotab, - std::vector lnotab); + BytecodeManipulator(std::vector bytecode, const bool has_linedata, + std::vector linedata); // Gets the transformed method bytecode. const std::vector& bytecode() const { return data_.bytecode; } // Returns true if this class was initialized with line numbers table. - bool has_lnotab() const { return has_lnotab_; } + bool has_linedata() const { return has_linedata_; } // Gets the method line numbers table or empty vector if not available. - const std::vector& lnotab() const { return data_.lnotab; } + const std::vector& linedata() const { return data_.linedata; } // Rewrites the method bytecode to invoke callable at the specified offset. // Return false if the method call could not be inserted. The bytecode @@ -109,8 +109,8 @@ class BytecodeManipulator { // Bytecode of a transformed method. std::vector bytecode; - // Method line numbers table or empty vector if "has_lnotab_" is false. - std::vector lnotab; + // Method line numbers table or empty vector if "has_linedata_" is false. + std::vector linedata; }; // Insert space into the bytecode. This space is later used to add new @@ -130,7 +130,7 @@ class BytecodeManipulator { Data data_; // True if the method has line number table. - const bool has_lnotab_; + const bool has_linedata_; // Algorithm to insert breakpoint callback into method bytecode. Strategy strategy_; diff --git a/src/googleclouddebugger/immutability_tracer.cc b/src/googleclouddebugger/immutability_tracer.cc index d5f102a..c05d407 100644 --- a/src/googleclouddebugger/immutability_tracer.cc +++ b/src/googleclouddebugger/immutability_tracer.cc @@ -400,6 +400,16 @@ static OpcodeMutableStatus IsOpcodeMutable(const uint8_t opcode) { #if PY_VERSION_HEX >= 0x03080000 // Added back in Python 3.8 (was in 2.7 as well) case ROT_FOUR: +#endif +#if PY_VERSION_HEX >= 0x030A0000 + // Added in Python 3.10 + case COPY_DICT_WITHOUT_KEYS: + case GET_LEN: + case MATCH_MAPPING: + case MATCH_SEQUENCE: + case MATCH_KEYS: + case MATCH_CLASS: + case ROT_N: #endif return OPCODE_NOT_MUTABLE; @@ -468,6 +478,10 @@ static OpcodeMutableStatus IsOpcodeMutable(const uint8_t opcode) { case RERAISE: case WITH_EXCEPT_START: case LOAD_ASSERTION_ERROR: +#endif +#if PY_VERSION_HEX >= 0x030A0000 + // Added in Python 3.10 + case GEN_START: #endif return OPCODE_MUTABLE; diff --git a/src/googleclouddebugger/module_explorer.py b/src/googleclouddebugger/module_explorer.py index acecea9..99829df 100644 --- a/src/googleclouddebugger/module_explorer.py +++ b/src/googleclouddebugger/module_explorer.py @@ -78,15 +78,24 @@ def _GetLineNumbers(code_object): Yields: The next line number in the code object. """ - # Get the line number deltas, which are the odd number entries, from the - # lnotab. See - # https://svn.python.org/projects/python/branches/pep-0384/Objects/lnotab_notes.txt - # In Python 3, this is just a byte array. - line_incrs = code_object.co_lnotab[1::2] - current_line = code_object.co_firstlineno - for line_incr in line_incrs: - current_line += line_incr - yield current_line + + if sys.version_info.minor < 10: + # Get the line number deltas, which are the odd number entries, from the + # lnotab. See + # https://svn.python.org/projects/python/branches/pep-0384/Objects/lnotab_notes.txt + # In Python 3, prior to 3.10, this is just a byte array. + line_incrs = code_object.co_lnotab[1::2] + current_line = code_object.co_firstlineno + for line_incr in line_incrs: + current_line += line_incr + yield current_line + else: + # Get the line numbers directly, which are the third entry in the tuples. + # https://peps.python.org/pep-0626/#the-new-co-lines-method-of-code-objects + line_numbers = [entry[2] for entry in code_object.co_lines()] + for line_number in line_numbers: + if line_number is not None: + yield line_number def _GetModuleCodeObjects(module): diff --git a/src/googleclouddebugger/python_util.cc b/src/googleclouddebugger/python_util.cc index 90b67ce..e28a142 100644 --- a/src/googleclouddebugger/python_util.cc +++ b/src/googleclouddebugger/python_util.cc @@ -23,6 +23,11 @@ #include +#if PY_VERSION_HEX >= 0x030A0000 +#include "../third_party/pylinetable.h" +#endif // PY_VERSION_HEX >= 0x030A0000 + + namespace devtools { namespace cdbg { @@ -32,17 +37,22 @@ static PyObject* g_debuglet_module = nullptr; CodeObjectLinesEnumerator::CodeObjectLinesEnumerator( PyCodeObject* code_object) { +#if PY_VERSION_HEX < 0x030A0000 Initialize(code_object->co_firstlineno, code_object->co_lnotab); +#else + Initialize(code_object->co_firstlineno, code_object->co_linetable); +#endif // PY_VERSION_HEX < 0x030A0000 } CodeObjectLinesEnumerator::CodeObjectLinesEnumerator( int firstlineno, - PyObject* lnotab) { - Initialize(firstlineno, lnotab); + PyObject* linedata) { + Initialize(firstlineno, linedata); } +#if PY_VERSION_HEX < 0x030A0000 void CodeObjectLinesEnumerator::Initialize( int firstlineno, PyObject* lnotab) { @@ -86,7 +96,26 @@ bool CodeObjectLinesEnumerator::Next() { } } } +#else + +void CodeObjectLinesEnumerator::Initialize( + int firstlineno, + PyObject* linetable) { + Py_ssize_t length = PyBytes_Size(linetable); + _PyLineTable_InitAddressRange(PyBytes_AsString(linetable), length, firstlineno, &range_); +} +bool CodeObjectLinesEnumerator::Next() { + while (_PyLineTable_NextAddressRange(&range_)) { + if (range_.ar_line >= 0) { + line_number_ = range_.ar_line; + offset_ = range_.ar_start; + return true; + } + } + return false; +} +#endif // PY_VERSION_HEX < 0x030A0000 PyObject* GetDebugletModule() { DCHECK(g_debuglet_module != nullptr); diff --git a/src/googleclouddebugger/python_util.h b/src/googleclouddebugger/python_util.h index 57b5425..10116be 100644 --- a/src/googleclouddebugger/python_util.h +++ b/src/googleclouddebugger/python_util.h @@ -178,7 +178,7 @@ class CodeObjectLinesEnumerator { explicit CodeObjectLinesEnumerator(PyCodeObject* code_object); // Uses explicitly provided line table. - CodeObjectLinesEnumerator(int firstlineno, PyObject* lnotab); + CodeObjectLinesEnumerator(int firstlineno, PyObject* linedata); // Moves over to the next entry in code object line table. bool Next(); @@ -190,24 +190,31 @@ class CodeObjectLinesEnumerator { int32_t line_number() const { return line_number_; } private: - void Initialize(int firstlineno, PyObject* lnotab); + void Initialize(int firstlineno, PyObject* linedata); private: + // Bytecode offset of the current line. + int32_t offset_; + + // Current source code line number + int32_t line_number_; + +#if PY_VERSION_HEX < 0x030A0000 // Number of remaining entries in line table. int remaining_entries_; // Pointer to the next entry of line table. const uint8_t* next_entry_; - // Bytecode offset of the current line. - int32_t offset_; - - // Current source code line number - int32_t line_number_; +#else + // Current address range in the linetable data. + PyCodeAddressRange range_; +#endif DISALLOW_COPY_AND_ASSIGN(CodeObjectLinesEnumerator); }; + template bool operator== (TPointer* ref1, const ScopedPyObjectT& ref2) { return ref2 == ref1; diff --git a/src/googleclouddebugger/version.py b/src/googleclouddebugger/version.py index 267e47d..a61798c 100644 --- a/src/googleclouddebugger/version.py +++ b/src/googleclouddebugger/version.py @@ -4,4 +4,4 @@ # The major version should only change on breaking changes. Minor version # changes go between regular updates. Instances running debuggers with # different major versions will show up as two different debuggees. -__version__ = '3.2' +__version__ = '3.3' diff --git a/src/setup.py b/src/setup.py index 0c24bad..6b380d5 100644 --- a/src/setup.py +++ b/src/setup.py @@ -117,6 +117,7 @@ def ReadConfig(section, value, default): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', ]) diff --git a/src/third_party/BUILD b/src/third_party/BUILD new file mode 100644 index 0000000..bcce1e2 --- /dev/null +++ b/src/third_party/BUILD @@ -0,0 +1,7 @@ +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "pylinetable", + hdrs = ["pylinetable.h"], +) + diff --git a/src/third_party/pylinetable.h b/src/third_party/pylinetable.h new file mode 100644 index 0000000..ea44c64 --- /dev/null +++ b/src/third_party/pylinetable.h @@ -0,0 +1,210 @@ +/** + * Copyright (c) 2001-2023 Python Software Foundation; All Rights Reserved + * + * You may obtain a copy of the PSF License at + * + * https://docs.python.org/3/license.html + */ + +#ifndef DEVTOOLS_CDBG_DEBUGLETS_PYTHON_PYLINETABLE_H_ +#define DEVTOOLS_CDBG_DEBUGLETS_PYTHON_PYLINETABLE_H_ + +/* Python Linetable helper methods. + * They are not part of the cpython api. + * This code has been extracted from: + * https://github.com/python/cpython/blob/main/Objects/codeobject.c + * + * See https://peps.python.org/pep-0626/#out-of-process-debuggers-and-profilers + * for more information about this code and its usage. + */ + +#if PY_VERSION_HEX >= 0x030B0000 +// Things are different in 3.11 than 3.10. +// See https://github.com/python/cpython/blob/main/Objects/locations.md + +typedef enum _PyCodeLocationInfoKind { + /* short forms are 0 to 9 */ + PY_CODE_LOCATION_INFO_SHORT0 = 0, + /* one lineforms are 10 to 12 */ + PY_CODE_LOCATION_INFO_ONE_LINE0 = 10, + PY_CODE_LOCATION_INFO_ONE_LINE1 = 11, + PY_CODE_LOCATION_INFO_ONE_LINE2 = 12, + + PY_CODE_LOCATION_INFO_NO_COLUMNS = 13, + PY_CODE_LOCATION_INFO_LONG = 14, + PY_CODE_LOCATION_INFO_NONE = 15 +} _PyCodeLocationInfoKind; + +/** Out of process API for initializing the location table. */ +extern void _PyLineTable_InitAddressRange( + const char *linetable, + Py_ssize_t length, + int firstlineno, + PyCodeAddressRange *range); + +/** API for traversing the line number table. */ +extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); + + +void _PyLineTable_InitAddressRange(const char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range) { + range->opaque.lo_next = linetable; + range->opaque.limit = range->opaque.lo_next + length; + range->ar_start = -1; + range->ar_end = 0; + range->opaque.computed_line = firstlineno; + range->ar_line = -1; +} + +static int +scan_varint(const uint8_t *ptr) +{ + unsigned int read = *ptr++; + unsigned int val = read & 63; + unsigned int shift = 0; + while (read & 64) { + read = *ptr++; + shift += 6; + val |= (read & 63) << shift; + } + return val; +} + +static int +scan_signed_varint(const uint8_t *ptr) +{ + unsigned int uval = scan_varint(ptr); + if (uval & 1) { + return -(int)(uval >> 1); + } + else { + return uval >> 1; + } +} + +static int +get_line_delta(const uint8_t *ptr) +{ + int code = ((*ptr) >> 3) & 15; + switch (code) { + case PY_CODE_LOCATION_INFO_NONE: + return 0; + case PY_CODE_LOCATION_INFO_NO_COLUMNS: + case PY_CODE_LOCATION_INFO_LONG: + return scan_signed_varint(ptr+1); + case PY_CODE_LOCATION_INFO_ONE_LINE0: + return 0; + case PY_CODE_LOCATION_INFO_ONE_LINE1: + return 1; + case PY_CODE_LOCATION_INFO_ONE_LINE2: + return 2; + default: + /* Same line */ + return 0; + } +} + +static int +is_no_line_marker(uint8_t b) +{ + return (b >> 3) == 0x1f; +} + + +#define ASSERT_VALID_BOUNDS(bounds) \ + assert(bounds->opaque.lo_next <= bounds->opaque.limit && \ + (bounds->ar_line == -1 || bounds->ar_line == bounds->opaque.computed_line) && \ + (bounds->opaque.lo_next == bounds->opaque.limit || \ + (*bounds->opaque.lo_next) & 128)) + +static int +next_code_delta(PyCodeAddressRange *bounds) +{ + assert((*bounds->opaque.lo_next) & 128); + return (((*bounds->opaque.lo_next) & 7) + 1) * sizeof(_Py_CODEUNIT); +} + +static void +advance(PyCodeAddressRange *bounds) +{ + ASSERT_VALID_BOUNDS(bounds); + bounds->opaque.computed_line += get_line_delta(reinterpret_cast(bounds->opaque.lo_next)); + if (is_no_line_marker(*bounds->opaque.lo_next)) { + bounds->ar_line = -1; + } + else { + bounds->ar_line = bounds->opaque.computed_line; + } + bounds->ar_start = bounds->ar_end; + bounds->ar_end += next_code_delta(bounds); + do { + bounds->opaque.lo_next++; + } while (bounds->opaque.lo_next < bounds->opaque.limit && + ((*bounds->opaque.lo_next) & 128) == 0); + ASSERT_VALID_BOUNDS(bounds); +} + +static inline int +at_end(PyCodeAddressRange *bounds) { + return bounds->opaque.lo_next >= bounds->opaque.limit; +} + +int +_PyLineTable_NextAddressRange(PyCodeAddressRange *range) +{ + if (at_end(range)) { + return 0; + } + advance(range); + assert(range->ar_end > range->ar_start); + return 1; +} +#elif PY_VERSION_HEX >= 0x030A0000 +void +_PyLineTable_InitAddressRange(const char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range) +{ + range->opaque.lo_next = linetable; + range->opaque.limit = range->opaque.lo_next + length; + range->ar_start = -1; + range->ar_end = 0; + range->opaque.computed_line = firstlineno; + range->ar_line = -1; +} + +static void +advance(PyCodeAddressRange *bounds) +{ + bounds->ar_start = bounds->ar_end; + int delta = ((unsigned char *)bounds->opaque.lo_next)[0]; + bounds->ar_end += delta; + int ldelta = ((signed char *)bounds->opaque.lo_next)[1]; + bounds->opaque.lo_next += 2; + if (ldelta == -128) { + bounds->ar_line = -1; + } + else { + bounds->opaque.computed_line += ldelta; + bounds->ar_line = bounds->opaque.computed_line; + } +} + +static inline int +at_end(PyCodeAddressRange *bounds) { + return bounds->opaque.lo_next >= bounds->opaque.limit; +} + +int +_PyLineTable_NextAddressRange(PyCodeAddressRange *range) +{ + if (at_end(range)) { + return 0; + } + advance(range); + while (range->ar_start == range->ar_end) { + assert(!at_end(range)); + advance(range); + } + return 1; +} +#endif + +#endif // DEVTOOLS_CDBG_DEBUGLETS_PYTHON_PYLINETABLE_H_ diff --git a/tests/cpp/BUILD b/tests/cpp/BUILD new file mode 100644 index 0000000..7536fe9 --- /dev/null +++ b/tests/cpp/BUILD @@ -0,0 +1,10 @@ +package(default_visibility = ["//visibility:public"]) + +cc_test( + name = "bytecode_manipulator_test", + srcs = ["bytecode_manipulator_test.cc"], + deps = [ + "//src/googleclouddebugger:bytecode_manipulator", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/tests/cpp/bytecode_manipulator_test.cc b/tests/cpp/bytecode_manipulator_test.cc new file mode 100644 index 0000000..934dfef --- /dev/null +++ b/tests/cpp/bytecode_manipulator_test.cc @@ -0,0 +1,1059 @@ +/** + * Copyright 2023 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "src/googleclouddebugger/bytecode_manipulator.h" + +#include +#include + +namespace devtools { +namespace cdbg { + +static std::string FormatOpcode(uint8_t opcode) { + switch (opcode) { + case POP_TOP: return "POP_TOP"; + case ROT_TWO: return "ROT_TWO"; + case ROT_THREE: return "ROT_THREE"; + case DUP_TOP: return "DUP_TOP"; + case NOP: return "NOP"; + case UNARY_POSITIVE: return "UNARY_POSITIVE"; + case UNARY_NEGATIVE: return "UNARY_NEGATIVE"; + case UNARY_NOT: return "UNARY_NOT"; + case UNARY_INVERT: return "UNARY_INVERT"; + case BINARY_POWER: return "BINARY_POWER"; + case BINARY_MULTIPLY: return "BINARY_MULTIPLY"; + case BINARY_MODULO: return "BINARY_MODULO"; + case BINARY_ADD: return "BINARY_ADD"; + case BINARY_SUBTRACT: return "BINARY_SUBTRACT"; + case BINARY_SUBSCR: return "BINARY_SUBSCR"; + case BINARY_FLOOR_DIVIDE: return "BINARY_FLOOR_DIVIDE"; + case BINARY_TRUE_DIVIDE: return "BINARY_TRUE_DIVIDE"; + case INPLACE_FLOOR_DIVIDE: return "INPLACE_FLOOR_DIVIDE"; + case INPLACE_TRUE_DIVIDE: return "INPLACE_TRUE_DIVIDE"; + case INPLACE_ADD: return "INPLACE_ADD"; + case INPLACE_SUBTRACT: return "INPLACE_SUBTRACT"; + case INPLACE_MULTIPLY: return "INPLACE_MULTIPLY"; + case INPLACE_MODULO: return "INPLACE_MODULO"; + case STORE_SUBSCR: return "STORE_SUBSCR"; + case DELETE_SUBSCR: return "DELETE_SUBSCR"; + case BINARY_LSHIFT: return "BINARY_LSHIFT"; + case BINARY_RSHIFT: return "BINARY_RSHIFT"; + case BINARY_AND: return "BINARY_AND"; + case BINARY_XOR: return "BINARY_XOR"; + case BINARY_OR: return "BINARY_OR"; + case INPLACE_POWER: return "INPLACE_POWER"; + case GET_ITER: return "GET_ITER"; + case PRINT_EXPR: return "PRINT_EXPR"; + case INPLACE_LSHIFT: return "INPLACE_LSHIFT"; + case INPLACE_RSHIFT: return "INPLACE_RSHIFT"; + case INPLACE_AND: return "INPLACE_AND"; + case INPLACE_XOR: return "INPLACE_XOR"; + case INPLACE_OR: return "INPLACE_OR"; + case RETURN_VALUE: return "RETURN_VALUE"; + case IMPORT_STAR: return "IMPORT_STAR"; + case YIELD_VALUE: return "YIELD_VALUE"; + case POP_BLOCK: return "POP_BLOCK"; +#if PY_VERSION_HEX <= 0x03080000 + case END_FINALLY: return "END_FINALLY"; +#endif + case STORE_NAME: return "STORE_NAME"; + case DELETE_NAME: return "DELETE_NAME"; + case UNPACK_SEQUENCE: return "UNPACK_SEQUENCE"; + case FOR_ITER: return "FOR_ITER"; + case LIST_APPEND: return "LIST_APPEND"; + case STORE_ATTR: return "STORE_ATTR"; + case DELETE_ATTR: return "DELETE_ATTR"; + case STORE_GLOBAL: return "STORE_GLOBAL"; + case DELETE_GLOBAL: return "DELETE_GLOBAL"; + case LOAD_CONST: return "LOAD_CONST"; + case LOAD_NAME: return "LOAD_NAME"; + case BUILD_TUPLE: return "BUILD_TUPLE"; + case BUILD_LIST: return "BUILD_LIST"; + case BUILD_SET: return "BUILD_SET"; + case BUILD_MAP: return "BUILD_MAP"; + case LOAD_ATTR: return "LOAD_ATTR"; + case COMPARE_OP: return "COMPARE_OP"; + case IMPORT_NAME: return "IMPORT_NAME"; + case IMPORT_FROM: return "IMPORT_FROM"; + case JUMP_FORWARD: return "JUMP_FORWARD"; + case JUMP_IF_FALSE_OR_POP: return "JUMP_IF_FALSE_OR_POP"; + case JUMP_IF_TRUE_OR_POP: return "JUMP_IF_TRUE_OR_POP"; + case JUMP_ABSOLUTE: return "JUMP_ABSOLUTE"; + case POP_JUMP_IF_FALSE: return "POP_JUMP_IF_FALSE"; + case POP_JUMP_IF_TRUE: return "POP_JUMP_IF_TRUE"; + case LOAD_GLOBAL: return "LOAD_GLOBAL"; + case SETUP_FINALLY: return "SETUP_FINALLY"; + case LOAD_FAST: return "LOAD_FAST"; + case STORE_FAST: return "STORE_FAST"; + case DELETE_FAST: return "DELETE_FAST"; + case RAISE_VARARGS: return "RAISE_VARARGS"; + case CALL_FUNCTION: return "CALL_FUNCTION"; + case MAKE_FUNCTION: return "MAKE_FUNCTION"; + case BUILD_SLICE: return "BUILD_SLICE"; + case LOAD_CLOSURE: return "LOAD_CLOSURE"; + case LOAD_DEREF: return "LOAD_DEREF"; + case STORE_DEREF: return "STORE_DEREF"; + case CALL_FUNCTION_KW: return "CALL_FUNCTION_KW"; + case SETUP_WITH: return "SETUP_WITH"; + case EXTENDED_ARG: return "EXTENDED_ARG"; + case SET_ADD: return "SET_ADD"; + case MAP_ADD: return "MAP_ADD"; +#if PY_VERSION_HEX < 0x03080000 + case BREAK_LOOP: return "BREAK_LOOP"; + case CONTINUE_LOOP: return "CONTINUE_LOOP"; + case SETUP_LOOP: return "SETUP_LOOP"; + case SETUP_EXCEPT: return "SETUP_EXCEPT"; +#endif + case DUP_TOP_TWO: return "DUP_TOP_TWO"; + case BINARY_MATRIX_MULTIPLY: return "BINARY_MATRIX_MULTIPLY"; + case INPLACE_MATRIX_MULTIPLY: return "INPLACE_MATRIX_MULTIPLY"; + case GET_AITER: return "GET_AITER"; + case GET_ANEXT: return "GET_ANEXT"; + case BEFORE_ASYNC_WITH: return "BEFORE_ASYNC_WITH"; + case GET_YIELD_FROM_ITER: return "GET_YIELD_FROM_ITER"; + case LOAD_BUILD_CLASS: return "LOAD_BUILD_CLASS"; + case YIELD_FROM: return "YIELD_FROM"; + case GET_AWAITABLE: return "GET_AWAITABLE"; +#if PY_VERSION_HEX <= 0x03080000 + case WITH_CLEANUP_START: return "WITH_CLEANUP_START"; + case WITH_CLEANUP_FINISH: return "WITH_CLEANUP_FINISH"; +#endif + case SETUP_ANNOTATIONS: return "SETUP_ANNOTATIONS"; + case POP_EXCEPT: return "POP_EXCEPT"; + case UNPACK_EX: return "UNPACK_EX"; +#if PY_VERSION_HEX < 0x03070000 + case STORE_ANNOTATION: return "STORE_ANNOTATION"; +#endif + case CALL_FUNCTION_EX: return "CALL_FUNCTION_EX"; + case LOAD_CLASSDEREF: return "LOAD_CLASSDEREF"; +#if PY_VERSION_HEX <= 0x03080000 + case BUILD_LIST_UNPACK: return "BUILD_LIST_UNPACK"; + case BUILD_MAP_UNPACK: return "BUILD_MAP_UNPACK"; + case BUILD_MAP_UNPACK_WITH_CALL: return "BUILD_MAP_UNPACK_WITH_CALL"; + case BUILD_TUPLE_UNPACK: return "BUILD_TUPLE_UNPACK"; + case BUILD_SET_UNPACK: return "BUILD_SET_UNPACK"; +#endif + case SETUP_ASYNC_WITH: return "SETUP_ASYNC_WITH"; + case FORMAT_VALUE: return "FORMAT_VALUE"; + case BUILD_CONST_KEY_MAP: return "BUILD_CONST_KEY_MAP"; + case BUILD_STRING: return "BUILD_STRING"; +#if PY_VERSION_HEX <= 0x03080000 + case BUILD_TUPLE_UNPACK_WITH_CALL: return "BUILD_TUPLE_UNPACK_WITH_CALL"; +#endif +#if PY_VERSION_HEX >= 0x03070000 + case LOAD_METHOD: return "LOAD_METHOD"; + case CALL_METHOD: return "CALL_METHOD"; +#endif +#if PY_VERSION_HEX >= 0x03080000 && PY_VERSION_HEX < 0x03090000 + case BEGIN_FINALLY: return "BEGIN_FINALLY": + case POP_FINALLY: return "POP_FINALLY"; +#endif +#if PY_VERSION_HEX >= 0x03080000 + case ROT_FOUR: return "ROT_FOUR"; + case END_ASYNC_FOR: return "END_ASYNC_FOR"; +#endif +#if PY_VERSION_HEX >= 0x03080000 && PY_VERSION_HEX < 0x03090000 + // Added in Python 3.8 and removed in 3.9 + case CALL_FINALLY: return "CALL_FINALLY"; +#endif +#if PY_VERSION_HEX >= 0x03090000 + case RERAISE: return "RERAISE"; + case WITH_EXCEPT_START: return "WITH_EXCEPT_START"; + case LOAD_ASSERTION_ERROR: return "LOAD_ASSERTION_ERROR"; + case LIST_TO_TUPLE: return "LIST_TO_TUPLE"; + case IS_OP: return "IS_OP"; + case CONTAINS_OP: return "CONTAINS_OP"; + case JUMP_IF_NOT_EXC_MATCH: return "JUMP_IF_NOT_EXC_MATCH"; + case LIST_EXTEND: return "LIST_EXTEND"; + case SET_UPDATE: return "SET_UPDATE"; + case DICT_MERGE: return "DICT_MERGE"; + case DICT_UPDATE: return "DICT_UPDATE"; +#endif + + default: return std::to_string(static_cast(opcode)); + } +} + +static std::string FormatBytecode(const std::vector& bytecode, + int indent) { + std::string rc; + int remaining_argument_bytes = 0; + for (auto it = bytecode.begin(); it != bytecode.end(); ++it) { + std::string line; + if (remaining_argument_bytes == 0) { + line = FormatOpcode(*it); + remaining_argument_bytes = 1; + } else { + line = std::to_string(static_cast(*it)); + --remaining_argument_bytes; + } + + if (it < bytecode.end() - 1) { + line += ','; + } + + line.resize(20, ' '); + line += "// offset "; + line += std::to_string(it - bytecode.begin()); + line += '.'; + + rc += std::string(indent, ' '); + rc += line; + + if (it < bytecode.end() - 1) { + rc += '\n'; + } + } + + return rc; +} + +static void VerifyBytecode(const BytecodeManipulator& bytecode_manipulator, + std::vector expected_bytecode) { + EXPECT_EQ(expected_bytecode, bytecode_manipulator.bytecode()) + << "Actual bytecode:\n" + << " {\n" + << FormatBytecode(bytecode_manipulator.bytecode(), 10) << "\n" + << " }"; +} + +static void VerifyLineNumbersTable( + const BytecodeManipulator& bytecode_manipulator, + std::vector expected_linedata) { + // Convert to integers to better logging by EXPECT_EQ. + std::vector expected(expected_linedata.begin(), expected_linedata.end()); + std::vector actual( + bytecode_manipulator.linedata().begin(), + bytecode_manipulator.linedata().end()); + + EXPECT_EQ(expected, actual); +} + +TEST(BytecodeManipulatorTest, EmptyBytecode) { + BytecodeManipulator instance({}, false, {}); + EXPECT_FALSE(instance.InjectMethodCall(0, 0)); +} + + +TEST(BytecodeManipulatorTest, HasLineNumbersTable) { + BytecodeManipulator instance1({}, false, {}); + EXPECT_FALSE(instance1.has_linedata()); + + BytecodeManipulator instance2({}, true, {}); + EXPECT_TRUE(instance2.has_linedata()); +} + + + + +TEST(BytecodeManipulatorTest, InsertionSimple) { + BytecodeManipulator instance({ NOP, 0, RETURN_VALUE, 0 }, false, {}); + ASSERT_TRUE(instance.InjectMethodCall(2, 47)); + + VerifyBytecode( + instance, + { + NOP, // offset 0. + 0, // offset 1. + LOAD_CONST, // offset 4. + 47, // offset 5. + CALL_FUNCTION, // offset 6. + 0, // offset 7. + POP_TOP, // offset 8. + 0, // offset 9. + RETURN_VALUE, // offset 10. + 0 // offset 11. + }); +} + + +TEST(BytecodeManipulatorTest, InsertionExtended) { + BytecodeManipulator instance({ NOP, 0, RETURN_VALUE, 0 }, false, {}); + ASSERT_TRUE(instance.InjectMethodCall(2, 0x12345678)); + + VerifyBytecode( + instance, + { + NOP, // offset 0. + 0, // offset 1. + EXTENDED_ARG, // offset 2. + 0x12, // offset 3. + EXTENDED_ARG, // offset 2. + 0x34, // offset 3. + EXTENDED_ARG, // offset 2. + 0x56, // offset 3. + LOAD_CONST, // offset 4. + 0x78, // offset 5. + CALL_FUNCTION, // offset 6. + 0, // offset 7. + POP_TOP, // offset 8. + 0, // offset 9. + RETURN_VALUE, // offset 10. + 0 // offset 11. + }); +} + + +TEST(BytecodeManipulatorTest, InsertionBeginning) { + BytecodeManipulator instance({ NOP, 0, RETURN_VALUE, 0 }, false, {}); + ASSERT_TRUE(instance.InjectMethodCall(0, 47)); + + VerifyBytecode( + instance, + { + LOAD_CONST, // offset 0. + 47, // offset 1. + CALL_FUNCTION, // offset 2. + 0, // offset 3. + POP_TOP, // offset 4. + 0, // offset 5. + NOP, // offset 6. + 0, // offset 7. + RETURN_VALUE, // offset 8. + 0 // offset 9. + }); +} + + +TEST(BytecodeManipulatorTest, InsertionOffsetUpdates) { + BytecodeManipulator instance( + { + JUMP_FORWARD, + 12, + NOP, + 0, + JUMP_ABSOLUTE, + 34, + }, + false, + {}); + ASSERT_TRUE(instance.InjectMethodCall(2, 47)); + +#if PY_VERSION_HEX >= 0x030A0000 + // Jump offsets are instruction offsets, not byte offsets. + VerifyBytecode( + instance, + { + JUMP_FORWARD, // offset 0. + 12 + 3, // offset 1. + LOAD_CONST, // offset 2. + 47, // offset 3. + CALL_FUNCTION, // offset 4. + 0, // offset 5. + POP_TOP, // offset 6. + 0, // offset 7. + NOP, // offset 8. + 0, // offset 9. + JUMP_ABSOLUTE, // offset 10. + 34 + 3 // offset 11. + }); +#else + VerifyBytecode( + instance, + { + JUMP_FORWARD, // offset 0. + 12 + 6, // offset 1. + LOAD_CONST, // offset 2. + 47, // offset 3. + CALL_FUNCTION, // offset 4. + 0, // offset 5. + POP_TOP, // offset 6. + 0, // offset 7. + NOP, // offset 8. + 0, // offset 9. + JUMP_ABSOLUTE, // offset 10. + 34 + 6 // offset 11. + }); +#endif +} + + +TEST(BytecodeManipulatorTest, InsertionExtendedOffsetUpdates) { + BytecodeManipulator instance( + { + EXTENDED_ARG, + 12, + EXTENDED_ARG, + 34, + EXTENDED_ARG, + 56, + JUMP_FORWARD, + 78, + NOP, + 0, + EXTENDED_ARG, + 98, + EXTENDED_ARG, + 76, + EXTENDED_ARG, + 54, + JUMP_ABSOLUTE, + 32 + }, + false, + {}); + ASSERT_TRUE(instance.InjectMethodCall(8, 11)); + +#if PY_VERSION_HEX >= 0x030A0000 + // Jump offsets are instruction offsets, not byte offsets. + VerifyBytecode( + instance, + { + EXTENDED_ARG, // offset 0. + 12, // offset 1. + EXTENDED_ARG, // offset 2. + 34, // offset 3. + EXTENDED_ARG, // offset 4. + 56, // offset 5. + JUMP_FORWARD, // offset 6. + 78 + 3, // offset 7. + LOAD_CONST, // offset 8. + 11, // offset 9. + CALL_FUNCTION, // offset 10. + 0, // offset 11. + POP_TOP, // offset 12. + 0, // offset 13. + NOP, // offset 14. + 0, // offset 15. + EXTENDED_ARG, // offset 16. + 98, // offset 17. + EXTENDED_ARG, // offset 18. + 76, // offset 19. + EXTENDED_ARG, // offset 20. + 54, // offset 21. + JUMP_ABSOLUTE, // offset 22. + 32 + 3 // offset 23. + }); +#else + VerifyBytecode( + instance, + { + EXTENDED_ARG, // offset 0. + 12, // offset 1. + EXTENDED_ARG, // offset 2. + 34, // offset 3. + EXTENDED_ARG, // offset 4. + 56, // offset 5. + JUMP_FORWARD, // offset 6. + 78 + 6, // offset 7. + LOAD_CONST, // offset 8. + 11, // offset 9. + CALL_FUNCTION, // offset 10. + 0, // offset 11. + POP_TOP, // offset 12. + 0, // offset 13. + NOP, // offset 14. + 0, // offset 15. + EXTENDED_ARG, // offset 16. + 98, // offset 17. + EXTENDED_ARG, // offset 18. + 76, // offset 19. + EXTENDED_ARG, // offset 20. + 54, // offset 21. + JUMP_ABSOLUTE, // offset 22. + 32 + 6 // offset 23. + }); +#endif +} + + +TEST(BytecodeManipulatorTest, InsertionDeltaOffsetNoUpdate) { + BytecodeManipulator instance( + { + JUMP_FORWARD, + 2, + NOP, + 0, + RETURN_VALUE, + 0, + JUMP_FORWARD, + 2, + }, + false, {}); + ASSERT_TRUE(instance.InjectMethodCall(4, 99)); + + VerifyBytecode( + instance, + { + JUMP_FORWARD, // offset 0. + 2, // offset 1. + NOP, // offset 2. + 0, // offset 3. + LOAD_CONST, // offset 4. + 99, // offset 5. + CALL_FUNCTION, // offset 6. + 0, // offset 7. + POP_TOP, // offset 8. + 0, // offset 9. + RETURN_VALUE, // offset 10. + 0, // offset 11. + JUMP_FORWARD, // offset 12. + 2 // offset 13. + }); +} + + +TEST(BytecodeManipulatorTest, InsertionAbsoluteOffsetNoUpdate) { + BytecodeManipulator instance( + { + JUMP_ABSOLUTE, + 2, + RETURN_VALUE, + 0 + }, + false, + {}); + ASSERT_TRUE(instance.InjectMethodCall(2, 99)); + + VerifyBytecode( + instance, + { + JUMP_ABSOLUTE, // offset 0. + 2, // offset 1. + LOAD_CONST, // offset 2. + 99, // offset 3. + CALL_FUNCTION, // offset 4. + 0, // offset 5. + POP_TOP, // offset 6. + 0, // offset 7. + RETURN_VALUE, // offset 8. + 0 // offset 9. + }); +} + + +TEST(BytecodeManipulatorTest, InsertionOffsetUneededExtended) { + BytecodeManipulator instance( + { EXTENDED_ARG, 0, JUMP_FORWARD, 2, NOP, 0 }, + false, + {}); + ASSERT_TRUE(instance.InjectMethodCall(4, 11)); + +#if PY_VERSION_HEX >= 0x030A0000 + // Jump offsets are instruction offsets, not byte offsets. + VerifyBytecode( + instance, + { + EXTENDED_ARG, // offset 0. + 0, // offset 1. + JUMP_FORWARD, // offset 2. + 2 + 3, // offset 3. + LOAD_CONST, // offset 4. + 11, // offset 5. + CALL_FUNCTION, // offset 6. + 0, // offset 7. + POP_TOP, // offset 8. + 0, // offset 9. + NOP, // offset 10. + 0 // offset 11. + }); +#else + VerifyBytecode( + instance, + { + EXTENDED_ARG, // offset 0. + 0, // offset 1. + JUMP_FORWARD, // offset 2. + 2 + 6, // offset 3. + LOAD_CONST, // offset 4. + 11, // offset 5. + CALL_FUNCTION, // offset 6. + 0, // offset 7. + POP_TOP, // offset 8. + 0, // offset 9. + NOP, // offset 10. + 0 // offset 11. + }); +#endif +} + + +TEST(BytecodeManipulatorTest, InsertionOffsetUpgradeExtended) { + BytecodeManipulator instance({ JUMP_ABSOLUTE, 254 , NOP, 0 }, false, {}); + ASSERT_TRUE(instance.InjectMethodCall(2, 11)); + +#if PY_VERSION_HEX >= 0x030A0000 + // Jump offsets are instruction offsets, not byte offsets. + VerifyBytecode( + instance, + { + EXTENDED_ARG, // offset 0. + 1, // offset 1. + JUMP_ABSOLUTE, // offset 2. + 2, // offset 3. + LOAD_CONST, // offset 4. + 11, // offset 5. + CALL_FUNCTION, // offset 6. + 0, // offset 7. + POP_TOP, // offset 8. + 0, // offset 9. + NOP, // offset 10. + 0 // offset 11. + }); +#else + VerifyBytecode( + instance, + { + EXTENDED_ARG, // offset 0. + 1, // offset 1. + JUMP_ABSOLUTE, // offset 2. + 6, // offset 3. + LOAD_CONST, // offset 4. + 11, // offset 5. + CALL_FUNCTION, // offset 6. + 0, // offset 7. + POP_TOP, // offset 8. + 0, // offset 9. + NOP, // offset 10. + 0 // offset 11. + }); +#endif +} + + +TEST(BytecodeManipulatorTest, InsertionOffsetUpgradeExtendedTwice) { + BytecodeManipulator instance( + { JUMP_ABSOLUTE, 252, JUMP_ABSOLUTE, 254, NOP, 0 }, + false, + {}); + ASSERT_TRUE(instance.InjectMethodCall(4, 12)); + +#if PY_VERSION_HEX >= 0x030A0000 + // Jump offsets are instruction offsets, not byte offsets. + VerifyBytecode( + instance, + { + EXTENDED_ARG, // offset 0. + 1, // offset 1. + JUMP_ABSOLUTE, // offset 2. + 1, // offset 3. + EXTENDED_ARG, // offset 4. + 1, // offset 5. + JUMP_ABSOLUTE, // offset 6. + 3, // offset 7. + LOAD_CONST, // offset 8. + 12, // offset 9. + CALL_FUNCTION, // offset 10. + 0, // offset 11. + POP_TOP, // offset 12. + 0, // offset 13. + NOP, // offset 14. + 0 // offset 15. + }); +#else + VerifyBytecode( + instance, + { + EXTENDED_ARG, // offset 0. + 1, // offset 1. + JUMP_ABSOLUTE, // offset 2. + 6, // offset 3. + EXTENDED_ARG, // offset 4. + 1, // offset 5. + JUMP_ABSOLUTE, // offset 6. + 8, // offset 7. + LOAD_CONST, // offset 8. + 12, // offset 9. + CALL_FUNCTION, // offset 10. + 0, // offset 11. + POP_TOP, // offset 12. + 0, // offset 13. + NOP, // offset 14. + 0 // offset 15. + }); +#endif +} + + +TEST(BytecodeManipulatorTest, InsertionBadInstruction) { + BytecodeManipulator instance( + { NOP, 0, NOP, 0, LOAD_CONST }, + false, + {}); + EXPECT_FALSE(instance.InjectMethodCall(2, 0)); +} + + +TEST(BytecodeManipulatorTest, InsertionNegativeOffset) { + BytecodeManipulator instance({ NOP, 0, RETURN_VALUE, 0 }, false, {}); + EXPECT_FALSE(instance.InjectMethodCall(-1, 0)); +} + + +TEST(BytecodeManipulatorTest, InsertionOutOfRangeOffset) { + BytecodeManipulator instance({ NOP, 0, RETURN_VALUE, 0 }, false, {}); + EXPECT_FALSE(instance.InjectMethodCall(4, 0)); +} + + +TEST(BytecodeManipulatorTest, InsertionMidInstruction) { + BytecodeManipulator instance( + { NOP, 0, LOAD_CONST, 0, NOP, 0 }, + false, + {}); + + EXPECT_FALSE(instance.InjectMethodCall(1, 0)); + EXPECT_FALSE(instance.InjectMethodCall(3, 0)); + EXPECT_FALSE(instance.InjectMethodCall(5, 0)); +} + + +TEST(BytecodeManipulatorTest, InsertionTooManyUpgrades) { + BytecodeManipulator instance( + { + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + JUMP_ABSOLUTE, 254, + NOP, 0 + }, + false, + {}); + EXPECT_FALSE(instance.InjectMethodCall(20, 0)); +} + + +TEST(BytecodeManipulatorTest, IncompleteBytecodeInsert) { + BytecodeManipulator instance({ NOP, 0, LOAD_CONST }, false, {}); + EXPECT_FALSE(instance.InjectMethodCall(2, 0)); +} + + +TEST(BytecodeManipulatorTest, IncompleteBytecodeAppend) { + BytecodeManipulator instance( + { YIELD_VALUE, 0, NOP, 0, LOAD_CONST }, + false, {}); + EXPECT_FALSE(instance.InjectMethodCall(4, 0)); +} + + +TEST(BytecodeManipulatorTest, LineNumbersTableUpdateBeginning) { + BytecodeManipulator instance( + { NOP, 0, RETURN_VALUE, 0 }, + true, + { 2, 1, 2, 1 }); + ASSERT_TRUE(instance.InjectMethodCall(0, 99)); + + VerifyLineNumbersTable(instance, { 8, 1, 2, 1 }); +} + + +TEST(BytecodeManipulatorTest, LineNumbersTableUpdateLineBoundary) { + BytecodeManipulator instance( + { NOP, 0, RETURN_VALUE, 0 }, + true, + { 0, 1, 2, 1, 2, 1 }); + ASSERT_TRUE(instance.InjectMethodCall(2, 99)); + + VerifyLineNumbersTable(instance, { 0, 1, 2, 1, 8, 1 }); +} + + +TEST(BytecodeManipulatorTest, LineNumbersTableUpdateMidLine) { + BytecodeManipulator instance( + { NOP, 0, NOP, 0, RETURN_VALUE, 0 }, + true, + { 0, 1, 4, 1 }); + ASSERT_TRUE(instance.InjectMethodCall(2, 99)); + + VerifyLineNumbersTable(instance, { 0, 1, 10, 1 }); +} + + +TEST(BytecodeManipulatorTest, LineNumbersTablePastEnd) { + BytecodeManipulator instance( + { NOP, 0, NOP, 0, NOP, 0, RETURN_VALUE, 0 }, + true, + { 0, 1 }); + ASSERT_TRUE(instance.InjectMethodCall(6, 99)); + + VerifyLineNumbersTable(instance, { 0, 1 }); +} + + +TEST(BytecodeManipulatorTest, LineNumbersTableUpgradeExtended) { + BytecodeManipulator instance( + { JUMP_ABSOLUTE, 254, RETURN_VALUE, 0 }, + true, + { 2, 1, 2, 1 }); + ASSERT_TRUE(instance.InjectMethodCall(2, 99)); + + VerifyLineNumbersTable(instance, { 4, 1, 8, 1 }); +} + + +TEST(BytecodeManipulatorTest, LineNumbersTableOverflow) { + std::vector bytecode(300, 0); + BytecodeManipulator instance( + bytecode, + true, + { 254, 1 }); + ASSERT_TRUE(instance.InjectMethodCall(2, 99)); + +#if PY_VERSION_HEX >= 0x030A0000 + VerifyLineNumbersTable(instance, { 254, 0, 6, 1 }); +#else + VerifyLineNumbersTable(instance, { 255, 0, 5, 1 }); +#endif +} + + +TEST(BytecodeManipulatorTest, SuccessAppend) { + BytecodeManipulator instance( + { YIELD_VALUE, 0, LOAD_CONST, 0, NOP, 0 }, + false, + {}); + ASSERT_TRUE(instance.InjectMethodCall(2, 57)); + + VerifyBytecode( + instance, + { + YIELD_VALUE, // offset 0. + 0, // offset 1. + JUMP_ABSOLUTE, // offset 2. + 6, // offset 3. + NOP, // offset 4. + 0, // offset 5. + LOAD_CONST, // offset 6. + 57, // offset 7. + CALL_FUNCTION, // offset 8. + 0, // offset 9. + POP_TOP, // offset 10. + 0, // offset 11. + LOAD_CONST, // offset 12. + 0, // offset 13. + JUMP_ABSOLUTE, // offset 14. + 4 // offset 15. + }); +} + + +TEST(BytecodeManipulatorTest, SuccessAppendYieldFrom) { + BytecodeManipulator instance( + { YIELD_FROM, 0, LOAD_CONST, 0, NOP, 0 }, + false, + {}); + ASSERT_TRUE(instance.InjectMethodCall(2, 57)); + + VerifyBytecode( + instance, + { + YIELD_FROM, // offset 0. + 0, // offset 1. + JUMP_ABSOLUTE, // offset 2. + 6, // offset 3. + NOP, // offset 4. + 0, // offset 5. + LOAD_CONST, // offset 6. + 57, // offset 7. + CALL_FUNCTION, // offset 8. + 0, // offset 9. + POP_TOP, // offset 10. + 0, // offset 11. + LOAD_CONST, // offset 12. + 0, // offset 13. + JUMP_ABSOLUTE, // offset 14. + 4 // offset 15. + }); +} + + +TEST(BytecodeManipulatorTest, AppendExtraPadding) { + BytecodeManipulator instance( + { + YIELD_VALUE, + 0, + EXTENDED_ARG, + 15, + EXTENDED_ARG, + 16, + EXTENDED_ARG, + 17, + LOAD_CONST, + 18, + RETURN_VALUE, + 0 + }, + false, {}); + ASSERT_TRUE(instance.InjectMethodCall(2, 0x7273)); + + VerifyBytecode( + instance, + { + YIELD_VALUE, // offset 0. + 0, // offset 1. + JUMP_ABSOLUTE, // offset 2. + 12, // offset 3. + NOP, // offset 4. Args for NOP do not matter. + 9, // offset 5. + NOP, // offset 6. + 9, // offset 7. + NOP, // offset 8. + 9, // offset 9. + RETURN_VALUE, // offset 10. + 0, // offset 11. + EXTENDED_ARG, // offset 12. + 0x72, // offset 13. + LOAD_CONST, // offset 14. + 0x73, // offset 15. + CALL_FUNCTION, // offset 16. + 0, // offset 17. + POP_TOP, // offset 18. + 0, // offset 19. + EXTENDED_ARG, // offset 20. + 15, // offset 21. + EXTENDED_ARG, // offset 22. + 16, // offset 23. + EXTENDED_ARG, // offset 24. + 17, // offset 25. + LOAD_CONST, // offset 26. + 18, // offset 27. + JUMP_ABSOLUTE, // offset 28. + 10 // offset 29. + }); +} + + +TEST(BytecodeManipulatorTest, AppendToEnd) { + std::vector bytecode = {YIELD_VALUE, 0}; + // Case where trampoline requires 4 bytes to write. + bytecode.resize(300); + BytecodeManipulator instance(bytecode, false, {}); + + // This scenario could be supported in theory, but it's not. The purpose of + // this test case is to verify there are no crashes or corruption. + ASSERT_FALSE(instance.InjectMethodCall(298, 0x12)); +} + + +TEST(BytecodeManipulatorTest, NoSpaceForTrampoline) { + const std::vector test_cases[] = { + {YIELD_VALUE, 0, YIELD_VALUE, 0, NOP, 0}, + {YIELD_VALUE, 0, FOR_ITER, 0, NOP, 0}, + {YIELD_VALUE, 0, JUMP_FORWARD, 0, NOP, 0}, +#if PY_VERSION_HEX < 0x03080000 + {YIELD_VALUE, 0, SETUP_LOOP, 0, NOP, 0}, +#endif + {YIELD_VALUE, 0, SETUP_FINALLY, 0, NOP, 0}, +#if PY_VERSION_HEX < 0x03080000 + {YIELD_VALUE, 0, SETUP_LOOP, 0, NOP, 0}, + {YIELD_VALUE, 0, SETUP_EXCEPT, 0, NOP, 0}, +#endif +#if PY_VERSION_HEX >= 0x03080000 && PY_VERSION_HEX < 0x03090000 + {YIELD_VALUE, 0, CALL_FINALLY, 0, NOP, 0}, +#endif + }; + + for (const auto& test_case : test_cases) { + BytecodeManipulator instance(test_case, false, {}); + EXPECT_FALSE(instance.InjectMethodCall(2, 0)) + << "Input:\n" + << FormatBytecode(test_case, 4) << "\n" + << "Unexpected output:\n" + << FormatBytecode(instance.bytecode(), 4); + } + + // Case where trampoline requires 4 bytes to write. + std::vector bytecode(300, 0); + bytecode[0] = YIELD_VALUE; + bytecode[2] = NOP; + bytecode[4] = YIELD_VALUE; + BytecodeManipulator instance(bytecode, false, {}); + ASSERT_FALSE(instance.InjectMethodCall(2, 0x12)); +} + +// Tests that we don't allow jumping into the middle of the space reserved for +// the trampoline. See the comments in AppendMethodCall() in +// bytecode_manipulator.cc. +TEST(BytecodeManipulatorTest, JumpMidRelocatedInstructions) { + std::vector test_cases[] = { + {YIELD_VALUE, 0, FOR_ITER, 2, LOAD_CONST, 0}, + {YIELD_VALUE, 0, JUMP_FORWARD, 2, LOAD_CONST, 0}, + {YIELD_VALUE, 0, SETUP_FINALLY, 2, LOAD_CONST, 0}, + {YIELD_VALUE, 0, SETUP_WITH, 2, LOAD_CONST, 0}, + {YIELD_VALUE, 0, SETUP_FINALLY, 2, LOAD_CONST, 0}, + {YIELD_VALUE, 0, JUMP_IF_FALSE_OR_POP, 6, LOAD_CONST, 0}, + {YIELD_VALUE, 0, JUMP_IF_TRUE_OR_POP, 6, LOAD_CONST, 0}, + {YIELD_VALUE, 0, JUMP_ABSOLUTE, 6, LOAD_CONST, 0}, + {YIELD_VALUE, 0, POP_JUMP_IF_FALSE, 6, LOAD_CONST, 0}, + {YIELD_VALUE, 0, POP_JUMP_IF_TRUE, 6, LOAD_CONST, 0}, +#if PY_VERSION_HEX < 0x03080000 + {YIELD_VALUE, 0, SETUP_LOOP, 2, LOAD_CONST, 0}, + {YIELD_VALUE, 0, CONTINUE_LOOP, 6, LOAD_CONST, 0}, +#endif + }; + + for (auto& test_case : test_cases) { + // Case where trampoline requires 4 bytes to write. + test_case.resize(300); + BytecodeManipulator instance(test_case, false, {}); + EXPECT_FALSE(instance.InjectMethodCall(4, 0)) + << "Input:\n" + << FormatBytecode(test_case, 4) << "\n" + << "Unexpected output:\n" + << FormatBytecode(instance.bytecode(), 4); + } +} + + +// Test that we allow jumping to the start of the space reserved for the +// trampoline. +TEST(BytecodeManipulatorTest, JumpStartOfRelocatedInstructions) { + const std::vector test_cases[] = { + {YIELD_VALUE, 0, FOR_ITER, 0, LOAD_CONST, 0}, + {YIELD_VALUE, 0, SETUP_WITH, 0, LOAD_CONST, 0}, + {YIELD_VALUE, 0, JUMP_ABSOLUTE, 4, LOAD_CONST, 0}}; + + for (const auto& test_case : test_cases) { + BytecodeManipulator instance(test_case, false, {}); + EXPECT_TRUE(instance.InjectMethodCall(4, 0)) + << "Input:\n" << FormatBytecode(test_case, 4); + } +} + + +// Test that we allow jumping after the space reserved for the trampoline. +TEST(BytecodeManipulatorTest, JumpAfterRelocatedInstructions) { + const std::vector test_cases[] = { + {YIELD_VALUE, 0, FOR_ITER, 2, LOAD_CONST, 0, NOP, 0}, + {YIELD_VALUE, 0, SETUP_WITH, 2, LOAD_CONST, 0, NOP, 0}, + {YIELD_VALUE, 0, JUMP_ABSOLUTE, 6, LOAD_CONST, 0, NOP, 0}}; + + for (const auto& test_case : test_cases) { + BytecodeManipulator instance(test_case, false, {}); + EXPECT_TRUE(instance.InjectMethodCall(4, 0)) + << "Input:\n" << FormatBytecode(test_case, 4); + } +} + + +TEST(BytecodeManipulatorTest, InsertionRevertOnFailure) { + const std::vector input{JUMP_FORWARD, 0, NOP, 0, JUMP_ABSOLUTE, 2}; + + BytecodeManipulator instance(input, false, {}); + ASSERT_FALSE(instance.InjectMethodCall(1, 47)); + + VerifyBytecode(instance, input); +} + + +} // namespace cdbg +} // namespace devtools diff --git a/tests/application_info_test.py b/tests/py/application_info_test.py similarity index 100% rename from tests/application_info_test.py rename to tests/py/application_info_test.py diff --git a/tests/backoff_test.py b/tests/py/backoff_test.py similarity index 100% rename from tests/backoff_test.py rename to tests/py/backoff_test.py diff --git a/tests/breakpoints_manager_test.py b/tests/py/breakpoints_manager_test.py similarity index 100% rename from tests/breakpoints_manager_test.py rename to tests/py/breakpoints_manager_test.py diff --git a/tests/collector_test.py b/tests/py/collector_test.py similarity index 98% rename from tests/collector_test.py rename to tests/py/collector_test.py index fe936ad..abc39b2 100644 --- a/tests/collector_test.py +++ b/tests/py/collector_test.py @@ -5,6 +5,7 @@ import inspect import logging import os +import sys import time from unittest import mock @@ -1428,7 +1429,8 @@ def testLogBytesQuota(self): def testMissingLogLevel(self): # Missing is equivalent to INFO. - log_collector = LogCollectorWithDefaultLocation({'logMessageFormat': 'hello'}) + log_collector = LogCollectorWithDefaultLocation( + {'logMessageFormat': 'hello'}) self.assertIsNone(log_collector.Log(inspect.currentframe())) self.assertTrue(self._verifier.GotMessage('LOGPOINT: hello')) @@ -1487,11 +1489,17 @@ def testBadExpression(self): 'expressions': ['-', '+'] }) self.assertIsNone(log_collector.Log(inspect.currentframe())) - self.assertTrue( - self._verifier.GotMessage( - 'LOGPOINT: a=, b=')) + if sys.version_info.minor < 10: + self.assertTrue( + self._verifier.GotMessage( + 'LOGPOINT: a=, b=')) + else: + self.assertTrue( + self._verifier.GotMessage( + 'LOGPOINT: a=, ' + 'b=')) def testDollarEscape(self): unused_integer = 12345 diff --git a/tests/error_data_visibility_policy_test.py b/tests/py/error_data_visibility_policy_test.py similarity index 100% rename from tests/error_data_visibility_policy_test.py rename to tests/py/error_data_visibility_policy_test.py diff --git a/tests/firebase_client_test.py b/tests/py/firebase_client_test.py similarity index 100% rename from tests/firebase_client_test.py rename to tests/py/firebase_client_test.py diff --git a/tests/gcp_hub_client_test.py b/tests/py/gcp_hub_client_test.py similarity index 100% rename from tests/gcp_hub_client_test.py rename to tests/py/gcp_hub_client_test.py diff --git a/tests/glob_data_visibility_policy_test.py b/tests/py/glob_data_visibility_policy_test.py similarity index 100% rename from tests/glob_data_visibility_policy_test.py rename to tests/py/glob_data_visibility_policy_test.py diff --git a/tests/imphook_test.py b/tests/py/imphook_test.py similarity index 100% rename from tests/imphook_test.py rename to tests/py/imphook_test.py diff --git a/tests/integration_test_disabled.py b/tests/py/integration_test_disabled.py similarity index 100% rename from tests/integration_test_disabled.py rename to tests/py/integration_test_disabled.py diff --git a/tests/integration_test_helper.py b/tests/py/integration_test_helper.py similarity index 100% rename from tests/integration_test_helper.py rename to tests/py/integration_test_helper.py diff --git a/tests/labels_test.py b/tests/py/labels_test.py similarity index 100% rename from tests/labels_test.py rename to tests/py/labels_test.py diff --git a/tests/module_explorer_test_disabled.py b/tests/py/module_explorer_test_disabled.py similarity index 100% rename from tests/module_explorer_test_disabled.py rename to tests/py/module_explorer_test_disabled.py diff --git a/tests/module_search_test.py b/tests/py/module_search_test.py similarity index 97% rename from tests/module_search_test.py rename to tests/py/module_search_test.py index 70c67b4..3a12c57 100644 --- a/tests/module_search_test.py +++ b/tests/py/module_search_test.py @@ -83,8 +83,7 @@ def testSearchSymLinkInSysPath(self): # Returned result should have a successful file match and symbolic # links should be kept. - self.assertEndsWith( - module_search.Search('b/first.py'), 'link/b/first.py') + self.assertEndsWith(module_search.Search('b/first.py'), 'link/b/first.py') finally: sys.path.remove(os.path.join(self._test_package_dir, 'link')) diff --git a/tests/module_utils_test.py b/tests/py/module_utils_test.py similarity index 100% rename from tests/module_utils_test.py rename to tests/py/module_utils_test.py diff --git a/tests/native_module_test.py b/tests/py/native_module_test.py similarity index 100% rename from tests/native_module_test.py rename to tests/py/native_module_test.py diff --git a/tests/python_breakpoint_test_disabled.py b/tests/py/python_breakpoint_test_disabled.py similarity index 100% rename from tests/python_breakpoint_test_disabled.py rename to tests/py/python_breakpoint_test_disabled.py diff --git a/tests/python_test_util.py b/tests/py/python_test_util.py similarity index 100% rename from tests/python_test_util.py rename to tests/py/python_test_util.py diff --git a/tests/uniquifier_computer_test.py b/tests/py/uniquifier_computer_test.py similarity index 100% rename from tests/uniquifier_computer_test.py rename to tests/py/uniquifier_computer_test.py diff --git a/tests/yaml_data_visibility_config_reader_test.py b/tests/py/yaml_data_visibility_config_reader_test.py similarity index 100% rename from tests/yaml_data_visibility_config_reader_test.py rename to tests/py/yaml_data_visibility_config_reader_test.py