diff --git a/.cspell.json b/.cspell.json index 268e458139a..090b581c1a6 100644 --- a/.cspell.json +++ b/.cspell.json @@ -30,6 +30,7 @@ "bindgen", "cstring", "chrono", + "insta", "peekable", "lalrpop", "memmap", diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b0538abad20..a06efd52cf3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,7 +127,7 @@ jobs: echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >>$GITHUB_ENV if: runner.os == 'Windows' - name: Set up the Mac environment - run: brew install autoconf automake libtool + run: brew install autoconf automake libtool openssl@3 if: runner.os == 'macOS' - uses: Swatinem/rust-cache@v2 @@ -137,6 +137,12 @@ jobs: - name: run rust tests run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }} + if: runner.os != 'macOS' + # temp skip ssl linking for Mac to avoid CI failure + - name: run rust tests (MacOS no ssl) + run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit ${{ env.NON_WASM_PACKAGES }} + if: runner.os == 'macOS' + - name: check compilation without threading run: cargo check ${{ env.CARGO_ARGS }} @@ -252,7 +258,7 @@ jobs: echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >>$GITHUB_ENV if: runner.os == 'Windows' - name: Set up the Mac environment - run: brew install autoconf automake libtool + run: brew install autoconf automake libtool openssl@3 if: runner.os == 'macOS' - uses: Swatinem/rust-cache@v2 diff --git a/Cargo.lock b/Cargo.lock index a021269788d..0c36cd78510 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,12 +8,6 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -[[package]] -name = "abort_on_panic" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955f37ac58af2416bac687c8ab66a4ccba282229bd7422a28d2281a5e66a6116" - [[package]] name = "adler" version = "1.0.2" @@ -66,9 +60,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "approx" @@ -115,7 +109,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -164,9 +158,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "blake2" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b12e5fd123190ce1c2e559308a94c9bacad77907d4c6005d9e58fe1a0689e55e" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ "digest", ] @@ -189,14 +183,13 @@ dependencies = [ "lazy_static 1.4.0", "memchr", "regex-automata", - "serde", ] [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byteorder" @@ -243,9 +236,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.77" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -285,9 +278,9 @@ dependencies = [ [[package]] name = "clipboard-win" -version = "4.4.2" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4ab1b92798304eedc095b53942963240037c0516452cb11aeba709d420b2219" +checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" dependencies = [ "error-code", "str-buf", @@ -306,15 +299,14 @@ dependencies = [ [[package]] name = "console" -version = "0.15.2" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ "encode_unicode", "lazy_static 1.4.0", "libc", - "terminal_size", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -586,13 +578,12 @@ dependencies = [ [[package]] name = "csv" -version = "1.1.6" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +checksum = "af91f40b7355f82b0a891f50e70399475945bb0b0da4f1700ce60761c9d3e359" dependencies = [ - "bstr", "csv-core", - "itoa 0.4.8", + "itoa", "ryu", "serde", ] @@ -608,9 +599,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.82" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453" +checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" dependencies = [ "cc", "cxxbridge-flags", @@ -620,9 +611,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.82" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0" +checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" dependencies = [ "cc", "codespan-reporting", @@ -635,15 +626,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.82" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71" +checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" [[package]] name = "cxxbridge-macro" -version = "1.0.82" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" +checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ "proc-macro2", "quote", @@ -702,9 +693,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "ena" @@ -777,13 +768,13 @@ checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" [[package]] name = "fd-lock" -version = "3.0.8" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb21c69b9fea5e15dbc1049e4b77145dd0ba1c84019c488102de0dc4ea4b0a27" +checksum = "8ef1a30ae415c3a691a4f41afddc2dbcd6d70baf338368d85ebc1e8ed92cedb9" dependencies = [ "cfg-if", "rustix", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -830,9 +821,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ "crc32fast", "libz-sys", @@ -904,9 +895,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "half" @@ -922,9 +913,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -935,6 +926,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" @@ -983,9 +983,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.21.1" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1e75aa1530e7385af7b2685478dece08dafb9db3b4225c753286decea83bef" +checksum = "fea5b3894afe466b4bcf0388630fc15e11938a6074af0cd637c825ba2ec8a099" dependencies = [ "console", "lazy_static 1.4.0", @@ -996,12 +996,12 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.1" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d367024b3f3414d8e01f437f704f41a9f64ab36f9067fa73e526ad4c763c87" +checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -1028,21 +1028,15 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -1132,26 +1126,25 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.137" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libffi" -version = "2.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b05b52bd89490a0b36c56715aef46d8580d25343ed243d01337663b287004bf" +checksum = "6cb06d5b4c428f3cd682943741c39ed4157ae989fffe1094a08eaf7c4014cf60" dependencies = [ - "abort_on_panic", "libc", "libffi-sys", ] [[package]] name = "libffi-sys" -version = "1.3.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7283a0ec88c0064eb8b3e40990d2a49cdca5a207f46f678e79ea7302b335401f" +checksum = "11c6f11e063a27ffe040a9d15f0b661bf41edc2383b7ae0e0ad5a7e7d53d9da3" dependencies = [ "cc", ] @@ -1181,9 +1174,9 @@ dependencies = [ [[package]] name = "link-cplusplus" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" dependencies = [ "cc", ] @@ -1196,9 +1189,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "lock_api" @@ -1234,7 +1227,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b238e3235c8382b7653c6408ed1b08dd379bdb9fdf990fb0bbae3db2cc0ae963" dependencies = [ - "nix 0.23.1", + "nix 0.23.2", "winapi", ] @@ -1255,9 +1248,9 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "md-5" @@ -1303,9 +1296,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" dependencies = [ "adler", ] @@ -1336,9 +1329,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" dependencies = [ "bitflags", "cc", @@ -1349,14 +1342,37 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ + "autocfg", "bitflags", "cfg-if", "libc", - "memoffset 0.6.5", +] + +[[package]] +name = "nix" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +dependencies = [ + "bitflags", + "cfg-if", + "libc", + "memoffset 0.7.1", + "pin-utils", + "static_assertions", +] + +[[package]] +name = "nom8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +dependencies = [ + "memchr", ] [[package]] @@ -1373,9 +1389,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" +checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" dependencies = [ "num-traits", "serde", @@ -1414,28 +1430,28 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] [[package]] name = "num_enum" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf5395665662ef45796a4ff5486c5d41d29e0c09640af4c5f17fd94ee2c119c9" +checksum = "8d829733185c1ca374f17e52b762f24f535ec625d2cc1f070e34c8a9068f341b" dependencies = [ "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0498641e53dd6ac1a4f22547548caa6864cc4933784319cd1775271c5a46ce" +checksum = "2be1598bf1c313dcdd12092e3f1920f463462525a21b7b4e11b4168353d0123e" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1445,9 +1461,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "oorandom" @@ -1457,9 +1473,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.43" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -1489,18 +1505,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.24.0+1.1.1s" +version = "111.25.0+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd" +checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.78" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07d5c8cb6e57b3a3612064d7b18b117912b4ce70955c2504d4b741c9e244b132" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -1538,28 +1554,28 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.4" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall 0.2.16", "smallvec", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] name = "paste" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" [[package]] name = "petgraph" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", "indexmap", @@ -1618,6 +1634,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.26" @@ -1677,20 +1699,19 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "proc-macro-crate" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +checksum = "66618389e4ec1c7afe67d51a9bf34ff9236480f8d51e7489b7d5ab0303c13f34" dependencies = [ "once_cell", - "thiserror", - "toml", + "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -1713,9 +1734,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -1768,20 +1789,19 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -1829,9 +1849,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1901,16 +1921,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.3" +version = "0.36.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1fbb4dfc4eb1d390c02df47760bb19a84bb80b301ecc947ab5406394d8223e" +checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -1928,7 +1948,6 @@ dependencies = [ "flamescope", "libc", "log", - "num-traits", "python3-sys", "rustpython-compiler", "rustpython-parser", @@ -2120,7 +2139,6 @@ dependencies = [ "gethostname", "hex", "itertools", - "lexical-parse-float", "libc", "libsqlite3-sys", "libz-sys", @@ -2129,7 +2147,7 @@ dependencies = [ "memchr", "memmap2", "mt19937", - "nix 0.24.2", + "nix 0.26.2", "num-bigint", "num-complex", "num-integer", @@ -2174,7 +2192,6 @@ dependencies = [ name = "rustpython-vm" version = "0.2.0" dependencies = [ - "adler32", "ahash", "ascii", "atty", @@ -2187,12 +2204,10 @@ dependencies = [ "exitcode", "flame", "flamer", - "flate2", "getrandom", "glob", "half", "hex", - "hexf-parse", "indexmap", "is-macro", "itertools", @@ -2200,7 +2215,7 @@ dependencies = [ "log", "memchr", "memoffset 0.6.5", - "nix 0.24.2", + "nix 0.26.2", "num-bigint", "num-complex", "num-integer", @@ -2253,7 +2268,6 @@ version = "0.2.0" dependencies = [ "console_error_panic_hook", "js-sys", - "parking_lot", "rustpython-common", "rustpython-parser", "rustpython-pylib", @@ -2268,15 +2282,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" [[package]] name = "rustyline" -version = "10.0.0" +version = "10.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1cd5ae51d3f7bf65d7969d579d502168ef578f289452bd8ccc91de28fda20e" +checksum = "c1e83c32c3f3c33b08496e0d1df9ea8c64d39adb8eb36a1ebb1440c690697aef" dependencies = [ "bitflags", "cfg-if", @@ -2286,7 +2300,7 @@ dependencies = [ "libc", "log", "memchr", - "nix 0.24.2", + "nix 0.25.1", "radix_trie", "scopeguard", "unicode-segmentation", @@ -2297,9 +2311,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "same-file" @@ -2312,12 +2326,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static 1.4.0", - "windows-sys 0.36.1", + "windows-sys 0.42.0", ] [[package]] @@ -2328,21 +2341,21 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" [[package]] name = "semver" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" [[package]] name = "serde" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] @@ -2371,9 +2384,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -2382,20 +2395,20 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" dependencies = [ - "itoa 1.0.4", + "itoa", "ryu", "serde", ] [[package]] name = "sha-1" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", "cpufeatures", @@ -2526,9 +2539,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -2567,9 +2580,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d" +checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" [[package]] name = "term" @@ -2584,23 +2597,13 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "termios" version = "0.3.3" @@ -2627,18 +2630,18 @@ checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d" [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -2658,18 +2661,19 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if", "once_cell", ] [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -2712,17 +2716,25 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] -name = "toml" -version = "0.5.9" +name = "toml_datetime" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" + +[[package]] +name = "toml_edit" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" dependencies = [ - "serde", + "indexmap", + "nom8", + "toml_datetime", ] [[package]] @@ -2737,9 +2749,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "uname" @@ -2874,9 +2886,9 @@ checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -2889,9 +2901,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -2919,9 +2931,9 @@ checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" [[package]] name = "uuid" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" dependencies = [ "atomic", "getrandom", @@ -2931,9 +2943,9 @@ dependencies = [ [[package]] name = "uuid-macro-internal" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73bc89f2894593e665241e0052c3791999e6787b7c4831daa0a5c2e637e276d8" +checksum = "c1b300a878652a387d2a0de915bdae8f1a548f0c6d45e072fe2688794b656cc9" dependencies = [ "proc-macro2", "quote", @@ -2989,9 +3001,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2999,9 +3011,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -3014,9 +3026,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -3026,9 +3038,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3036,9 +3048,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -3049,15 +3061,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -3065,9 +3077,9 @@ dependencies = [ [[package]] name = "which" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", "libc", @@ -3144,19 +3156,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.1", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" @@ -3172,9 +3208,9 @@ checksum = "ec7711666096bd4096ffa835238905bb33fb87267910e154b18b44eaabb340f2" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" @@ -3190,9 +3226,9 @@ checksum = "763fc57100a5f7042e3057e7e8d9bdd7860d330070251a73d003563a3bb49e1b" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" @@ -3208,9 +3244,9 @@ checksum = "7bc7cbfe58828921e10a9f446fcaaf649204dcfe6c1ddd712c5eebae6bda1106" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" @@ -3226,15 +3262,15 @@ checksum = "6868c165637d653ae1e8dc4d82c25d4f97dd6605eaa8d784b5c6e0ab2a252b65" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" @@ -3250,9 +3286,9 @@ checksum = "5e4d40883ae9cae962787ca76ba76390ffa29214667a111db9e0a1ad8377e809" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winreg" diff --git a/Cargo.toml b/Cargo.toml index 11df8b91f4f..f305420ba9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,45 @@ members = [ ".", "common", "derive", "jit", "vm", "pylib", "stdlib", "wasm/lib", "derive-impl", ] +[workspace.dependencies] +ahash = "0.7.6" +anyhow = "1.0.45" +ascii = "1.0" +atty = "0.2.14" +bincode = "1.3.3" +bitflags = "1.3.2" +bstr = "0.2.17" +cfg-if = "1.0" +chrono = "0.4.19" +crossbeam-utils = "0.8.9" +flame = "0.2.2" +glob = "0.3" +hex = "0.4.3" +indexmap = "1.8.1" +insta = "1.14.0" +itertools = "0.10.3" +libc = "0.2.133" +log = "0.4.16" +nix = "0.26" +num-complex = "0.4.0" +num-bigint = "0.4.3" +num-integer = "0.1.44" +num-rational = "0.4.0" +num-traits = "0.2" +num_enum = "0.5.7" +once_cell = "1.13" +parking_lot = "0.12" +paste = "1.0.7" +rand = "0.8.5" +rustyline = "10.0.0" +serde = "1.0" +schannel = "0.1.19" +static_assertions = "1.1" +syn = "1.0.91" +thiserror = "1.0" +thread_local = "1.1.4" +widestring = "0.5.1" + [features] default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"] importlib = ["rustpython-vm/importlib"] @@ -39,19 +78,21 @@ rustpython-pylib = { path = "pylib", optional = true, default-features = false } rustpython-stdlib = { path = "stdlib", optional = true, default-features = false } rustpython-vm = { path = "vm", version = "0.2.0", default-features = false, features = ["compiler"] } -cfg-if = "1.0.0" +atty = { workspace = true } +cfg-if = { workspace = true } +log = { workspace = true } +flame = { workspace = true, optional = true } + clap = "2.34" dirs = { package = "dirs-next", version = "2.0.0" } env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] } -flame = { version = "0.2.2", optional = true } flamescope = { version = "0.1.2", optional = true } -libc = "0.2.133" -log = "0.4.16" -num-traits = "0.2.14" -atty = "0.2.14" + +[target.'cfg(windows)'.dependencies] +libc = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -rustyline = "10.0.0" +rustyline = { workspace = true } [dev-dependencies] cpython = "0.7.0" @@ -87,5 +128,5 @@ opt-level = 3 lto = "thin" [patch.crates-io] -# REDOX START, Uncommment when you want to compile/check with redoxer +# REDOX START, Uncomment when you want to compile/check with redoxer # REDOX END diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 9051c6f2583..0eca7759032 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -1427,8 +1427,6 @@ def test_find_etc_raise_correct_error_messages(self): class MixinStrUnicodeTest: # Additional tests that only work with str. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug1001011(self): # Make sure join returns a NEW object for single item sequences # involving a subclass. diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 2382322cd2b..61b9bce3302 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -765,16 +765,6 @@ def test_translate(self, size): self.assertEqual(s.count(_('!')), repeats * 2) self.assertEqual(s.count(_('z')), repeats * 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_lstrip(self, size): - super().test_lstrip(size) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_rstrip(self, size): - super().test_rstrip(size) - class BytesTest(unittest.TestCase, BaseStrTest): diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 74e47c8117a..6bd9d37fadc 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1954,8 +1954,6 @@ class R: self.assertEqual(new_sample.x, another_new_sample.x) self.assertEqual(sample.y, another_new_sample.y) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dataclasses_qualnames(self): @dataclass(order=True, unsafe_hash=True, frozen=True) class A: @@ -3442,8 +3440,6 @@ class C: self.assertEqual(c1.x, 3) self.assertEqual(c1.y, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_frozen(self): @dataclass(frozen=True) class C: @@ -3476,8 +3472,6 @@ class C: "keyword argument 'a'"): c1 = replace(c, x=20, a=5) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_invalid_field_name(self): @dataclass(frozen=True) class C: @@ -3521,8 +3515,6 @@ class C: with self.assertRaisesRegex(ValueError, 'init=False'): replace(c, y=30) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_classvar(self): @dataclass class C: diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index dfd4c064410..5cbf2970957 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -374,8 +374,6 @@ def test_annotation_with_complex_target(self): "object.__debug__: int" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_annotations_symbol_table_pass(self): namespace = self._exec_future(dedent(""" from __future__ import annotations diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index c27ffcf933e..4e618d1d90f 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -157,7 +157,8 @@ def test_sleep(self): self.assertRaises(ValueError, time.sleep, -1) time.sleep(1.2) - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'a Display implementation returned an error unexpectedly: Error'") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_strftime(self): tt = time.gmtime(self.t) for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I', diff --git a/common/Cargo.toml b/common/Cargo.toml index 87229dce9ab..d98ec8e78d3 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -11,24 +11,25 @@ license = "MIT" threading = ["parking_lot"] [dependencies] -ascii = "1.0" -bitflags = "1.3.2" -cfg-if = "1.0" +ascii = { workspace = true } +bitflags = { workspace = true } +cfg-if = { workspace = true } +itertools = { workspace = true } +libc = { workspace = true } +num-bigint = { workspace = true } +num-complex = { workspace = true } +num-traits = { workspace = true } +once_cell = { workspace = true } +parking_lot = { workspace = true, optional = true } +rand = { workspace = true } + hexf-parse = "0.2.1" -itertools = "0.10.3" lexical-parse-float = { version = "0.8.0", features = ["format"] } -libc = "0.2.133" lock_api = "0.4" -num-bigint = "0.4.2" -num-complex = "0.4.0" -num-traits = "0.2" -once_cell = "1.4.1" -parking_lot = { version = "0.12.0", optional = true } radium = "0.7" -rand = "0.8" siphasher = "0.3" unic-ucd-category = "0.9" volatile = "0.3" [target.'cfg(windows)'.dependencies] -widestring = "0.5.1" +widestring = { workspace = true } diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 34dbad32608..b6e204dfdc3 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -10,4 +10,4 @@ rustpython-compiler-core = { path = "core" } rustpython-codegen = { path = "codegen" } rustpython-parser = { path = "parser" } -thiserror = "1.0" +thiserror = { workspace = true } diff --git a/compiler/ast/Cargo.toml b/compiler/ast/Cargo.toml index 08f29fcc99b..bb280f48ffd 100644 --- a/compiler/ast/Cargo.toml +++ b/compiler/ast/Cargo.toml @@ -14,6 +14,7 @@ fold = [] unparse = ["rustpython-common"] [dependencies] -num-bigint = "0.4.3" rustpython-compiler-core = { path = "../core", version = "0.2.0" } rustpython-common = { path = "../../common", version = "0.2.0", optional = true } + +num-bigint = { workspace = true } diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index 4265e4b23ec..fd6ad6533fe 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -11,16 +11,16 @@ edition = "2021" rustpython-ast = { path = "../ast", features = ["unparse"] } rustpython-compiler-core = { path = "../core", version = "0.2.0" } -ahash = "0.7.6" -bitflags = "1.3.2" -indexmap = "1.8.1" -itertools = "0.10.3" -log = "0.4.16" -num-complex = { version = "0.4.0", features = ["serde"] } -num-traits = "0.2.14" -thiserror = "1.0" +ahash = { workspace = true } +bitflags = { workspace = true } +indexmap = { workspace = true } +itertools = { workspace = true } +log = { workspace = true } +num-complex = { workspace = true, features = ["serde"] } +num-traits = { workspace = true } +thiserror = { workspace = true } [dev-dependencies] rustpython-parser = { path = "../parser" } -insta = "1.14.0" +insta = { workspace = true } diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml index f9cbce313f6..bf385a71826 100644 --- a/compiler/core/Cargo.toml +++ b/compiler/core/Cargo.toml @@ -8,13 +8,14 @@ repository = "https://github.com/RustPython/RustPython" license = "MIT" [dependencies] -bincode = "1.3.3" -bitflags = "1.3.2" -bstr = "0.2.17" -itertools = "0.10.3" +bincode = { workspace = true } +bitflags = { workspace = true } +bstr = { workspace = true } +itertools = { workspace = true } +num-bigint = { workspace = true, features = ["serde"] } +num-complex = { workspace = true, features = ["serde"] } +num_enum = { workspace = true } +serde = { workspace = true, features = ["derive"] } +thiserror = { workspace = true } + lz4_flex = "0.9.2" -num-bigint = { version = "0.4.3", features = ["serde"] } -num-complex = { version = "0.4.0", features = ["serde"] } -num_enum = "0.5.7" -serde = { version = "1.0.136", features = ["derive"] } -thiserror = "1.0" diff --git a/compiler/parser/Cargo.toml b/compiler/parser/Cargo.toml index c7b8af64dd1..56f3d161f59 100644 --- a/compiler/parser/Cargo.toml +++ b/compiler/parser/Cargo.toml @@ -12,7 +12,7 @@ edition = "2021" default = ["lalrpop"] # removing this causes potential build failure [build-dependencies] -anyhow = "1.0.45" +anyhow = { workspace = true } lalrpop = { version = "0.19.8", optional = true } phf_codegen = "0.11.1" tiny-keccak = { version = "2", features = ["sha3"] } @@ -21,18 +21,19 @@ tiny-keccak = { version = "2", features = ["sha3"] } rustpython-ast = { path = "../ast", version = "0.2.0" } rustpython-compiler-core = { path = "../core", version = "0.2.0" } -ahash = "0.7.6" -itertools = "0.10.3" -lalrpop-util = "0.19.8" -log = "0.4.16" -num-bigint = "0.4.3" -num-traits = "0.2.14" -phf = "0.11.1" -rustc-hash = "1.1.0" -thiserror = "1.0" +ahash = { workspace = true } +itertools = { workspace = true } +log = { workspace = true } +num-bigint = { workspace = true } +num-traits = { workspace = true } +thiserror = { workspace = true } + unic-emoji-char = "0.9.0" unic-ucd-ident = "0.9.0" unicode_names2 = "0.5.0" +lalrpop-util = "0.19.8" +phf = "0.11.1" +rustc-hash = "1.1.0" [dev-dependencies] -insta = "1.14.0" +insta = { workspace = true } diff --git a/compiler/parser/python.lalrpop b/compiler/parser/python.lalrpop index 719cef464bd..78b71361869 100644 --- a/compiler/parser/python.lalrpop +++ b/compiler/parser/python.lalrpop @@ -1274,11 +1274,11 @@ ArgumentList: ArgumentList = { }; FunctionArgument: (Option<(ast::Location, ast::Location, Option)>, ast::Expr) = { - => { + => { let expr = match c { Some(c) => ast::Expr { - location: e.location, - end_location: e.end_location, + location, + end_location: Some(end_location), custom: (), node: ast::ExprKind::GeneratorExp { elt: Box::new(e), diff --git a/compiler/parser/src/context.rs b/compiler/parser/src/context.rs index f10e1054357..eb1953f6931 100644 --- a/compiler/parser/src/context.rs +++ b/compiler/parser/src/context.rs @@ -1,6 +1,6 @@ use rustpython_ast::{Expr, ExprContext, ExprKind}; -pub fn set_context(expr: Expr, ctx: ExprContext) -> Expr { +pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr { match expr.node { ExprKind::Name { id, .. } => Expr { node: ExprKind::Name { id, ctx }, diff --git a/compiler/parser/src/error.rs b/compiler/parser/src/error.rs index 4edc02eaee8..dab07f8cdba 100644 --- a/compiler/parser/src/error.rs +++ b/compiler/parser/src/error.rs @@ -1,40 +1,71 @@ -//! Define internal parse error types -//! The goal is to provide a matching and a safe error API, maksing errors from LALR +//! Error types for the parser. +//! +//! These types are used to represent errors that occur during lexing and parsing and are +//! returned by the `parse_*` functions in the [parser] module and the iterator in the +//! [lexer] implementation. +//! +//! [parser]: crate::parser +//! [lexer]: crate::lexer +// Define internal parse error types. +// The goal is to provide a matching and a safe error API, masking errors from LALR use crate::{ast::Location, token::Tok}; use lalrpop_util::ParseError as LalrpopError; use std::fmt; -/// Represents an error during lexical scanning. +/// Represents an error during lexing. #[derive(Debug, PartialEq)] pub struct LexicalError { + /// The type of error that occurred. pub error: LexicalErrorType, + /// The location of the error. pub location: Location, } impl LexicalError { + /// Creates a new `LexicalError` with the given error type and location. pub fn new(error: LexicalErrorType, location: Location) -> Self { Self { error, location } } } +/// Represents the different types of errors that can occur during lexing. #[derive(Debug, PartialEq)] pub enum LexicalErrorType { + // TODO: Can probably be removed, the places it is used seem to be able + // to use the `UnicodeError` variant instead. + #[doc(hidden)] StringError, + // TODO: Should take a start/end position to report. + /// Decoding of a unicode escape sequence in a string literal failed. UnicodeError, + /// The nesting of brackets/braces/parentheses is not balanced. NestingError, + /// The indentation is not consistent. IndentationError, + /// Inconsistent use of tabs and spaces. TabError, + /// Encountered a tab after a space. TabsAfterSpaces, + /// A non-default argument follows a default argument. DefaultArgumentError, + /// A duplicate argument was found in a function definition. DuplicateArgumentError(String), + /// A positional argument follows a keyword argument. PositionalArgumentError, + /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`. UnpackedArgumentError, + /// A keyword argument was repeated. DuplicateKeywordArgumentError(String), + /// An unrecognized token was encountered. UnrecognizedToken { tok: char }, + /// An f-string error containing the [`FStringErrorType`]. FStringError(FStringErrorType), + /// An unexpected character was encountered after a line continuation. LineContinuationError, + /// An unexpected end of file was encountered. Eof, + /// An unexpected error occurred. OtherError(String), } @@ -85,13 +116,17 @@ impl fmt::Display for LexicalErrorType { } // TODO: consolidate these with ParseError +/// An error that occurred during parsing of an f-string. #[derive(Debug, PartialEq)] pub struct FStringError { + /// The type of error that occurred. pub error: FStringErrorType, + /// The location of the error. pub location: Location, } impl FStringError { + /// Creates a new `FStringError` with the given error type and location. pub fn new(error: FStringErrorType, location: Location) -> Self { Self { error, location } } @@ -106,19 +141,33 @@ impl From for LexicalError { } } +/// Represents the different types of errors that can occur during parsing of an f-string. #[derive(Debug, PartialEq)] pub enum FStringErrorType { + /// Expected a right brace after an opened left brace. UnclosedLbrace, + /// Expected a left brace after an ending right brace. UnopenedRbrace, + /// Expected a right brace after a conversion flag. ExpectedRbrace, + /// An error occurred while parsing an f-string expression. InvalidExpression(Box), + /// An invalid conversion flag was encountered. InvalidConversionFlag, + /// An empty expression was encountered. EmptyExpression, + /// An opening delimiter was not closed properly. MismatchedDelimiter(char, char), + /// Too many nested expressions in an f-string. ExpressionNestedTooDeeply, + /// The f-string expression cannot include the given character. ExpressionCannotInclude(char), + /// A single right brace was encountered. SingleRbrace, + /// A closing delimiter was not opened properly. Unmatched(char), + // TODO: Test this case. + /// Unterminated string. UnterminatedString, } @@ -167,9 +216,10 @@ impl From for LalrpopError { } } -/// Represents an error during parsing +/// Represents an error during parsing. pub type ParseError = rustpython_compiler_core::BaseError; +/// Represents the different types of errors that can occur during parsing. #[derive(Debug, PartialEq, thiserror::Error)] pub enum ParseErrorType { /// Parser encountered an unexpected end of input @@ -180,11 +230,12 @@ pub enum ParseErrorType { InvalidToken, /// Parser encountered an unexpected token UnrecognizedToken(Tok, Option), - /// Maps to `User` type from `lalrpop-util` + // Maps to `User` type from `lalrpop-util` + /// Parser encountered an error during lexing. Lexical(LexicalErrorType), } -/// Convert `lalrpop_util::ParseError` to our internal type +// Convert `lalrpop_util::ParseError` to our internal type pub(crate) fn parse_error_from_lalrpop( err: LalrpopError, source_path: &str, @@ -258,6 +309,7 @@ impl fmt::Display for ParseErrorType { } impl ParseErrorType { + /// Returns true if the error is an indentation error. pub fn is_indentation_error(&self) -> bool { match self { ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true, @@ -267,6 +319,8 @@ impl ParseErrorType { _ => false, } } + + /// Returns true if the error is a tab error. pub fn is_tab_error(&self) -> bool { matches!( self, diff --git a/compiler/parser/src/function.rs b/compiler/parser/src/function.rs index 21ccf013b8e..35c3ad29c85 100644 --- a/compiler/parser/src/function.rs +++ b/compiler/parser/src/function.rs @@ -1,8 +1,10 @@ +// Contains functions that perform validation and parsing of arguments and parameters. +// Checks apply both to functions and to lambdas. use crate::ast; use crate::error::{LexicalError, LexicalErrorType}; use rustc_hash::FxHashSet; -pub struct ArgumentList { +pub(crate) struct ArgumentList { pub args: Vec, pub keywords: Vec, } @@ -10,7 +12,10 @@ pub struct ArgumentList { type ParameterDefs = (Vec, Vec, Vec); type ParameterDef = (ast::Arg, Option); -pub fn validate_arguments(arguments: ast::Arguments) -> Result { +// Perform validation of function/lambda arguments in a function definition. +pub(crate) fn validate_arguments( + arguments: ast::Arguments, +) -> Result { let mut all_args: Vec<&ast::Located> = vec![]; all_args.extend(arguments.posonlyargs.iter()); @@ -29,6 +34,7 @@ pub fn validate_arguments(arguments: ast::Arguments) -> Result Result, Vec), ) -> Result { let mut pos_only = Vec::with_capacity(params.0.len()); @@ -52,7 +59,7 @@ pub fn parse_params( defaults.push(default); } else if !defaults.is_empty() { // Once we have started with defaults, all remaining arguments must - // have defaults + // have defaults. return Err(LexicalError { error: LexicalErrorType::DefaultArgumentError, location: name.location, @@ -79,7 +86,8 @@ type FunctionArgument = ( ast::Expr, ); -pub fn parse_args(func_args: Vec) -> Result { +// Parse arguments as supplied during a function/lambda *call*. +pub(crate) fn parse_args(func_args: Vec) -> Result { let mut args = vec![]; let mut keywords = vec![]; @@ -89,6 +97,7 @@ pub fn parse_args(func_args: Vec) -> Result { + // Check for duplicate keyword arguments in the call. if let Some(keyword_name) = &name { if keyword_names.contains(keyword_name) { return Err(LexicalError { @@ -111,13 +120,14 @@ pub fn parse_args(func_args: Vec) -> Result { - // Allow starred arguments after keyword arguments but - // not after double-starred arguments. + // Positional arguments mustn't follow keyword arguments. if !keywords.is_empty() && !is_starred(&value) { return Err(LexicalError { error: LexicalErrorType::PositionalArgumentError, location: value.location, }); + // Allow starred arguments after keyword arguments but + // not after double-starred arguments. } else if double_starred { return Err(LexicalError { error: LexicalErrorType::UnpackedArgumentError, @@ -132,6 +142,86 @@ pub fn parse_args(func_args: Vec) -> Result bool { matches!(exp.node, ast::ExprKind::Starred { .. }) } + +#[cfg(test)] +mod tests { + use crate::error::{LexicalErrorType, ParseErrorType}; + use crate::parser::parse_program; + + macro_rules! function_and_lambda { + ($($name:ident: $code:expr,)*) => { + $( + #[test] + fn $name() { + let parse_ast = parse_program($code, ""); + insta::assert_debug_snapshot!(parse_ast); + } + )* + } + } + + function_and_lambda! { + test_function_no_args: "def f(): pass", + test_function_pos_args: "def f(a, b, c): pass", + test_function_pos_args_with_defaults: "def f(a, b=20, c=30): pass", + test_function_kw_only_args: "def f(*, a, b, c): pass", + test_function_kw_only_args_with_defaults: "def f(*, a, b=20, c=30): pass", + test_function_pos_and_kw_only_args: "def f(a, b, c, *, d, e, f): pass", + test_function_pos_and_kw_only_args_with_defaults: "def f(a, b, c, *, d, e=20, f=30): pass", + test_function_pos_and_kw_only_args_with_defaults_and_varargs: "def f(a, b, c, *args, d, e=20, f=30): pass", + test_function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs: "def f(a, b, c, *args, d, e=20, f=30, **kwargs): pass", + test_lambda_no_args: "lambda: 1", + test_lambda_pos_args: "lambda a, b, c: 1", + test_lambda_pos_args_with_defaults: "lambda a, b=20, c=30: 1", + test_lambda_kw_only_args: "lambda *, a, b, c: 1", + test_lambda_kw_only_args_with_defaults: "lambda *, a, b=20, c=30: 1", + test_lambda_pos_and_kw_only_args: "lambda a, b, c, *, d, e: 0", + } + + fn function_parse_error(src: &str) -> LexicalErrorType { + let parse_ast = parse_program(src, ""); + parse_ast + .map_err(|e| match e.error { + ParseErrorType::Lexical(e) => e, + _ => panic!("Expected LexicalError"), + }) + .expect_err("Expected error") + } + + macro_rules! function_and_lambda_error { + ($($name:ident: $code:expr, $error:expr,)*) => { + $( + #[test] + fn $name() { + let error = function_parse_error($code); + assert_eq!(error, $error); + } + )* + } + } + + function_and_lambda_error! { + // Check definitions + test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_default_arg_error_f: "def f(a, b=20, c): pass", LexicalErrorType::DefaultArgumentError, + test_default_arg_error_l: "lambda a, b=20, c: 1", LexicalErrorType::DefaultArgumentError, + + // Check some calls. + test_positional_arg_error_f: "f(b=20, c)", LexicalErrorType::PositionalArgumentError, + test_unpacked_arg_error_f: "f(**b, *c)", LexicalErrorType::UnpackedArgumentError, + test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string()), + } +} diff --git a/compiler/parser/src/lexer.rs b/compiler/parser/src/lexer.rs index 1c124f6272c..20e6b148644 100644 --- a/compiler/parser/src/lexer.rs +++ b/compiler/parser/src/lexer.rs @@ -1,7 +1,37 @@ -//! This module takes care of lexing python source text. +//! This module takes care of lexing Python source text. //! -//! This means source code is translated into separate tokens. - +//! This means source code is scanned and translated into separate tokens. The rules +//! governing what is and is not a valid token are defined in the Python reference +//! guide section on [Lexical analysis]. +//! +//! The primary function in this module is [`make_tokenizer`], which takes a string slice +//! and returns an iterator over the tokens in the source code. The tokens are currently returned +//! as a `Result`, where [`Spanned`] is a tuple containing the +//! start and end [`Location`] and a [`Tok`] denoting the token. +//! +//! # Example +//! +//! ``` +//! use rustpython_parser::lexer::{make_tokenizer, Tok}; +//! use rustpython_parser::token::StringKind; +//! +//! let source = "x = 'RustPython'"; +//! let tokens = make_tokenizer(source) +//! .map(|tok| tok.expect("Failed to lex")) +//! .collect::>(); +//! +//! for (start, token, end) in tokens { +//! println!( +//! "{0},{1}-{2},{3:<5} {token:?}", +//! start.row(), +//! start.column(), +//! end.row(), +//! end.column(), +//! ); +//! } +//! ``` +//! +//! [Lexical analysis]: https://docs.python.org/3/reference/lexical_analysis.html pub use super::token::{StringKind, Tok}; use crate::ast::Location; use crate::error::{LexicalError, LexicalErrorType}; @@ -16,10 +46,12 @@ use std::str::FromStr; use unic_emoji_char::is_emoji_presentation; use unic_ucd_ident::{is_xid_continue, is_xid_start}; +// Indentations are tracked by a stack of indentation levels. IndentationLevel keeps +// track of the number of tabs and spaces at the current level. #[derive(Clone, Copy, PartialEq, Debug, Default)] struct IndentationLevel { - tabs: usize, - spaces: usize, + tabs: u32, + spaces: u32, } impl IndentationLevel { @@ -57,6 +89,9 @@ impl IndentationLevel { } } +// The indentations stack is used to keep track of the current indentation level. +// Similar to the CPython implementation, the Indentations stack always has at +// least one level which is never popped. See Reference 2.1.8. #[derive(Debug)] struct Indentations { indent_stack: Vec, @@ -93,6 +128,8 @@ impl Default for Indentations { } } +// A CharWindow is a sliding window over an iterator of chars. It is used to +// allow for look-ahead when scanning tokens from the source code. struct CharWindow, const N: usize> { source: T, window: [Option; N], @@ -115,10 +152,6 @@ where *self.window.last_mut().expect("never empty") = next; next } - - fn change_first(&mut self, ch: char) { - *self.window.first_mut().expect("never empty") = Some(ch); - } } impl Index for CharWindow @@ -133,102 +166,77 @@ where } } +/// A lexer for Python source code. pub struct Lexer> { + // Contains the source code to be lexed. window: CharWindow, - + // Are we at the beginning of a line? at_begin_of_line: bool, - nesting: usize, // Amount of parenthesis + // Amount of parenthesis. + nesting: usize, + // Indentation levels. indentations: Indentations, - + // Pending list of tokens to be returned. pending: Vec, + // The current location. location: Location, } // generated in build.rs, in gen_phf() +/// A map of keywords to their tokens. pub static KEYWORDS: phf::Map<&'static str, Tok> = include!(concat!(env!("OUT_DIR"), "/keywords.rs")); +/// Contains a Token along with its start and end location. pub type Spanned = (Location, Tok, Location); +/// The result of lexing a token. pub type LexResult = Result; +/// Create a new tokenizer from a source string. +/// +/// # Examples +/// +/// ``` +/// use rustpython_parser::lexer::{make_tokenizer}; +/// +/// let source = "def hello(): return 'world'"; +/// let tokenizer = make_tokenizer(source); +/// +/// for token in tokenizer { +/// println!("{:?}", token); +/// } +/// ``` #[inline] pub fn make_tokenizer(source: &str) -> impl Iterator + '_ { make_tokenizer_located(source, Location::default()) } +/// Create a new tokenizer from a source string, starting at a given location. +/// You probably want to use [`make_tokenizer`] instead. pub fn make_tokenizer_located( source: &str, start_location: Location, ) -> impl Iterator + '_ { - let nlh = NewlineHandler::new(source.chars()); - Lexer::new(nlh, start_location) -} - -// The newline handler is an iterator which collapses different newline -// types into \n always. -pub struct NewlineHandler> { - window: CharWindow, -} - -impl NewlineHandler -where - T: Iterator, -{ - pub fn new(source: T) -> Self { - let mut nlh = NewlineHandler { - window: CharWindow::new(source), - }; - nlh.shift(); - nlh.shift(); - nlh - } - - fn shift(&mut self) -> Option { - let result = self.window[0]; - self.window.slide(); - result - } -} - -impl Iterator for NewlineHandler -where - T: Iterator, -{ - type Item = char; - - fn next(&mut self) -> Option { - // Collapse \r\n into \n - loop { - match self.window[..2] { - [Some('\r'), Some('\n')] => { - // Windows EOL into \n - self.shift(); - } - [Some('\r'), _] => { - // MAC EOL into \n - self.window.change_first('\n'); - } - _ => break, - } - } - - self.shift() - } + Lexer::new(source.chars(), start_location) } impl Lexer where T: Iterator, { + /// Create a new lexer from T and a starting location. You probably want to use + /// [`make_tokenizer`] instead. pub fn new(input: T, start: Location) -> Self { let mut lxr = Lexer { at_begin_of_line: true, nesting: 0, indentations: Indentations::default(), - pending: Vec::new(), + // Usually we have less than 5 tokens pending. + pending: Vec::with_capacity(5), location: start, window: CharWindow::new(input), }; + // Fill the window. lxr.window.slide(); lxr.window.slide(); lxr.window.slide(); @@ -239,7 +247,7 @@ where lxr } - // Lexer helper functions: + /// Lex an identifier. Also used for keywords and string/bytes literals with a prefix. fn lex_identifier(&mut self) -> LexResult { // Detect potential string like rb'' b'' f'' u'' r'' match self.window[..3] { @@ -257,13 +265,13 @@ where }; let start_pos = self.get_pos(); - let mut name = String::new(); + let mut name = String::with_capacity(8); while self.is_identifier_continuation() { name.push(self.next_char().unwrap()); } let end_pos = self.get_pos(); - if let Some(tok) = KEYWORDS.get(name.as_str()) { + if let Some(tok) = KEYWORDS.get(&name) { Ok((start_pos, tok.clone(), end_pos)) } else { Ok((start_pos, Tok::Name { name }, end_pos)) @@ -441,14 +449,13 @@ where } } - /// Skip everything until end of line + /// Lex a single comment. fn lex_comment(&mut self) -> LexResult { let start_pos = self.get_pos(); let mut value = String::new(); - value.push(self.next_char().unwrap()); loop { match self.window[0] { - Some('\n') | None => { + Some('\n' | '\r') | None => { let end_pos = self.get_pos(); return Ok((start_pos, Tok::Comment(value), end_pos)); } @@ -458,13 +465,14 @@ where } } + /// Lex a string literal. fn lex_string(&mut self, kind: StringKind) -> LexResult { let start_pos = self.get_pos(); for _ in 0..kind.prefix_len() { self.next_char(); } let quote_char = self.next_char().unwrap(); - let mut string_content = String::new(); + let mut string_content = String::with_capacity(5); // If the next two characters are also the quote character, then we have a triple-quoted // string; consume those two characters and ensure that we require a triple-quote to close @@ -486,7 +494,6 @@ where continue; } } - if c == '\n' && !triple_quoted { return Err(LexicalError { error: LexicalErrorType::OtherError( @@ -533,20 +540,27 @@ where Ok((start_pos, tok, end_pos)) } + // Checks if the character c is a valid starting character as described + // in https://docs.python.org/3/reference/lexical_analysis.html#identifiers fn is_identifier_start(&self, c: char) -> bool { - c == '_' || is_xid_start(c) + match c { + 'a'..='z' | 'A'..='Z' | '_' => true, + _ => is_xid_start(c), + } } + // Checks if the character c is a valid continuation character as described + // in https://docs.python.org/3/reference/lexical_analysis.html#identifiers fn is_identifier_continuation(&self) -> bool { match self.window[0] { - Some('_' | '0'..='9') => true, + Some('a'..='z' | 'A'..='Z' | '_' | '0'..='9') => true, Some(c) => is_xid_continue(c), _ => false, } } - /// This is the main entry point. Call this function to retrieve the next token. - /// This function is used by the iterator implementation. + // This is the main entry point. Call this function to retrieve the next token. + // This function is used by the iterator implementation. fn inner_next(&mut self) -> LexResult { // top loop, keep on processing, until we have something pending. while self.pending.is_empty() { @@ -561,11 +575,11 @@ where Ok(self.pending.remove(0)) } - /// Given we are at the start of a line, count the number of spaces and/or tabs until the first character. + // Given we are at the start of a line, count the number of spaces and/or tabs until the first character. fn eat_indentation(&mut self) -> Result { // Determine indentation: - let mut spaces: usize = 0; - let mut tabs: usize = 0; + let mut spaces: u32 = 0; + let mut tabs: u32 = 0; loop { match self.window[0] { Some(' ') => { @@ -609,7 +623,7 @@ where spaces = 0; tabs = 0; } - Some('\n') => { + Some('\n' | '\r') => { // Empty line! self.next_char(); spaces = 0; @@ -630,6 +644,7 @@ where Ok(IndentationLevel { tabs, spaces }) } + // Push/pop indents/dedents based on the current indentation level. fn handle_indentations(&mut self) -> Result<(), LexicalError> { let indentation_level = self.eat_indentation()?; @@ -682,25 +697,13 @@ where Ok(()) } - /// Take a look at the next character, if any, and decide upon the next steps. + // Take a look at the next character, if any, and decide upon the next steps. fn consume_normal(&mut self) -> Result<(), LexicalError> { - // Check if we have some character: if let Some(c) = self.window[0] { - // First check identifier: + // Identifiers are the most common case. if self.is_identifier_start(c) { let identifier = self.lex_identifier()?; self.emit(identifier); - } else if is_emoji_presentation(c) { - let tok_start = self.get_pos(); - self.next_char(); - let tok_end = self.get_pos(); - self.emit(( - tok_start, - Tok::Name { - name: c.to_string(), - }, - tok_end, - )); } else { self.consume_character(c)?; } @@ -734,7 +737,7 @@ where Ok(()) } - /// Okay, we are facing a weird character, what is it? Determine that. + // Dispatch based on the given character. fn consume_character(&mut self, c: char) -> Result<(), LexicalError> { match c { '0'..='9' => { @@ -1047,10 +1050,7 @@ where } } ',' => { - let tok_start = self.get_pos(); - self.next_char(); - let tok_end = self.get_pos(); - self.emit((tok_start, Tok::Comma, tok_end)); + self.eat_single_char(Tok::Comma); } '.' => { if let Some('0'..='9') = self.window[1] { @@ -1070,7 +1070,7 @@ where } } } - '\n' => { + '\n' | '\r' => { let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); @@ -1093,13 +1093,16 @@ where } '\\' => { self.next_char(); - if let Some('\n') = self.window[0] { - self.next_char(); - } else { - return Err(LexicalError { - error: LexicalErrorType::LineContinuationError, - location: self.get_pos(), - }); + match self.window[0] { + Some('\n' | '\r') => { + self.next_char(); + } + _ => { + return Err(LexicalError { + error: LexicalErrorType::LineContinuationError, + location: self.get_pos(), + }) + } } if self.window[0].is_none() { @@ -1109,54 +1112,79 @@ where }); } } - _ => { - let c = self.next_char(); - return Err(LexicalError { - error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() }, - location: self.get_pos(), - }); - } // Ignore all the rest.. + if is_emoji_presentation(c) { + let tok_start = self.get_pos(); + self.next_char(); + let tok_end = self.get_pos(); + self.emit(( + tok_start, + Tok::Name { + name: c.to_string(), + }, + tok_end, + )); + } else { + let c = self.next_char(); + return Err(LexicalError { + error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() }, + location: self.get_pos(), + }); + } + } } Ok(()) } + // Used by single character tokens to advance the window and emit the correct token. fn eat_single_char(&mut self, ty: Tok) { let tok_start = self.get_pos(); - self.next_char().unwrap(); + self.next_char().unwrap_or_else(|| unsafe { + // SAFETY: eat_single_char has been called only after a character has been read + // from the window, so the window is guaranteed to be non-empty. + std::hint::unreachable_unchecked() + }); let tok_end = self.get_pos(); self.emit((tok_start, ty, tok_end)); } - /// Helper function to go to the next character coming up. + // Helper function to go to the next character coming up. fn next_char(&mut self) -> Option { - let c = self.window[0]; + let mut c = self.window[0]; self.window.slide(); - if c == Some('\n') { - self.location.newline(); - } else { - self.location.go_right(); + match c { + Some('\n') => { + self.location.newline(); + } + Some('\r') => { + if self.window[0] == Some('\n') { + self.window.slide(); + } + self.location.newline(); + c = Some('\n'); + } + _ => { + self.location.go_right(); + } } c } - /// Helper function to retrieve the current position. + // Helper function to retrieve the current position. fn get_pos(&self) -> Location { self.location } - /// Helper function to emit a lexed token to the queue of tokens. + // Helper function to emit a lexed token to the queue of tokens. fn emit(&mut self, spanned: Spanned) { self.pending.push(spanned); } } -/* Implement iterator pattern for the get_tok function. - -Calling the next element in the iterator will yield the next lexical -token. -*/ +// Implement iterator pattern for Lexer. +// Calling the next element in the iterator will yield the next lexical +// token. impl Iterator for Lexer where T: Iterator, @@ -1164,9 +1192,6 @@ where type Item = LexResult; fn next(&mut self) -> Option { - // Idea: create some sort of hash map for single char tokens: - // let mut X = HashMap::new(); - // X.insert('=', Tok::Equal); let token = self.inner_next(); trace!( "Lex token {:?}, nesting={:?}, indent stack: {:?}", @@ -1184,7 +1209,7 @@ where #[cfg(test)] mod tests { - use super::{make_tokenizer, NewlineHandler, StringKind, Tok}; + use super::{make_tokenizer, StringKind, Tok}; use num_bigint::BigInt; const WINDOWS_EOL: &str = "\r\n"; @@ -1196,16 +1221,6 @@ mod tests { lexer.map(|x| x.unwrap().1).collect() } - #[test] - fn test_newline_processor() { - // Escape \ followed by \n (by removal): - let src = "b\\\r\n"; - assert_eq!(4, src.len()); - let nlh = NewlineHandler::new(src.chars()); - let x: Vec = nlh.collect(); - assert_eq!(vec!['b', '\\', '\n'], x); - } - fn stok(s: &str) -> Tok { Tok::String { value: s.to_owned(), @@ -1640,4 +1655,33 @@ mod tests { let tokens = lex_source(source); assert_eq!(tokens, vec![stok(r"\N{EN SPACE}"), Tok::Newline]) } + + macro_rules! test_triple_quoted { + ($($name:ident: $eol:expr,)*) => { + $( + #[test] + fn $name() { + let source = format!("\"\"\"{0} test string{0} \"\"\"", $eol); + let tokens = lex_source(&source); + assert_eq!( + tokens, + vec![ + Tok::String { + value: "\n test string\n ".to_owned(), + kind: StringKind::String, + triple_quoted: true, + }, + Tok::Newline, + ] + ) + } + )* + } + } + + test_triple_quoted! { + test_triple_quoted_windows_eol: WINDOWS_EOL, + test_triple_quoted_mac_eol: MAC_EOL, + test_triple_quoted_unix_eol: UNIX_EOL, + } } diff --git a/compiler/parser/src/lib.rs b/compiler/parser/src/lib.rs index ce9dde1d129..5c7c9640336 100644 --- a/compiler/parser/src/lib.rs +++ b/compiler/parser/src/lib.rs @@ -1,19 +1,119 @@ -//! This crate can be used to parse python sourcecode into a so -//! called AST (abstract syntax tree). +//! This crate can be used to parse Python source code into an Abstract +//! Syntax Tree. //! -//! The stages involved in this process are lexical analysis and -//! parsing. The lexical analysis splits the sourcecode into -//! tokens, and the parsing transforms those tokens into an AST. +//! ## Overview: //! -//! For example, one could do this: +//! The process by which source code is parsed into an AST can be broken down +//! into two general stages: [lexical analysis] and [parsing]. //! +//! During lexical analysis, the source code is converted into a stream of lexical +//! tokens that represent the smallest meaningful units of the language. For example, +//! the source code `print("Hello world")` would _roughly_ be converted into the following +//! stream of tokens: +//! +//! ```text +//! Name("print"), LeftParen, String("Hello world"), RightParen +//! ``` +//! +//! these tokens are then consumed by the parser, which matches them against a set of +//! grammar rules to verify that the source code is syntactically valid and to construct +//! an AST that represents the source code. +//! +//! During parsing, the parser consumes the tokens generated by the lexer and constructs +//! a tree representation of the source code. The tree is made up of nodes that represent +//! the different syntactic constructs of the language. If the source code is syntactically +//! invalid, parsing fails and an error is returned. After a successful parse, the AST can +//! be used to perform further analysis on the source code. Continuing with the example +//! above, the AST generated by the parser would _roughly_ look something like this: +//! +//! ```text +//! node: Expr { +//! value: { +//! node: Call { +//! func: { +//! node: Name { +//! id: "print", +//! ctx: Load, +//! }, +//! }, +//! args: [ +//! node: Constant { +//! value: Str("Hello World"), +//! kind: None, +//! }, +//! ], +//! keywords: [], +//! }, +//! }, +//! }, +//!``` +//! +//! Note: The Tokens/ASTs shown above are not the exact tokens/ASTs generated by the parser. +//! +//! ## Source code layout: +//! +//! The functionality of this crate is split into several modules: +//! +//! - [token]: This module contains the definition of the tokens that are generated by the lexer. +//! - [lexer]: This module contains the lexer and is responsible for generating the tokens. +//! - [parser]: This module contains an interface to the parser and is responsible for generating the AST. +//! - Functions and strings have special parsing requirements that are handled in additional files. +//! - [mode]: This module contains the definition of the different modes that the parser can be in. +//! - [error]: This module contains the definition of the errors that can be returned by the parser. +//! +//! # Examples +//! +//! For example, to get a stream of tokens from a given string, one could do this: +//! +//! ``` +//! use rustpython_parser::lexer::make_tokenizer; +//! +//! let python_source = r#" +//! def is_odd(i): +//! return bool(i & 1) +//! "#; +//! let mut tokens = make_tokenizer(python_source); +//! assert!(tokens.all(|t| t.is_ok())); //! ``` -//! use rustpython_parser::{parser, ast}; //! -//! let python_source = "print('Hello world')"; -//! let python_ast = parser::parse_expression(python_source, "").unwrap(); +//! These tokens can be directly fed into the parser to generate an AST: //! //! ``` +//! use rustpython_parser::parser::{parse_tokens, Mode}; +//! use rustpython_parser::lexer::make_tokenizer; +//! +//! let python_source = r#" +//! def is_odd(i): +//! return bool(i & 1) +//! "#; +//! let tokens = make_tokenizer(python_source); +//! let ast = parse_tokens(tokens, Mode::Module, ""); +//! +//! assert!(ast.is_ok()); +//! ``` +//! +//! Alternatively, you can use one of the other `parse_*` functions to parse a string directly without using a specific +//! mode or tokenizing the source beforehand: +//! +//! ``` +//! use rustpython_parser::parser::parse_program; +//! +//! let python_source = r#" +//! def is_odd(i): +//! return bool(i & 1) +//! "#; +//! let ast = parse_program(python_source, ""); +//! +//! assert!(ast.is_ok()); +//! ``` +//! +//! [lexical analysis]: https://en.wikipedia.org/wiki/Lexical_analysis +//! [parsing]: https://en.wikipedia.org/wiki/Parsing +//! [token]: crate::token +//! [lexer]: crate::lexer +//! [parser]: crate::parser +//! [mode]: crate::mode +//! [error]: crate::error #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] #![doc(html_root_url = "https://docs.rs/rustpython-parser/")] @@ -27,9 +127,8 @@ mod function; pub mod lexer; pub mod mode; pub mod parser; -mod string_parser; +mod string; #[rustfmt::skip] mod python; mod context; -mod string; pub mod token; diff --git a/compiler/parser/src/mode.rs b/compiler/parser/src/mode.rs index cd84a098ace..4403fbee8f2 100644 --- a/compiler/parser/src/mode.rs +++ b/compiler/parser/src/mode.rs @@ -1,9 +1,14 @@ +//! Control in the different modes by which a source file can be parsed. use crate::token::Tok; +/// The mode argument specifies in what way code must be parsed. #[derive(Clone, Copy)] pub enum Mode { + /// The code consists of a sequence of statements. Module, + /// The code consists of a sequence of interactive statement. Interactive, + /// The code consists of a single expression. Expression, } @@ -39,6 +44,7 @@ impl std::str::FromStr for Mode { } } +/// Returned when a given mode is not valid. #[derive(Debug)] pub struct ModeParseError { _priv: (), diff --git a/compiler/parser/src/parser.rs b/compiler/parser/src/parser.rs index 8abbcacf71c..7f828021ec5 100644 --- a/compiler/parser/src/parser.rs +++ b/compiler/parser/src/parser.rs @@ -1,9 +1,16 @@ -//! Python parsing. +//! Contains the interface to the Python parser. //! -//! Use this module to parse python code into an AST. -//! There are three ways to parse python code. You could -//! parse a whole program, a single statement, or a single -//! expression. +//! Functions in this module can be used to parse Python code into an [Abstract Syntax Tree] +//! (AST) that is then transformed into bytecode. +//! +//! There are three ways to parse Python code corresponding to the different [`Mode`]s +//! defined in the [`mode`] module. +//! +//! All functions return a [`Result`](std::result::Result) containing the parsed AST or +//! a [`ParseError`] if parsing failed. +//! +//! [Abstract Syntax Tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree +//! [`Mode`]: crate::mode use crate::lexer::{LexResult, Tok}; pub use crate::mode::Mode; @@ -12,13 +19,26 @@ use ast::Location; use itertools::Itertools; use std::iter; -/* - * Parse python code. - * Grammar may be inspired by antlr grammar for python: - * https://github.com/antlr/grammars-v4/tree/master/python3 - */ - -/// Parse a full python program, containing usually multiple lines. +/// Parse a full Python program usually consisting of multiple lines. +/// +/// This is a convenience function that can be used to parse a full Python program without having to +/// specify the [`Mode`] or the location. It is probably what you want to use most of the time. +/// +/// # Example +/// +/// For example, parsing a simple function definition and a call to that function: +/// +/// ``` +/// use rustpython_parser::parser; +/// let source = r#" +/// def foo(): +/// return 42 +/// +/// print(foo()) +/// "#; +/// let program = parser::parse_program(source, ""); +/// assert!(program.is_ok()); +/// ``` pub fn parse_program(source: &str, source_path: &str) -> Result { parse(source, Mode::Module, source_path).map(|top| match top { ast::Mod::Module { body, .. } => body, @@ -26,49 +46,44 @@ pub fn parse_program(source: &str, source_path: &str) -> Result").unwrap(); -/// -/// assert_eq!( -/// expr, -/// ast::Expr { -/// location: ast::Location::new(1, 0), -/// end_location: Some(ast::Location::new(1, 5)), -/// custom: (), -/// node: ast::ExprKind::BinOp { -/// left: Box::new(ast::Expr { -/// location: ast::Location::new(1, 0), -/// end_location: Some(ast::Location::new(1, 1)), -/// custom: (), -/// node: ast::ExprKind::Constant { -/// value: ast::Constant::Int(1.into()), -/// kind: None, -/// } -/// }), -/// op: ast::Operator::Add, -/// right: Box::new(ast::Expr { -/// location: ast::Location::new(1, 4), -/// end_location: Some(ast::Location::new(1, 5)), -/// custom: (), -/// node: ast::ExprKind::Constant { -/// value: ast::Constant::Int(2.into()), -/// kind: None, -/// } -/// }) -/// } -/// }, -/// ); +/// let expr = parser::parse_expression("1 + 2", ""); +/// +/// assert!(expr.is_ok()); /// /// ``` pub fn parse_expression(source: &str, path: &str) -> Result { parse_expression_located(source, path, Location::new(1, 0)) } +/// Parses a Python expression from a given location. +/// +/// This function allows to specify the location of the expression in the source code, other than +/// that, it behaves exactly like [`parse_expression`]. +/// +/// # Example +/// +/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different, +/// somewhat silly, location: +/// +/// ``` +/// use rustpython_parser::parser::parse_expression_located; +/// use rustpython_parser::ast::Location; +/// +/// let expr = parse_expression_located("1 + 2", "", Location::new(5, 20)); +/// assert!(expr.is_ok()); +/// ``` pub fn parse_expression_located( source: &str, path: &str, @@ -80,12 +95,64 @@ pub fn parse_expression_located( }) } -// Parse a given source code +/// Parse the given Python source code using the specified [`Mode`]. +/// +/// This function is the most general function to parse Python code. Based on the [`Mode`] supplied, +/// it can be used to parse a single expression, a full Python program or an interactive expression. +/// +/// # Example +/// +/// If we want to parse a simple expression, we can use the [`Mode::Expression`] mode during +/// parsing: +/// +/// ``` +/// use rustpython_parser::parser::{parse, Mode}; +/// +/// let expr = parse("1 + 2", Mode::Expression, ""); +/// assert!(expr.is_ok()); +/// ``` +/// +/// Alternatively, we can parse a full Python program consisting of multiple lines: +/// +/// ``` +/// use rustpython_parser::parser::{parse, Mode}; +/// +/// let source = r#" +/// class Greeter: +/// +/// def greet(self): +/// print("Hello, world!") +/// "#; +/// let program = parse(source, Mode::Module, ""); +/// assert!(program.is_ok()); +/// ``` pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result { parse_located(source, mode, source_path, Location::new(1, 0)) } -// Parse a given source code from a given location +/// Parse the given Python source code using the specified [`Mode`] and [`Location`]. +/// +/// This function allows to specify the location of the the source code, other than +/// that, it behaves exactly like [`parse`]. +/// +/// # Example +/// +/// ``` +/// use rustpython_parser::parser::{parse_located, Mode}; +/// use rustpython_parser::ast::Location; +/// +/// let source = r#" +/// def fib(i): +/// a, b = 0, 1 +/// for _ in range(i): +/// a, b = b, a + b +/// return a +/// +/// print(fib(42)) +/// "#; +/// let program = parse_located(source, Mode::Module, "", Location::new(1, 0)); +/// assert!(program.is_ok()); +/// ``` pub fn parse_located( source: &str, mode: Mode, @@ -96,7 +163,22 @@ pub fn parse_located( parse_tokens(lxr, mode, source_path) } -// Parse a given token iterator. +/// Parse an iterator of [`LexResult`]s using the specified [`Mode`]. +/// +/// This could allow you to perform some preprocessing on the tokens before parsing them. +/// +/// # Example +/// +/// As an example, instead of parsing a string, we can parse a list of tokens after we generate +/// them using the [`lexer::make_tokenizer`] function: +/// +/// ``` +/// use rustpython_parser::parser::{parse_tokens, Mode}; +/// use rustpython_parser::lexer::make_tokenizer; +/// +/// let expr = parse_tokens(make_tokenizer("1 + 2"), Mode::Expression, ""); +/// assert!(expr.is_ok()); +/// ``` pub fn parse_tokens( lxr: impl IntoIterator, mode: Mode, @@ -310,9 +392,31 @@ with (0 as a, 1 as b,): pass } } + #[test] + fn test_generator_expression_argument() { + let source = r#"' '.join( + sql + for sql in ( + "LIMIT %d" % limit if limit else None, + ("OFFSET %d" % offset) if offset else None, + ) +)"#; + let parse_ast = parse_expression(source, "").unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + #[test] fn test_dict_unpacking() { let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } + + #[test] + fn test_modes() { + let source = "a[0][1][2][3][4]"; + + assert!(parse(&source, Mode::Expression, "").is_ok()); + assert!(parse(&source, Mode::Module, "").is_ok()); + assert!(parse(&source, Mode::Interactive, "").is_ok()); + } } diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap new file mode 100644 index 00000000000..bfb7074ebe2 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap @@ -0,0 +1,108 @@ +--- +source: compiler/parser/src/function.rs +assertion_line: 165 +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 15, + }, + end_location: Some( + Location { + row: 1, + column: 16, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: [ + Located { + location: Location { + row: 1, + column: 19, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap new file mode 100644 index 00000000000..2004bbeedeb --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap @@ -0,0 +1,147 @@ +--- +source: compiler/parser/src/function.rs +assertion_line: 165 +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 29, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 18, + }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [ + Located { + location: Location { + row: 1, + column: 14, + }, + end_location: Some( + Location { + row: 1, + column: 16, + }, + ), + custom: (), + node: Constant { + value: Int( + 20, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 20, + }, + end_location: Some( + Location { + row: 1, + column: 22, + }, + ), + custom: (), + node: Constant { + value: Int( + 30, + ), + kind: None, + }, + }, + ], + kwarg: None, + defaults: [], + }, + body: [ + Located { + location: Location { + row: 1, + column: 25, + }, + end_location: Some( + Location { + row: 1, + column: 29, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_no_args.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_no_args.snap new file mode 100644 index 00000000000..a8d08b9add5 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_no_args.snap @@ -0,0 +1,52 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: [ + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args.snap new file mode 100644 index 00000000000..766a36b1ada --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args.snap @@ -0,0 +1,162 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 32, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 6, + }, + end_location: Some( + Location { + row: 1, + column: 7, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 18, + }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), + custom: (), + node: ArgData { + arg: "d", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 21, + }, + end_location: Some( + Location { + row: 1, + column: 22, + }, + ), + custom: (), + node: ArgData { + arg: "e", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 24, + }, + end_location: Some( + Location { + row: 1, + column: 25, + }, + ), + custom: (), + node: ArgData { + arg: "f", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: [ + Located { + location: Location { + row: 1, + column: 28, + }, + end_location: Some( + Location { + row: 1, + column: 32, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults.snap new file mode 100644 index 00000000000..c6ff5e5d2c7 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults.snap @@ -0,0 +1,201 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 38, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 6, + }, + end_location: Some( + Location { + row: 1, + column: 7, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 18, + }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), + custom: (), + node: ArgData { + arg: "d", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 21, + }, + end_location: Some( + Location { + row: 1, + column: 22, + }, + ), + custom: (), + node: ArgData { + arg: "e", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 27, + }, + end_location: Some( + Location { + row: 1, + column: 28, + }, + ), + custom: (), + node: ArgData { + arg: "f", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [ + Located { + location: Location { + row: 1, + column: 23, + }, + end_location: Some( + Location { + row: 1, + column: 25, + }, + ), + custom: (), + node: Constant { + value: Int( + 20, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 29, + }, + end_location: Some( + Location { + row: 1, + column: 31, + }, + ), + custom: (), + node: Constant { + value: Int( + 30, + ), + kind: None, + }, + }, + ], + kwarg: None, + defaults: [], + }, + body: [ + Located { + location: Location { + row: 1, + column: 34, + }, + end_location: Some( + Location { + row: 1, + column: 38, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs.snap new file mode 100644 index 00000000000..830a9c5298c --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs.snap @@ -0,0 +1,220 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 42, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 6, + }, + end_location: Some( + Location { + row: 1, + column: 7, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: Some( + Located { + location: Location { + row: 1, + column: 16, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: ArgData { + arg: "args", + annotation: None, + type_comment: None, + }, + }, + ), + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 22, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: ArgData { + arg: "d", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 25, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: ArgData { + arg: "e", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 31, + }, + end_location: Some( + Location { + row: 1, + column: 32, + }, + ), + custom: (), + node: ArgData { + arg: "f", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [ + Located { + location: Location { + row: 1, + column: 27, + }, + end_location: Some( + Location { + row: 1, + column: 29, + }, + ), + custom: (), + node: Constant { + value: Int( + 20, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 33, + }, + end_location: Some( + Location { + row: 1, + column: 35, + }, + ), + custom: (), + node: Constant { + value: Int( + 30, + ), + kind: None, + }, + }, + ], + kwarg: None, + defaults: [], + }, + body: [ + Located { + location: Location { + row: 1, + column: 38, + }, + end_location: Some( + Location { + row: 1, + column: 42, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs.snap new file mode 100644 index 00000000000..fde81e980eb --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs.snap @@ -0,0 +1,239 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 52, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 6, + }, + end_location: Some( + Location { + row: 1, + column: 7, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: Some( + Located { + location: Location { + row: 1, + column: 16, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: ArgData { + arg: "args", + annotation: None, + type_comment: None, + }, + }, + ), + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 22, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: ArgData { + arg: "d", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 25, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: ArgData { + arg: "e", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 31, + }, + end_location: Some( + Location { + row: 1, + column: 32, + }, + ), + custom: (), + node: ArgData { + arg: "f", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [ + Located { + location: Location { + row: 1, + column: 27, + }, + end_location: Some( + Location { + row: 1, + column: 29, + }, + ), + custom: (), + node: Constant { + value: Int( + 20, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 33, + }, + end_location: Some( + Location { + row: 1, + column: 35, + }, + ), + custom: (), + node: Constant { + value: Int( + 30, + ), + kind: None, + }, + }, + ], + kwarg: Some( + Located { + location: Location { + row: 1, + column: 39, + }, + end_location: Some( + Location { + row: 1, + column: 45, + }, + ), + custom: (), + node: ArgData { + arg: "kwargs", + annotation: None, + type_comment: None, + }, + }, + ), + defaults: [], + }, + body: [ + Located { + location: Location { + row: 1, + column: 48, + }, + end_location: Some( + Location { + row: 1, + column: 52, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args.snap new file mode 100644 index 00000000000..251bba870a4 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args.snap @@ -0,0 +1,107 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 6, + }, + end_location: Some( + Location { + row: 1, + column: 7, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: [ + Located { + location: Location { + row: 1, + column: 16, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args_with_defaults.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args_with_defaults.snap new file mode 100644 index 00000000000..71b4bb86945 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args_with_defaults.snap @@ -0,0 +1,146 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: FunctionDef { + name: "f", + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 6, + }, + end_location: Some( + Location { + row: 1, + column: 7, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 9, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 15, + }, + end_location: Some( + Location { + row: 1, + column: 16, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [ + Located { + location: Location { + row: 1, + column: 11, + }, + end_location: Some( + Location { + row: 1, + column: 13, + }, + ), + custom: (), + node: Constant { + value: Int( + 20, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 17, + }, + end_location: Some( + Location { + row: 1, + column: 19, + }, + ), + custom: (), + node: Constant { + value: Int( + 30, + ), + kind: None, + }, + }, + ], + }, + body: [ + Located { + location: Location { + row: 1, + column: 22, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: Pass, + }, + ], + decorator_list: [], + returns: None, + type_comment: None, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args.snap new file mode 100644 index 00000000000..59987372f8a --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args.snap @@ -0,0 +1,121 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: Lambda { + args: Arguments { + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 10, + }, + end_location: Some( + Location { + row: 1, + column: 11, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 13, + }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 16, + }, + end_location: Some( + Location { + row: 1, + column: 17, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: Located { + location: Location { + row: 1, + column: 19, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + }, + }, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args_with_defaults.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args_with_defaults.snap new file mode 100644 index 00000000000..d48792dc118 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args_with_defaults.snap @@ -0,0 +1,160 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: Lambda { + args: Arguments { + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 10, + }, + end_location: Some( + Location { + row: 1, + column: 11, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 13, + }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 19, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [ + Located { + location: Location { + row: 1, + column: 15, + }, + end_location: Some( + Location { + row: 1, + column: 17, + }, + ), + custom: (), + node: Constant { + value: Int( + 20, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 21, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: Constant { + value: Int( + 30, + ), + kind: None, + }, + }, + ], + kwarg: None, + defaults: [], + }, + body: Located { + location: Location { + row: 1, + column: 25, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + }, + }, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_no_args.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_no_args.snap new file mode 100644 index 00000000000..aab5ec4d503 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_no_args.snap @@ -0,0 +1,66 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), + custom: (), + node: Lambda { + args: Arguments { + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: Located { + location: Location { + row: 1, + column: 8, + }, + end_location: Some( + Location { + row: 1, + column: 9, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + }, + }, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_and_kw_only_args.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_and_kw_only_args.snap new file mode 100644 index 00000000000..22f695b3091 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_and_kw_only_args.snap @@ -0,0 +1,158 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: Lambda { + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 7, + }, + end_location: Some( + Location { + row: 1, + column: 8, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 10, + }, + end_location: Some( + Location { + row: 1, + column: 11, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 13, + }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [ + Located { + location: Location { + row: 1, + column: 19, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: ArgData { + arg: "d", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 22, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: ArgData { + arg: "e", + annotation: None, + type_comment: None, + }, + }, + ], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: Located { + location: Location { + row: 1, + column: 25, + }, + end_location: Some( + Location { + row: 1, + column: 26, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + }, + }, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args.snap new file mode 100644 index 00000000000..f4ce79f9783 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args.snap @@ -0,0 +1,121 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 17, + }, + ), + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 17, + }, + ), + custom: (), + node: Lambda { + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 7, + }, + end_location: Some( + Location { + row: 1, + column: 8, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 10, + }, + end_location: Some( + Location { + row: 1, + column: 11, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 13, + }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [], + }, + body: Located { + location: Location { + row: 1, + column: 16, + }, + end_location: Some( + Location { + row: 1, + column: 17, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + }, + }, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args_with_defaults.snap b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args_with_defaults.snap new file mode 100644 index 00000000000..01691aa7d85 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args_with_defaults.snap @@ -0,0 +1,160 @@ +--- +source: compiler/parser/src/function.rs +expression: parse_ast +--- +Ok( + [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: Lambda { + args: Arguments { + posonlyargs: [], + args: [ + Located { + location: Location { + row: 1, + column: 7, + }, + end_location: Some( + Location { + row: 1, + column: 8, + }, + ), + custom: (), + node: ArgData { + arg: "a", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 10, + }, + end_location: Some( + Location { + row: 1, + column: 11, + }, + ), + custom: (), + node: ArgData { + arg: "b", + annotation: None, + type_comment: None, + }, + }, + Located { + location: Location { + row: 1, + column: 16, + }, + end_location: Some( + Location { + row: 1, + column: 17, + }, + ), + custom: (), + node: ArgData { + arg: "c", + annotation: None, + type_comment: None, + }, + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: None, + defaults: [ + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 14, + }, + ), + custom: (), + node: Constant { + value: Int( + 20, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 1, + column: 18, + }, + end_location: Some( + Location { + row: 1, + column: 20, + }, + ), + custom: (), + node: Constant { + value: Int( + 30, + ), + kind: None, + }, + }, + ], + }, + body: Located { + location: Location { + row: 1, + column: 22, + }, + end_location: Some( + Location { + row: 1, + column: 23, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + }, + }, + }, + }, + ], +) diff --git a/compiler/parser/src/snapshots/rustpython_parser__parser__tests__generator_expression_argument.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__generator_expression_argument.snap new file mode 100644 index 00000000000..863ece46067 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__generator_expression_argument.snap @@ -0,0 +1,333 @@ +--- +source: compiler/parser/src/parser.rs +expression: parse_ast +--- +Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 7, + column: 1, + }, + ), + custom: (), + node: Call { + func: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 8, + }, + ), + custom: (), + node: Attribute { + value: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 3, + }, + ), + custom: (), + node: Constant { + value: Str( + " ", + ), + kind: None, + }, + }, + attr: "join", + ctx: Load, + }, + }, + args: [ + Located { + location: Location { + row: 2, + column: 4, + }, + end_location: Some( + Location { + row: 6, + column: 5, + }, + ), + custom: (), + node: GeneratorExp { + elt: Located { + location: Location { + row: 2, + column: 4, + }, + end_location: Some( + Location { + row: 2, + column: 7, + }, + ), + custom: (), + node: Name { + id: "sql", + ctx: Load, + }, + }, + generators: [ + Comprehension { + target: Located { + location: Location { + row: 3, + column: 8, + }, + end_location: Some( + Location { + row: 3, + column: 11, + }, + ), + custom: (), + node: Name { + id: "sql", + ctx: Store, + }, + }, + iter: Located { + location: Location { + row: 3, + column: 15, + }, + end_location: Some( + Location { + row: 6, + column: 5, + }, + ), + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 4, + column: 8, + }, + end_location: Some( + Location { + row: 4, + column: 45, + }, + ), + custom: (), + node: IfExp { + test: Located { + location: Location { + row: 4, + column: 30, + }, + end_location: Some( + Location { + row: 4, + column: 35, + }, + ), + custom: (), + node: Name { + id: "limit", + ctx: Load, + }, + }, + body: Located { + location: Location { + row: 4, + column: 8, + }, + end_location: Some( + Location { + row: 4, + column: 26, + }, + ), + custom: (), + node: BinOp { + left: Located { + location: Location { + row: 4, + column: 8, + }, + end_location: Some( + Location { + row: 4, + column: 18, + }, + ), + custom: (), + node: Constant { + value: Str( + "LIMIT %d", + ), + kind: None, + }, + }, + op: Mod, + right: Located { + location: Location { + row: 4, + column: 21, + }, + end_location: Some( + Location { + row: 4, + column: 26, + }, + ), + custom: (), + node: Name { + id: "limit", + ctx: Load, + }, + }, + }, + }, + orelse: Located { + location: Location { + row: 4, + column: 41, + }, + end_location: Some( + Location { + row: 4, + column: 45, + }, + ), + custom: (), + node: Constant { + value: None, + kind: None, + }, + }, + }, + }, + Located { + location: Location { + row: 5, + column: 8, + }, + end_location: Some( + Location { + row: 5, + column: 50, + }, + ), + custom: (), + node: IfExp { + test: Located { + location: Location { + row: 5, + column: 34, + }, + end_location: Some( + Location { + row: 5, + column: 40, + }, + ), + custom: (), + node: Name { + id: "offset", + ctx: Load, + }, + }, + body: Located { + location: Location { + row: 5, + column: 9, + }, + end_location: Some( + Location { + row: 5, + column: 29, + }, + ), + custom: (), + node: BinOp { + left: Located { + location: Location { + row: 5, + column: 9, + }, + end_location: Some( + Location { + row: 5, + column: 20, + }, + ), + custom: (), + node: Constant { + value: Str( + "OFFSET %d", + ), + kind: None, + }, + }, + op: Mod, + right: Located { + location: Location { + row: 5, + column: 23, + }, + end_location: Some( + Location { + row: 5, + column: 29, + }, + ), + custom: (), + node: Name { + id: "offset", + ctx: Load, + }, + }, + }, + }, + orelse: Located { + location: Location { + row: 5, + column: 46, + }, + end_location: Some( + Location { + row: 5, + column: 50, + }, + ), + custom: (), + node: Constant { + value: None, + kind: None, + }, + }, + }, + }, + ], + ctx: Load, + }, + }, + ifs: [], + is_async: 0, + }, + ], + }, + }, + ], + keywords: [], + }, +} diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap similarity index 96% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap index de77be84423..cca26adc401 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 698 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap similarity index 98% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap index e7f5bff8943..c425d676bcf 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 706 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap similarity index 98% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap index 5efa6f6c72b..c402ed6867c 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 714 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap new file mode 100644 index 00000000000..073ac428e7b --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap @@ -0,0 +1,6 @@ +--- +source: compiler/parser/src/string.rs +assertion_line: 690 +expression: "parse_fstring(\"\").unwrap()" +--- +[] diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap similarity index 97% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap index 32b63f95456..10b6f3acbc3 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 669 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap similarity index 97% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap index e9d8120a5a9..194061c3ddd 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 766 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap similarity index 98% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap index 268a16e96bf..96ec3e72b0d 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 677 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap similarity index 97% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap index aaa2ad15628..cbfe435ddc0 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 759 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap similarity index 97% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap index 0d1107ecfa0..2adc5a48159 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 685 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap similarity index 96% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap index 7851495f39d..16d62ea19e5 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 773 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap similarity index 96% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap index 621e49908c9..16171f2fe77 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 780 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap similarity index 93% rename from compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap index 62a0cf43588..e3a9ff79a0b 100644 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap +++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap @@ -1,5 +1,6 @@ --- -source: compiler/parser/src/string_parser.rs +source: compiler/parser/src/string.rs +assertion_line: 787 expression: parse_ast --- [ diff --git a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap b/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap deleted file mode 100644 index fbc08b4abd4..00000000000 --- a/compiler/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap +++ /dev/null @@ -1,5 +0,0 @@ ---- -source: compiler/parser/src/string_parser.rs -expression: "parse_fstring(\"\").unwrap()" ---- -[] diff --git a/compiler/parser/src/string.rs b/compiler/parser/src/string.rs index 027d4a99013..84c6fe22d11 100644 --- a/compiler/parser/src/string.rs +++ b/compiler/parser/src/string.rs @@ -1,12 +1,545 @@ +// Contains the logic for parsing string literals (mostly concerned with f-strings.) +// +// The lexer doesn't do any special handling of f-strings, it just treats them as +// regular strings. Since the parser has no definition of f-string formats (Pending PEP 701) +// we have to do the parsing here, manually. +use itertools::Itertools; + +use self::FStringErrorType::*; use crate::{ - ast::{Constant, Expr, ExprKind, Location}, - error::{LexicalError, LexicalErrorType}, - string_parser::parse_string, + ast::{Constant, ConversionFlag, Expr, ExprKind, Location}, + error::{FStringError, FStringErrorType, LexicalError, LexicalErrorType, ParseError}, + parser::parse_expression_located, token::StringKind, }; -use itertools::Itertools; +use std::{iter, str}; + +// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798 +const MAX_UNICODE_NAME: usize = 88; + +struct StringParser<'a> { + chars: iter::Peekable>, + kind: StringKind, + start: Location, + end: Location, + location: Location, +} + +impl<'a> StringParser<'a> { + fn new( + source: &'a str, + kind: StringKind, + triple_quoted: bool, + start: Location, + end: Location, + ) -> Self { + let offset = kind.prefix_len() + if triple_quoted { 3 } else { 1 }; + Self { + chars: source.chars().peekable(), + kind, + start, + end, + location: start.with_col_offset(offset), + } + } + + #[inline] + fn next_char(&mut self) -> Option { + let Some(c) = self.chars.next() else { + return None + }; + if c == '\n' { + self.location.newline(); + } else { + self.location.go_right(); + } + Some(c) + } + + #[inline] + fn peek(&mut self) -> Option<&char> { + self.chars.peek() + } + + #[inline] + fn get_pos(&self) -> Location { + self.location + } + + #[inline] + fn expr(&self, node: ExprKind) -> Expr { + Expr::new(self.start, self.end, node) + } + + fn parse_unicode_literal(&mut self, literal_number: usize) -> Result { + let mut p: u32 = 0u32; + let unicode_error = LexicalError::new(LexicalErrorType::UnicodeError, self.get_pos()); + for i in 1..=literal_number { + match self.next_char() { + Some(c) => match c.to_digit(16) { + Some(d) => p += d << ((literal_number - i) * 4), + None => return Err(unicode_error), + }, + None => return Err(unicode_error), + } + } + match p { + 0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER), + _ => std::char::from_u32(p).ok_or(unicode_error), + } + } + + fn parse_octet(&mut self, first: char) -> char { + let mut octet_content = String::new(); + octet_content.push(first); + while octet_content.len() < 3 { + if let Some('0'..='7') = self.peek() { + octet_content.push(self.next_char().unwrap()) + } else { + break; + } + } + let value = u32::from_str_radix(&octet_content, 8).unwrap(); + char::from_u32(value).unwrap() + } + + fn parse_unicode_name(&mut self) -> Result { + let start_pos = self.get_pos(); + match self.next_char() { + Some('{') => {} + _ => return Err(LexicalError::new(LexicalErrorType::StringError, start_pos)), + } + let start_pos = self.get_pos(); + let mut name = String::new(); + loop { + match self.next_char() { + Some('}') => break, + Some(c) => name.push(c), + None => { + return Err(LexicalError::new( + LexicalErrorType::StringError, + self.get_pos(), + )) + } + } + } + + if name.len() > MAX_UNICODE_NAME { + return Err(LexicalError::new( + LexicalErrorType::UnicodeError, + self.get_pos(), + )); + } + + unicode_names2::character(&name) + .ok_or_else(|| LexicalError::new(LexicalErrorType::UnicodeError, start_pos)) + } + + fn parse_escaped_char(&mut self) -> Result { + match self.next_char() { + Some(c) => { + let char = match c { + '\\' => '\\', + '\'' => '\'', + '\"' => '"', + 'a' => '\x07', + 'b' => '\x08', + 'f' => '\x0c', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'v' => '\x0b', + o @ '0'..='7' => self.parse_octet(o), + 'x' => self.parse_unicode_literal(2)?, + 'u' if !self.kind.is_bytes() => self.parse_unicode_literal(4)?, + 'U' if !self.kind.is_bytes() => self.parse_unicode_literal(8)?, + 'N' if !self.kind.is_bytes() => self.parse_unicode_name()?, + // Special cases where the escape sequence is not a single character + '\n' => return Ok("".to_string()), + c => { + if self.kind.is_bytes() && !c.is_ascii() { + return Err(LexicalError { + error: LexicalErrorType::OtherError( + "bytes can only contain ASCII literal characters".to_owned(), + ), + location: self.get_pos(), + }); + } + return Ok(format!("\\{c}")); + } + }; + Ok(char.to_string()) + } + None => Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }), + } + } + + fn parse_formatted_value(&mut self, nested: u8) -> Result, LexicalError> { + let mut expression = String::new(); + let mut spec = None; + let mut delims = Vec::new(); + let mut conversion = ConversionFlag::None; + let mut self_documenting = false; + let mut trailing_seq = String::new(); + let location = self.get_pos(); + + while let Some(ch) = self.next_char() { + match ch { + // can be integrated better with the remaining code, but as a starting point ok + // in general I would do here a tokenizing of the fstrings to omit this peeking. + '!' | '=' | '>' | '<' if self.peek() == Some(&'=') => { + expression.push(ch); + expression.push('='); + self.next_char(); + } + '!' if delims.is_empty() && self.peek() != Some(&'=') => { + if expression.trim().is_empty() { + return Err(FStringError::new(EmptyExpression, self.get_pos()).into()); + } + + conversion = match self.next_char() { + Some('s') => ConversionFlag::Str, + Some('a') => ConversionFlag::Ascii, + Some('r') => ConversionFlag::Repr, + Some(_) => { + return Err( + FStringError::new(InvalidConversionFlag, self.get_pos()).into() + ); + } + None => { + return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()); + } + }; + + match self.peek() { + Some('}' | ':') => {} + Some(_) | None => { + return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()); + } + } + } + + // match a python 3.8 self documenting expression + // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}' + '=' if self.peek() != Some(&'=') && delims.is_empty() => { + self_documenting = true; + } + + ':' if delims.is_empty() => { + let parsed_spec = self.parse_spec(nested)?; + + spec = Some(Box::new(self.expr(ExprKind::JoinedStr { + values: parsed_spec, + }))); + } + '(' | '{' | '[' => { + expression.push(ch); + delims.push(ch); + } + ')' => { + let last_delim = delims.pop(); + match last_delim { + Some('(') => { + expression.push(ch); + } + Some(c) => { + return Err(FStringError::new( + MismatchedDelimiter(c, ')'), + self.get_pos(), + ) + .into()); + } + None => { + return Err(FStringError::new(Unmatched(')'), self.get_pos()).into()); + } + } + } + ']' => { + let last_delim = delims.pop(); + match last_delim { + Some('[') => { + expression.push(ch); + } + Some(c) => { + return Err(FStringError::new( + MismatchedDelimiter(c, ']'), + self.get_pos(), + ) + .into()); + } + None => { + return Err(FStringError::new(Unmatched(']'), self.get_pos()).into()); + } + } + } + '}' if !delims.is_empty() => { + let last_delim = delims.pop(); + match last_delim { + Some('{') => { + expression.push(ch); + } + Some(c) => { + return Err(FStringError::new( + MismatchedDelimiter(c, '}'), + self.get_pos(), + ) + .into()); + } + None => {} + } + } + '}' => { + if expression.trim().is_empty() { + return Err(FStringError::new(EmptyExpression, self.get_pos()).into()); + } + + let ret = if !self_documenting { + vec![self.expr(ExprKind::FormattedValue { + value: Box::new(parse_fstring_expr(&expression, location).map_err( + |e| { + FStringError::new( + InvalidExpression(Box::new(e.error)), + location, + ) + }, + )?), + conversion: conversion as _, + format_spec: spec, + })] + } else { + vec![ + self.expr(ExprKind::Constant { + value: Constant::Str(expression.to_owned() + "="), + kind: None, + }), + self.expr(ExprKind::Constant { + value: trailing_seq.into(), + kind: None, + }), + self.expr(ExprKind::FormattedValue { + value: Box::new( + parse_fstring_expr(&expression, location).map_err(|e| { + FStringError::new( + InvalidExpression(Box::new(e.error)), + location, + ) + })?, + ), + conversion: (if conversion == ConversionFlag::None && spec.is_none() + { + ConversionFlag::Repr + } else { + conversion + }) as _, + format_spec: spec, + }), + ] + }; + return Ok(ret); + } + '"' | '\'' => { + expression.push(ch); + loop { + let Some(c) = self.next_char() else { + return Err(FStringError::new(UnterminatedString, self.get_pos()).into()); + }; + expression.push(c); + if c == ch { + break; + } + } + } + ' ' if self_documenting => { + trailing_seq.push(ch); + } + '\\' => return Err(FStringError::new(UnterminatedString, self.get_pos()).into()), + _ => { + if self_documenting { + return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()); + } + + expression.push(ch); + } + } + } + Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()) + } + + fn parse_spec(&mut self, nested: u8) -> Result, LexicalError> { + let mut spec_constructor = Vec::new(); + let mut constant_piece = String::new(); + while let Some(&next) = self.peek() { + match next { + '{' => { + if !constant_piece.is_empty() { + spec_constructor.push(self.expr(ExprKind::Constant { + value: constant_piece.drain(..).collect::().into(), + kind: None, + })); + } + let parsed_expr = self.parse_fstring(nested + 1)?; + spec_constructor.extend(parsed_expr); + continue; + } + '}' => { + break; + } + _ => { + constant_piece.push(next); + } + } + self.next_char(); + } + if !constant_piece.is_empty() { + spec_constructor.push(self.expr(ExprKind::Constant { + value: constant_piece.drain(..).collect::().into(), + kind: None, + })); + } + Ok(spec_constructor) + } + + fn parse_fstring(&mut self, nested: u8) -> Result, LexicalError> { + if nested >= 2 { + return Err(FStringError::new(ExpressionNestedTooDeeply, self.get_pos()).into()); + } + + let mut content = String::new(); + let mut values = vec![]; + + while let Some(&ch) = self.peek() { + match ch { + '{' => { + self.next_char(); + if nested == 0 { + match self.peek() { + Some('{') => { + self.next_char(); + content.push('{'); + continue; + } + None => { + return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()) + } + _ => {} + } + } + if !content.is_empty() { + values.push(self.expr(ExprKind::Constant { + value: content.drain(..).collect::().into(), + kind: None, + })); + } + + let parsed_values = self.parse_formatted_value(nested)?; + values.extend(parsed_values); + } + '}' => { + if nested > 0 { + break; + } + self.next_char(); + if let Some('}') = self.peek() { + self.next_char(); + content.push('}'); + } else { + return Err(FStringError::new(SingleRbrace, self.get_pos()).into()); + } + } + '\\' if !self.kind.is_raw() => { + self.next_char(); + content.push_str(&self.parse_escaped_char()?); + } + _ => { + content.push(ch); + self.next_char(); + } + } + } + + if !content.is_empty() { + values.push(self.expr(ExprKind::Constant { + value: content.into(), + kind: None, + })) + } + + Ok(values) + } + + fn parse_bytes(&mut self) -> Result { + let mut content = String::new(); + while let Some(ch) = self.next_char() { + match ch { + '\\' if !self.kind.is_raw() => { + content.push_str(&self.parse_escaped_char()?); + } + ch => { + if !ch.is_ascii() { + return Err(LexicalError::new( + LexicalErrorType::OtherError( + "bytes can only contain ASCII literal characters".to_string(), + ), + self.get_pos(), + )); + } + content.push(ch); + } + } + } + + Ok(self.expr(ExprKind::Constant { + value: Constant::Bytes(content.chars().map(|c| c as u8).collect()), + kind: None, + })) + } + + fn parse_string(&mut self) -> Result { + let mut content = String::new(); + while let Some(ch) = self.next_char() { + match ch { + '\\' if !self.kind.is_raw() => { + content.push_str(&self.parse_escaped_char()?); + } + ch => content.push(ch), + } + } + Ok(self.expr(ExprKind::Constant { + value: Constant::Str(content), + kind: self.kind.is_unicode().then(|| "u".to_string()), + })) + } + + fn parse(&mut self) -> Result, LexicalError> { + if self.kind.is_fstring() { + self.parse_fstring(0) + } else if self.kind.is_bytes() { + self.parse_bytes().map(|expr| vec![expr]) + } else { + self.parse_string().map(|expr| vec![expr]) + } + } +} + +fn parse_fstring_expr(source: &str, location: Location) -> Result { + let fstring_body = format!("({source})"); + parse_expression_located(&fstring_body, "", location.with_col_offset(-1)) +} -pub fn parse_strings( +fn parse_string( + source: &str, + kind: StringKind, + triple_quoted: bool, + start: Location, + end: Location, +) -> Result, LexicalError> { + StringParser::new(source, kind, triple_quoted, start, end).parse() +} + +pub(crate) fn parse_strings( values: Vec<(Location, (String, StringKind, bool), Location)>, ) -> Result { // Preserve the initial location and kind. @@ -120,8 +653,146 @@ pub fn parse_strings( #[cfg(test)] mod tests { + use super::*; use crate::parser::parse_program; + fn parse_fstring(source: &str) -> Result, LexicalError> { + StringParser::new( + source, + StringKind::FString, + false, + Location::default(), + Location::default().with_col_offset(source.len() + 3), // 3 for prefix and quotes + ) + .parse() + } + + #[test] + fn test_parse_fstring() { + let source = "{a}{ b }{{foo}}"; + let parse_ast = parse_fstring(source).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_nested_spec() { + let source = "{foo:{spec}}"; + let parse_ast = parse_fstring(source).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_not_nested_spec() { + let source = "{foo:spec}"; + let parse_ast = parse_fstring(source).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_empty_fstring() { + insta::assert_debug_snapshot!(parse_fstring("").unwrap()); + } + + #[test] + fn test_fstring_parse_selfdocumenting_base() { + let src = "{user=}"; + let parse_ast = parse_fstring(src).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_fstring_parse_selfdocumenting_base_more() { + let src = "mix {user=} with text and {second=}"; + let parse_ast = parse_fstring(src).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_fstring_parse_selfdocumenting_format() { + let src = "{user=:>10}"; + let parse_ast = parse_fstring(src).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + fn parse_fstring_error(source: &str) -> FStringErrorType { + parse_fstring(source) + .map_err(|e| match e.error { + LexicalErrorType::FStringError(e) => e, + e => unreachable!("Expected FStringError: {:?}", e), + }) + .err() + .expect("Expected error") + } + + #[test] + fn test_parse_invalid_fstring() { + assert_eq!(parse_fstring_error("{5!a"), UnclosedLbrace); + assert_eq!(parse_fstring_error("{5!a1}"), UnclosedLbrace); + assert_eq!(parse_fstring_error("{5!"), UnclosedLbrace); + assert_eq!(parse_fstring_error("abc{!a 'cat'}"), EmptyExpression); + assert_eq!(parse_fstring_error("{!a"), EmptyExpression); + assert_eq!(parse_fstring_error("{ !a}"), EmptyExpression); + + assert_eq!(parse_fstring_error("{5!}"), InvalidConversionFlag); + assert_eq!(parse_fstring_error("{5!x}"), InvalidConversionFlag); + + assert_eq!( + parse_fstring_error("{a:{a:{b}}}"), + ExpressionNestedTooDeeply + ); + + assert_eq!(parse_fstring_error("{a:b}}"), SingleRbrace); + assert_eq!(parse_fstring_error("}"), SingleRbrace); + assert_eq!(parse_fstring_error("{a:{b}"), UnclosedLbrace); + assert_eq!(parse_fstring_error("{"), UnclosedLbrace); + + assert_eq!(parse_fstring_error("{}"), EmptyExpression); + + // TODO: check for InvalidExpression enum? + assert!(parse_fstring("{class}").is_err()); + } + + #[test] + fn test_parse_fstring_not_equals() { + let source = "{1 != 2}"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_equals() { + let source = "{42 == 42}"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_selfdoc_prec_space() { + let source = "{x =}"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_selfdoc_trailing_space() { + let source = "{x= }"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_yield_expr() { + let source = "{yield}"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + #[test] fn test_parse_string_concat() { let source = "'Hello ' 'world'"; diff --git a/compiler/parser/src/string_parser.rs b/compiler/parser/src/string_parser.rs deleted file mode 100644 index c18ba0f97d4..00000000000 --- a/compiler/parser/src/string_parser.rs +++ /dev/null @@ -1,675 +0,0 @@ -use self::FStringErrorType::*; -use crate::{ - ast::{Constant, ConversionFlag, Expr, ExprKind, Location}, - error::{FStringError, FStringErrorType, LexicalError, LexicalErrorType, ParseError}, - parser::parse_expression_located, - token::StringKind, -}; -use std::{iter, str}; - -/// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798 -pub const MAX_UNICODE_NAME: usize = 88; - -pub struct StringParser<'a> { - chars: iter::Peekable>, - kind: StringKind, - str_start: Location, - str_end: Location, - location: Location, -} - -impl<'a> StringParser<'a> { - pub fn new( - source: &'a str, - kind: StringKind, - triple_quoted: bool, - str_start: Location, - str_end: Location, - ) -> Self { - let offset = kind.prefix_len() + if triple_quoted { 3 } else { 1 }; - Self { - chars: source.chars().peekable(), - kind, - str_start, - str_end, - location: str_start.with_col_offset(offset), - } - } - - #[inline] - fn next_char(&mut self) -> Option { - let Some(c) = self.chars.next() else { - return None - }; - if c == '\n' { - self.location.newline(); - } else { - self.location.go_right(); - } - Some(c) - } - - #[inline] - fn peek(&mut self) -> Option<&char> { - self.chars.peek() - } - - #[inline] - fn get_pos(&self) -> Location { - self.location - } - - #[inline] - fn expr(&self, node: ExprKind) -> Expr { - Expr::new(self.str_start, self.str_end, node) - } - - fn parse_unicode_literal(&mut self, literal_number: usize) -> Result { - let mut p: u32 = 0u32; - let unicode_error = LexicalError::new(LexicalErrorType::UnicodeError, self.get_pos()); - for i in 1..=literal_number { - match self.next_char() { - Some(c) => match c.to_digit(16) { - Some(d) => p += d << ((literal_number - i) * 4), - None => return Err(unicode_error), - }, - None => return Err(unicode_error), - } - } - match p { - 0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER), - _ => std::char::from_u32(p).ok_or(unicode_error), - } - } - - fn parse_octet(&mut self, first: char) -> char { - let mut octet_content = String::new(); - octet_content.push(first); - while octet_content.len() < 3 { - if let Some('0'..='7') = self.peek() { - octet_content.push(self.next_char().unwrap()) - } else { - break; - } - } - let value = u32::from_str_radix(&octet_content, 8).unwrap(); - char::from_u32(value).unwrap() - } - - fn parse_unicode_name(&mut self) -> Result { - let start_pos = self.get_pos(); - match self.next_char() { - Some('{') => {} - _ => return Err(LexicalError::new(LexicalErrorType::StringError, start_pos)), - } - let start_pos = self.get_pos(); - let mut name = String::new(); - loop { - match self.next_char() { - Some('}') => break, - Some(c) => name.push(c), - None => { - return Err(LexicalError::new( - LexicalErrorType::StringError, - self.get_pos(), - )) - } - } - } - - if name.len() > MAX_UNICODE_NAME { - return Err(LexicalError::new( - LexicalErrorType::UnicodeError, - self.get_pos(), - )); - } - - unicode_names2::character(&name) - .ok_or_else(|| LexicalError::new(LexicalErrorType::UnicodeError, start_pos)) - } - - fn parse_escaped_char(&mut self) -> Result { - match self.next_char() { - Some(c) => { - let char = match c { - '\\' => '\\', - '\'' => '\'', - '\"' => '"', - 'a' => '\x07', - 'b' => '\x08', - 'f' => '\x0c', - 'n' => '\n', - 'r' => '\r', - 't' => '\t', - 'v' => '\x0b', - o @ '0'..='7' => self.parse_octet(o), - 'x' => self.parse_unicode_literal(2)?, - 'u' if !self.kind.is_bytes() => self.parse_unicode_literal(4)?, - 'U' if !self.kind.is_bytes() => self.parse_unicode_literal(8)?, - 'N' if !self.kind.is_bytes() => self.parse_unicode_name()?, - // Special cases where the escape sequence is not a single character - '\n' => return Ok("".to_string()), - c => { - if self.kind.is_bytes() && !c.is_ascii() { - return Err(LexicalError { - error: LexicalErrorType::OtherError( - "bytes can only contain ASCII literal characters".to_owned(), - ), - location: self.get_pos(), - }); - } - return Ok(format!("\\{c}")); - } - }; - Ok(char.to_string()) - } - None => Err(LexicalError { - error: LexicalErrorType::StringError, - location: self.get_pos(), - }), - } - } - - fn parse_formatted_value(&mut self, nested: u8) -> Result, LexicalError> { - let mut expression = String::new(); - let mut spec = None; - let mut delims = Vec::new(); - let mut conversion = ConversionFlag::None; - let mut self_documenting = false; - let mut trailing_seq = String::new(); - let location = self.get_pos(); - - while let Some(ch) = self.next_char() { - match ch { - // can be integrated better with the remaining code, but as a starting point ok - // in general I would do here a tokenizing of the fstrings to omit this peeking. - '!' | '=' | '>' | '<' if self.peek() == Some(&'=') => { - expression.push(ch); - expression.push('='); - self.next_char(); - } - '!' if delims.is_empty() && self.peek() != Some(&'=') => { - if expression.trim().is_empty() { - return Err(FStringError::new(EmptyExpression, self.get_pos()).into()); - } - - conversion = match self.next_char() { - Some('s') => ConversionFlag::Str, - Some('a') => ConversionFlag::Ascii, - Some('r') => ConversionFlag::Repr, - Some(_) => { - return Err( - FStringError::new(InvalidConversionFlag, self.get_pos()).into() - ); - } - None => { - return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()); - } - }; - - match self.peek() { - Some('}' | ':') => {} - Some(_) | None => { - return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()); - } - } - } - - // match a python 3.8 self documenting expression - // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}' - '=' if self.peek() != Some(&'=') && delims.is_empty() => { - self_documenting = true; - } - - ':' if delims.is_empty() => { - let parsed_spec = self.parse_spec(nested)?; - - spec = Some(Box::new(self.expr(ExprKind::JoinedStr { - values: parsed_spec, - }))); - } - '(' | '{' | '[' => { - expression.push(ch); - delims.push(ch); - } - ')' => { - let last_delim = delims.pop(); - match last_delim { - Some('(') => { - expression.push(ch); - } - Some(c) => { - return Err(FStringError::new( - MismatchedDelimiter(c, ')'), - self.get_pos(), - ) - .into()); - } - None => { - return Err(FStringError::new(Unmatched(')'), self.get_pos()).into()); - } - } - } - ']' => { - let last_delim = delims.pop(); - match last_delim { - Some('[') => { - expression.push(ch); - } - Some(c) => { - return Err(FStringError::new( - MismatchedDelimiter(c, ']'), - self.get_pos(), - ) - .into()); - } - None => { - return Err(FStringError::new(Unmatched(']'), self.get_pos()).into()); - } - } - } - '}' if !delims.is_empty() => { - let last_delim = delims.pop(); - match last_delim { - Some('{') => { - expression.push(ch); - } - Some(c) => { - return Err(FStringError::new( - MismatchedDelimiter(c, '}'), - self.get_pos(), - ) - .into()); - } - None => {} - } - } - '}' => { - if expression.trim().is_empty() { - return Err(FStringError::new(EmptyExpression, self.get_pos()).into()); - } - - let ret = if !self_documenting { - vec![self.expr(ExprKind::FormattedValue { - value: Box::new(parse_fstring_expr(&expression, location).map_err( - |e| { - FStringError::new( - InvalidExpression(Box::new(e.error)), - location, - ) - }, - )?), - conversion: conversion as _, - format_spec: spec, - })] - } else { - vec![ - self.expr(ExprKind::Constant { - value: Constant::Str(expression.to_owned() + "="), - kind: None, - }), - self.expr(ExprKind::Constant { - value: trailing_seq.into(), - kind: None, - }), - self.expr(ExprKind::FormattedValue { - value: Box::new( - parse_fstring_expr(&expression, location).map_err(|e| { - FStringError::new( - InvalidExpression(Box::new(e.error)), - location, - ) - })?, - ), - conversion: (if conversion == ConversionFlag::None && spec.is_none() - { - ConversionFlag::Repr - } else { - conversion - }) as _, - format_spec: spec, - }), - ] - }; - return Ok(ret); - } - '"' | '\'' => { - expression.push(ch); - loop { - let Some(c) = self.next_char() else { - return Err(FStringError::new(UnterminatedString, self.get_pos()).into()); - }; - expression.push(c); - if c == ch { - break; - } - } - } - ' ' if self_documenting => { - trailing_seq.push(ch); - } - '\\' => return Err(FStringError::new(UnterminatedString, self.get_pos()).into()), - _ => { - if self_documenting { - return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()); - } - - expression.push(ch); - } - } - } - Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()) - } - - fn parse_spec(&mut self, nested: u8) -> Result, LexicalError> { - let mut spec_constructor = Vec::new(); - let mut constant_piece = String::new(); - while let Some(&next) = self.peek() { - match next { - '{' => { - if !constant_piece.is_empty() { - spec_constructor.push(self.expr(ExprKind::Constant { - value: constant_piece.drain(..).collect::().into(), - kind: None, - })); - } - let parsed_expr = self.parse_fstring(nested + 1)?; - spec_constructor.extend(parsed_expr); - continue; - } - '}' => { - break; - } - _ => { - constant_piece.push(next); - } - } - self.next_char(); - } - if !constant_piece.is_empty() { - spec_constructor.push(self.expr(ExprKind::Constant { - value: constant_piece.drain(..).collect::().into(), - kind: None, - })); - } - Ok(spec_constructor) - } - - fn parse_fstring(&mut self, nested: u8) -> Result, LexicalError> { - if nested >= 2 { - return Err(FStringError::new(ExpressionNestedTooDeeply, self.get_pos()).into()); - } - - let mut content = String::new(); - let mut values = vec![]; - - while let Some(&ch) = self.peek() { - match ch { - '{' => { - self.next_char(); - if nested == 0 { - match self.peek() { - Some('{') => { - self.next_char(); - content.push('{'); - continue; - } - None => { - return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()) - } - _ => {} - } - } - if !content.is_empty() { - values.push(self.expr(ExprKind::Constant { - value: content.drain(..).collect::().into(), - kind: None, - })); - } - - let parsed_values = self.parse_formatted_value(nested)?; - values.extend(parsed_values); - } - '}' => { - if nested > 0 { - break; - } - self.next_char(); - if let Some('}') = self.peek() { - self.next_char(); - content.push('}'); - } else { - return Err(FStringError::new(SingleRbrace, self.get_pos()).into()); - } - } - '\\' if !self.kind.is_raw() => { - self.next_char(); - content.push_str(&self.parse_escaped_char()?); - } - _ => { - content.push(ch); - self.next_char(); - } - } - } - - if !content.is_empty() { - values.push(self.expr(ExprKind::Constant { - value: content.into(), - kind: None, - })) - } - - Ok(values) - } - - pub fn parse_bytes(&mut self) -> Result { - let mut content = String::new(); - while let Some(ch) = self.next_char() { - match ch { - '\\' if !self.kind.is_raw() => { - content.push_str(&self.parse_escaped_char()?); - } - ch => { - if !ch.is_ascii() { - return Err(LexicalError::new( - LexicalErrorType::OtherError( - "bytes can only contain ASCII literal characters".to_string(), - ), - self.get_pos(), - )); - } - content.push(ch); - } - } - } - - Ok(self.expr(ExprKind::Constant { - value: Constant::Bytes(content.chars().map(|c| c as u8).collect()), - kind: None, - })) - } - - pub fn parse_string(&mut self) -> Result { - let mut content = String::new(); - while let Some(ch) = self.next_char() { - match ch { - '\\' if !self.kind.is_raw() => { - content.push_str(&self.parse_escaped_char()?); - } - ch => content.push(ch), - } - } - Ok(self.expr(ExprKind::Constant { - value: Constant::Str(content), - kind: self.kind.is_unicode().then(|| "u".to_string()), - })) - } - - pub fn parse(&mut self) -> Result, LexicalError> { - if self.kind.is_fstring() { - self.parse_fstring(0) - } else if self.kind.is_bytes() { - self.parse_bytes().map(|expr| vec![expr]) - } else { - self.parse_string().map(|expr| vec![expr]) - } - } -} - -fn parse_fstring_expr(source: &str, location: Location) -> Result { - let fstring_body = format!("({source})"); - parse_expression_located(&fstring_body, "", location.with_col_offset(-1)) -} - -pub fn parse_string( - source: &str, - kind: StringKind, - triple_quoted: bool, - start: Location, - end: Location, -) -> Result, LexicalError> { - StringParser::new(source, kind, triple_quoted, start, end).parse() -} - -#[cfg(test)] -mod tests { - use super::*; - - fn parse_fstring(source: &str) -> Result, LexicalError> { - StringParser::new( - source, - StringKind::FString, - false, - Location::default(), - Location::default().with_col_offset(source.len() + 3), // 3 for prefix and quotes - ) - .parse() - } - - #[test] - fn test_parse_fstring() { - let source = "{a}{ b }{{foo}}"; - let parse_ast = parse_fstring(source).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_nested_spec() { - let source = "{foo:{spec}}"; - let parse_ast = parse_fstring(source).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_not_nested_spec() { - let source = "{foo:spec}"; - let parse_ast = parse_fstring(source).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_empty_fstring() { - insta::assert_debug_snapshot!(parse_fstring("").unwrap()); - } - - #[test] - fn test_fstring_parse_selfdocumenting_base() { - let src = "{user=}"; - let parse_ast = parse_fstring(src).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_fstring_parse_selfdocumenting_base_more() { - let src = "mix {user=} with text and {second=}"; - let parse_ast = parse_fstring(src).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_fstring_parse_selfdocumenting_format() { - let src = "{user=:>10}"; - let parse_ast = parse_fstring(src).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - fn parse_fstring_error(source: &str) -> FStringErrorType { - parse_fstring(source) - .map_err(|e| match e.error { - LexicalErrorType::FStringError(e) => e, - e => unreachable!("Expected FStringError: {:?}", e), - }) - .err() - .expect("Expected error") - } - - #[test] - fn test_parse_invalid_fstring() { - assert_eq!(parse_fstring_error("{5!a"), UnclosedLbrace); - assert_eq!(parse_fstring_error("{5!a1}"), UnclosedLbrace); - assert_eq!(parse_fstring_error("{5!"), UnclosedLbrace); - assert_eq!(parse_fstring_error("abc{!a 'cat'}"), EmptyExpression); - assert_eq!(parse_fstring_error("{!a"), EmptyExpression); - assert_eq!(parse_fstring_error("{ !a}"), EmptyExpression); - - assert_eq!(parse_fstring_error("{5!}"), InvalidConversionFlag); - assert_eq!(parse_fstring_error("{5!x}"), InvalidConversionFlag); - - assert_eq!( - parse_fstring_error("{a:{a:{b}}}"), - ExpressionNestedTooDeeply - ); - - assert_eq!(parse_fstring_error("{a:b}}"), SingleRbrace); - assert_eq!(parse_fstring_error("}"), SingleRbrace); - assert_eq!(parse_fstring_error("{a:{b}"), UnclosedLbrace); - assert_eq!(parse_fstring_error("{"), UnclosedLbrace); - - assert_eq!(parse_fstring_error("{}"), EmptyExpression); - - // TODO: check for InvalidExpression enum? - assert!(parse_fstring("{class}").is_err()); - } - - #[test] - fn test_parse_fstring_not_equals() { - let source = "{1 != 2}"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_equals() { - let source = "{42 == 42}"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_selfdoc_prec_space() { - let source = "{x =}"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_selfdoc_trailing_space() { - let source = "{x= }"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_yield_expr() { - let source = "{yield}"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } -} diff --git a/compiler/parser/src/token.rs b/compiler/parser/src/token.rs index b51b2f47768..3fa6ff145fa 100644 --- a/compiler/parser/src/token.rs +++ b/compiler/parser/src/token.rs @@ -1,86 +1,154 @@ -//! Different token definitions. -//! Loosely based on token.h from CPython source: +//! Token type for Python source code created by the lexer and consumed by the parser. +//! +//! This module defines the tokens that the lexer recognizes. The tokens are +//! loosely based on the token definitions found in the [CPython source]. +//! +//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Include/internal/pycore_token.h use num_bigint::BigInt; use std::fmt; -/// Python source code can be tokenized in a sequence of these tokens. +/// The set of tokens the Python source code can be tokenized in. #[derive(Clone, Debug, PartialEq)] pub enum Tok { + /// Token value for a name, commonly known as an identifier. Name { + /// The name value. name: String, }, + /// Token value for an integer. Int { + /// The integer value. value: BigInt, }, + /// Token value for a floating point number. Float { + /// The float value. value: f64, }, + /// Token value for a complex number. Complex { + /// The real part of the complex number. real: f64, + /// The imaginary part of the complex number. imag: f64, }, + /// Token value for a string. String { + /// The string value. value: String, + /// The kind of string. kind: StringKind, + /// Whether the string is triple quoted. triple_quoted: bool, }, + /// Token value for a comment. These are filtered out of the token stream prior to parsing. + Comment(String), + /// Token value for a newline. Newline, + /// Token value for a newline that is not a logical line break. These are filtered out of + /// the token stream prior to parsing. NonLogicalNewline, + /// Token value for an indent. Indent, + /// Token value for a dedent. Dedent, - StartModule, - StartInteractive, - StartExpression, EndOfFile, + /// Token value for a left parenthesis `(`. Lpar, + /// Token value for a right parenthesis `)`. Rpar, + /// Token value for a left square bracket `[`. Lsqb, + /// Token value for a right square bracket `]`. Rsqb, + /// Token value for a colon `:`. Colon, + /// Token value for a comma `,`. Comma, - Comment(String), + /// Token value for a semicolon `;`. Semi, + /// Token value for plus `+`. Plus, + /// Token value for minus `-`. Minus, + /// Token value for star `*`. Star, + /// Token value for slash `/`. Slash, - Vbar, // '|' - Amper, // '&' + /// Token value for vertical bar `|`. + Vbar, + /// Token value for ampersand `&`. + Amper, + /// Token value for less than `<`. Less, + /// Token value for greater than `>`. Greater, + /// Token value for equal `=`. Equal, + /// Token value for dot `.`. Dot, + /// Token value for percent `%`. Percent, + /// Token value for left bracket `{`. Lbrace, + /// Token value for right bracket `}`. Rbrace, + /// Token value for double equal `==`. EqEqual, + /// Token value for not equal `!=`. NotEqual, + /// Token value for less than or equal `<=`. LessEqual, + /// Token value for greater than or equal `>=`. GreaterEqual, + /// Token value for tilde `~`. Tilde, + /// Token value for caret `^`. CircumFlex, + /// Token value for left shift `<<`. LeftShift, + /// Token value for right shift `>>`. RightShift, + /// Token value for double star `**`. DoubleStar, - DoubleStarEqual, // '**=' + /// Token value for double star equal `**=`. + DoubleStarEqual, + /// Token value for plus equal `+=`. PlusEqual, + /// Token value for minus equal `-=`. MinusEqual, + /// Token value for star equal `*=`. StarEqual, + /// Token value for slash equal `/=`. SlashEqual, + /// Token value for percent equal `%=`. PercentEqual, - AmperEqual, // '&=' + /// Token value for ampersand equal `&=`. + AmperEqual, + /// Token value for vertical bar equal `|=`. VbarEqual, - CircumflexEqual, // '^=' + /// Token value for caret equal `^=`. + CircumflexEqual, + /// Token value for left shift equal `<<=`. LeftShiftEqual, + /// Token value for right shift equal `>>=`. RightShiftEqual, - DoubleSlash, // '//' + /// Token value for double slash `//`. + DoubleSlash, + /// Token value for double slash equal `//=`. DoubleSlashEqual, + /// Token value for colon equal `:=`. ColonEqual, + /// Token value for at `@`. At, + /// Token value for at equal `@=`. AtEqual, + /// Token value for arrow `->`. Rarrow, + /// Token value for ellipsis `...`. Ellipsis, + // Self documenting. // Keywords (alphabetically): False, None, @@ -118,6 +186,11 @@ pub enum Tok { While, With, Yield, + + // RustPython specific. + StartModule, + StartInteractive, + StartExpression, } impl fmt::Display for Tok { @@ -231,14 +304,25 @@ impl fmt::Display for Tok { } } +/// The kind of string literal as described in the [String and Bytes literals] +/// section of the Python reference. +/// +/// [String and Bytes literals]: https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals #[derive(PartialEq, Eq, Debug, Clone)] pub enum StringKind { + /// A normal string literal with no prefix. String, + /// A f-string literal, with a `f` or `F` prefix. FString, + /// A byte string literal, with a `b` or `B` prefix. Bytes, + /// A raw string literal, with a `r` or `R` prefix. RawString, + /// A raw f-string literal, with a `rf`/`fr` or `rF`/`Fr` or `Rf`/`fR` or `RF`/`FR` prefix. RawFString, + /// A raw byte string literal, with a `rb`/`br` or `rB`/`Br` or `Rb`/`bR` or `RB`/`BR` prefix. RawBytes, + /// A unicode string literal, with a `u` or `U` prefix. Unicode, } @@ -286,25 +370,33 @@ impl fmt::Display for StringKind { } impl StringKind { + /// Returns true if the string is a raw string, i,e one of + /// [`StringKind::RawString`] or [`StringKind::RawFString`] or [`StringKind::RawBytes`]. pub fn is_raw(&self) -> bool { use StringKind::{RawBytes, RawFString, RawString}; matches!(self, RawString | RawFString | RawBytes) } + /// Returns true if the string is an f-string, i,e one of + /// [`StringKind::FString`] or [`StringKind::RawFString`]. pub fn is_fstring(&self) -> bool { use StringKind::{FString, RawFString}; matches!(self, FString | RawFString) } + /// Returns true if the string is a byte string, i,e one of + /// [`StringKind::Bytes`] or [`StringKind::RawBytes`]. pub fn is_bytes(&self) -> bool { use StringKind::{Bytes, RawBytes}; matches!(self, Bytes | RawBytes) } + /// Returns true if the string is a unicode string, i,e [`StringKind::Unicode`]. pub fn is_unicode(&self) -> bool { matches!(self, StringKind::Unicode) } + /// Returns the number of characters in the prefix. pub fn prefix_len(&self) -> usize { use StringKind::*; match self { diff --git a/derive-impl/Cargo.toml b/derive-impl/Cargo.toml index 2f1d936a547..31fab8abcc4 100644 --- a/derive-impl/Cargo.toml +++ b/derive-impl/Cargo.toml @@ -7,12 +7,13 @@ edition = "2021" rustpython-compiler-core = { path = "../compiler/core", version = "0.2.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", branch = "main" } -indexmap = "1.8.1" -itertools = "0.10.3" +indexmap = { workspace = true } +itertools = { workspace = true } +once_cell = { workspace = true } +syn = { workspace = true, features = ["full", "extra-traits"] } + maplit = "1.0.2" -once_cell = "1.10.0" proc-macro2 = "1.0.37" quote = "1.0.18" -syn = { version = "1.0.91", features = ["full", "extra-traits"] } syn-ext = { version = "0.4.0", features = ["full"] } textwrap = { version = "0.15.0", default-features = false } diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 3548bcf8b43..f14d6db49e7 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -13,4 +13,4 @@ proc-macro = true [dependencies] rustpython-compiler = { path = "../compiler", version = "0.2.0" } rustpython-derive-impl = { path = "../derive-impl" } -syn = "1.0.91" +syn = { workspace = true } diff --git a/examples/call_between_rust_and_python.rs b/examples/call_between_rust_and_python.rs index be93fb971e0..59130422896 100644 --- a/examples/call_between_rust_and_python.rs +++ b/examples/call_between_rust_and_python.rs @@ -1,14 +1,17 @@ -use rustpython_vm::{ +use rustpython::vm::{ pyclass, pymodule, PyObject, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, }; -pub(crate) use rust_py_module::make_module; - pub fn main() { - let interp = rustpython_vm::Interpreter::with_init(Default::default(), |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - vm.add_native_module("rust_py_module".to_owned(), Box::new(make_module)); - }); + let interp = rustpython::InterpreterConfig::new() + .init_stdlib() + .init_hook(Box::new(|vm| { + vm.add_native_module( + "rust_py_module".to_owned(), + Box::new(rust_py_module::make_module), + ); + })) + .interpreter(); interp.enter(|vm| { vm.insert_sys_path(vm.new_pyobj("examples")) @@ -30,7 +33,7 @@ pub fn main() { #[pymodule] mod rust_py_module { use super::*; - use rustpython_vm::{builtins::PyList, convert::ToPyObject, PyObjectRef}; + use rustpython::vm::{builtins::PyList, convert::ToPyObject, PyObjectRef}; #[pyfunction] fn rust_function( diff --git a/extra_tests/snippets/builtin_eval.py b/extra_tests/snippets/builtin_eval.py new file mode 100644 index 00000000000..6375bd0c1ae --- /dev/null +++ b/extra_tests/snippets/builtin_eval.py @@ -0,0 +1,4 @@ +assert 3 == eval('1+2') + +code = compile('5+3', 'x.py', 'eval') +assert eval(code) == 8 diff --git a/extra_tests/snippets/builtin_str_unicode.py b/extra_tests/snippets/builtin_str_unicode.py index 82294697aa1..f270ee1149e 100644 --- a/extra_tests/snippets/builtin_str_unicode.py +++ b/extra_tests/snippets/builtin_str_unicode.py @@ -22,7 +22,7 @@ # testing unicodedata.ucd_3_2_0 for idna assert "abcСĤ".encode("idna") == b'xn--abc-7sa390b' -# TODO: fix: assert "abc䄣IJ".encode("idna") == b'xn--abcij-zb5f' +assert "abc䄣IJ".encode("idna") == b'xn--abcij-zb5f' # from CPython tests assert "python.org".encode("idna") == b"python.org" diff --git a/extra_tests/snippets/builtins.py b/extra_tests/snippets/builtins.py deleted file mode 100644 index 76b28a7b9c8..00000000000 --- a/extra_tests/snippets/builtins.py +++ /dev/null @@ -1,13 +0,0 @@ -x = sum(map(int, ['1', '2', '3'])) -assert x == 6 - -assert callable(type) -# TODO: -# assert callable(callable) - -assert type(frozenset) is type - -assert 3 == eval('1+2') - -code = compile('5+3', 'x.py', 'eval') -assert eval(code) == 8 diff --git a/extra_tests/snippets/stdlib_datetime.py b/extra_tests/snippets/stdlib_datetime.py index 5302117f370..e1b3f7db082 100644 --- a/extra_tests/snippets/stdlib_datetime.py +++ b/extra_tests/snippets/stdlib_datetime.py @@ -131,6 +131,11 @@ def __init__(self, offset, name): assert_raises(NotImplementedError, ne.utcoffset, dt) assert_raises(NotImplementedError, ne.dst, dt) +# unsupport format in strptime returns arg itself +# in linux. Todo: add cases for Mac/Windows +if sys.platform in ('linux'): + assert_equal(_time.strftime("%?"), "%?") + # XXX: bug #1302 # def test_normal(self): #fo = FixedOffset(3, "Three") diff --git a/extra_tests/snippets/stdlib_math.py b/extra_tests/snippets/stdlib_math.py index b40e750d117..442bbc97a5d 100644 --- a/extra_tests/snippets/stdlib_math.py +++ b/extra_tests/snippets/stdlib_math.py @@ -291,94 +291,3 @@ def assertAllNotClose(examples, *args, **kwargs): assert math.fmod(-3.0, NINF) == -3.0 assert math.fmod(0.0, 3.0) == 0.0 assert math.fmod(0.0, NINF) == 0.0 - -""" -TODO: math.remainder was added to CPython in 3.7 and RustPython CI runs on 3.6. -So put the tests of math.remainder in a comment for now. -https://github.com/RustPython/RustPython/pull/1589#issuecomment-551424940 -""" - -# testcases = [ -# # Remainders modulo 1, showing the ties-to-even behaviour. -# '-4.0 1 -0.0', -# '-3.8 1 0.8', -# '-3.0 1 -0.0', -# '-2.8 1 -0.8', -# '-2.0 1 -0.0', -# '-1.8 1 0.8', -# '-1.0 1 -0.0', -# '-0.8 1 -0.8', -# '-0.0 1 -0.0', -# ' 0.0 1 0.0', -# ' 0.8 1 0.8', -# ' 1.0 1 0.0', -# ' 1.8 1 -0.8', -# ' 2.0 1 0.0', -# ' 2.8 1 0.8', -# ' 3.0 1 0.0', -# ' 3.8 1 -0.8', -# ' 4.0 1 0.0', - -# # Reductions modulo 2*pi -# '0x0.0p+0 0x1.921fb54442d18p+2 0x0.0p+0', -# '0x1.921fb54442d18p+0 0x1.921fb54442d18p+2 0x1.921fb54442d18p+0', -# '0x1.921fb54442d17p+1 0x1.921fb54442d18p+2 0x1.921fb54442d17p+1', -# '0x1.921fb54442d18p+1 0x1.921fb54442d18p+2 0x1.921fb54442d18p+1', -# '0x1.921fb54442d19p+1 0x1.921fb54442d18p+2 -0x1.921fb54442d17p+1', -# '0x1.921fb54442d17p+2 0x1.921fb54442d18p+2 -0x0.0000000000001p+2', -# '0x1.921fb54442d18p+2 0x1.921fb54442d18p+2 0x0p0', -# '0x1.921fb54442d19p+2 0x1.921fb54442d18p+2 0x0.0000000000001p+2', -# '0x1.2d97c7f3321d1p+3 0x1.921fb54442d18p+2 0x1.921fb54442d14p+1', -# '0x1.2d97c7f3321d2p+3 0x1.921fb54442d18p+2 -0x1.921fb54442d18p+1', -# '0x1.2d97c7f3321d3p+3 0x1.921fb54442d18p+2 -0x1.921fb54442d14p+1', -# '0x1.921fb54442d17p+3 0x1.921fb54442d18p+2 -0x0.0000000000001p+3', -# '0x1.921fb54442d18p+3 0x1.921fb54442d18p+2 0x0p0', -# '0x1.921fb54442d19p+3 0x1.921fb54442d18p+2 0x0.0000000000001p+3', -# '0x1.f6a7a2955385dp+3 0x1.921fb54442d18p+2 0x1.921fb54442d14p+1', -# '0x1.f6a7a2955385ep+3 0x1.921fb54442d18p+2 0x1.921fb54442d18p+1', -# '0x1.f6a7a2955385fp+3 0x1.921fb54442d18p+2 -0x1.921fb54442d14p+1', -# '0x1.1475cc9eedf00p+5 0x1.921fb54442d18p+2 0x1.921fb54442d10p+1', -# '0x1.1475cc9eedf01p+5 0x1.921fb54442d18p+2 -0x1.921fb54442d10p+1', - -# # Symmetry with respect to signs. -# ' 1 0.c 0.4', -# '-1 0.c -0.4', -# ' 1 -0.c 0.4', -# '-1 -0.c -0.4', -# ' 1.4 0.c -0.4', -# '-1.4 0.c 0.4', -# ' 1.4 -0.c -0.4', -# '-1.4 -0.c 0.4', - -# # Huge modulus, to check that the underlying algorithm doesn't -# # rely on 2.0 * modulus being representable. -# '0x1.dp+1023 0x1.4p+1023 0x0.9p+1023', -# '0x1.ep+1023 0x1.4p+1023 -0x0.ap+1023', -# '0x1.fp+1023 0x1.4p+1023 -0x0.9p+1023', -# ] - -# for case in testcases: -# x_hex, y_hex, expected_hex = case.split() -# # print(x_hex, y_hex, expected_hex) -# x = float.fromhex(x_hex) -# y = float.fromhex(y_hex) -# expected = float.fromhex(expected_hex) -# actual = math.remainder(x, y) -# # Cheap way of checking that the floats are -# # as identical as we need them to be. -# assert actual.hex() == expected.hex() -# # self.assertEqual(actual.hex(), expected.hex()) - - -# # Test tiny subnormal modulus: there's potential for -# # getting the implementation wrong here (for example, -# # by assuming that modulus/2 is exactly representable). -# tiny = float.fromhex('1p-1074') # min +ve subnormal -# for n in range(-25, 25): -# if n == 0: -# continue -# y = n * tiny -# for m in range(100): -# x = m * tiny -# actual = math.remainder(x, y) -# actual = math.remainder(-x, y) \ No newline at end of file diff --git a/extra_tests/snippets/stdlib_string.py b/extra_tests/snippets/stdlib_string.py index 4435f663350..9151d2f593d 100644 --- a/extra_tests/snippets/stdlib_string.py +++ b/extra_tests/snippets/stdlib_string.py @@ -8,17 +8,15 @@ assert string.hexdigits == '0123456789abcdefABCDEF' assert string.octdigits == '01234567' assert string.punctuation == '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~' -# FIXME -#assert string.whitespace == ' \t\n\r\x0b\x0c', string.whitespace -#assert string.printable == '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c' +assert string.whitespace == ' \t\n\r\x0b\x0c', string.whitespace +assert string.printable == '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c' assert string.capwords('bla bla', ' ') == 'Bla Bla' from string import Template s = Template('$who likes $what') -# TODO: -# r = s.substitute(who='tim', what='kung pow') -# print(r) +r = s.substitute(who='tim', what='kung pow') +assert r == 'tim likes kung pow' from string import Formatter diff --git a/extra_tests/snippets/syntax_class.py b/extra_tests/snippets/syntax_class.py index d0ae4ef26aa..e0dc60723e2 100644 --- a/extra_tests/snippets/syntax_class.py +++ b/extra_tests/snippets/syntax_class.py @@ -193,22 +193,20 @@ class B: assert B.__doc__ == "Docstring" -# TODO: uncomment once free vars/cells are working # The symboltable sees that b() is referring to a in the nested scope, # so it marks it as non local. When it's executed, it walks up the scopes # and still finds the a from the class scope. -# a = 1 -# def nested_scope(): -# a = 2 -# class A: -# a = 3 -# def b(): -# assert a == 2 -# b() -# assert a == 3 -# A.b() -# nested_scope() - +a = 1 +def nested_scope(): + a = 2 + class A: + a = 3 + def b(): + assert a == 2 + b() + assert a == 3 + A.b() +nested_scope() # Multiple inheritance and mro tests. diff --git a/extra_tests/snippets/syntax_comprehension.py b/extra_tests/snippets/syntax_comprehension.py index f8b4cec5e08..41bd1ec8512 100644 --- a/extra_tests/snippets/syntax_comprehension.py +++ b/extra_tests/snippets/syntax_comprehension.py @@ -12,8 +12,7 @@ (3, 2), (3, 5), (3, 10)] v = {b * 2 for b in x} -# TODO: how to check set equality? -# assert v == {2, 6, 4} +assert v == {2, 6, 4} u = {str(b): b-2 for b in x} assert u['3'] == 1 diff --git a/extra_tests/snippets/syntax_fstring.py b/extra_tests/snippets/syntax_fstring.py index 4ea79795aa8..cf8ec899491 100644 --- a/extra_tests/snippets/syntax_fstring.py +++ b/extra_tests/snippets/syntax_fstring.py @@ -60,14 +60,13 @@ assert f'{-10:-{"#"}1{0}x}' == ' -0xa' assert f'{-10:{"-"}#{1}0{"x"}}' == ' -0xa' -# TODO: -# spec = "bla" -# assert_raises(ValueError, lambda: f"{16:{spec}}") +spec = "bla" +assert_raises(ValueError, lambda: f"{16:{spec}}") # Normally `!` cannot appear outside of delimiters in the expression but # cpython makes an exception for `!=`, so we should too. -# assert f'{1 != 2}' == 'True' +assert f'{1 != 2}' == 'True' # conversion flags @@ -103,19 +102,17 @@ def __str__(self): assert f'>{v!a}' == r">'\u262e'" - # Test format specifier after conversion flag -#assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:5}' +'#' # TODO: default alignment in cpython is left +assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:5}' +'#' assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:<5}' +'#' assert f'{"42"!s:>5}' == ' 42', '#' + f'{"42"!s:>5}' +'#' -#assert f'{"42"=!s:5}' == '"42"=42 ', '#'+ f'{"42"=!s:5}' +'#' # TODO default alingment in cpython is left +assert f'{"42"=!s:5}' == '"42"=42 ', '#'+ f'{"42"=!s:5}' +'#' assert f'{"42"=!s:<5}' == '"42"=42 ', '#'+ f'{"42"=!s:<5}' +'#' assert f'{"42"=!s:>5}' == '"42"= 42', '#'+ f'{"42"=!s:>5}' +'#' - ### Tests for fstring selfdocumenting form CPython class C: @@ -126,16 +123,16 @@ def assertEqual(self, a,b): x = 'A string' self.assertEqual(f'{10=}', '10=10') -# self.assertEqual(f'{x=}', 'x=' + x )#repr(x)) # TODO: add ' when printing strings -# self.assertEqual(f'{x =}', 'x =' + x )# + repr(x)) # TODO: implement ' handling +self.assertEqual(f'{x=}', 'x=' + repr(x)) +self.assertEqual(f'{x =}', 'x =' + repr(x)) self.assertEqual(f'{x=!s}', 'x=' + str(x)) -# # self.assertEqual(f'{x=!r}', 'x=' + x) #repr(x)) # !r not supported -# self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) +self.assertEqual(f'{x=!r}', 'x=' + repr(x)) +self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) x = 2.71828 self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) self.assertEqual(f'{x=:}', 'x=' + format(x, '')) -self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) # TODO formatspecifier after conversion flsg is currently not supported (also for classical fstrings) +self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) @@ -170,14 +167,13 @@ def assertEqual(self, a,b): self.assertEqual(f'{0>=1}', 'False') # Make sure leading and following text works. -# x = 'foo' -#self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') # TODO ' -# self.assertEqual(f'X{x=}Y', 'Xx='+x+'Y') # just for the moment +x = 'foo' +self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') # Make sure whitespace around the = works. -# self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y') # TODO ' -# self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') # TODO ' -# self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') # TODO ' +self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y') +self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') +self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') # self.assertEqual(f'X{x =}Y', 'Xx ='+x+'Y') # self.assertEqual(f'X{x= }Y', 'Xx= '+x+'Y') diff --git a/extra_tests/snippets/syntax_generator.py b/extra_tests/snippets/syntax_generator.py index 09791f1b057..9f087dde753 100644 --- a/extra_tests/snippets/syntax_generator.py +++ b/extra_tests/snippets/syntax_generator.py @@ -102,9 +102,9 @@ def binary(n): with assert_raises(StopIteration): try: next(binary(5)) - except StopIteration as stopiter: - # TODO: StopIteration.value - assert stopiter.args[0] == 31 + except StopIteration as stop_iter: + assert stop_iter.value == 31 + assert stop_iter.args[0] == 31 raise class SpamException(Exception): diff --git a/jit/Cargo.toml b/jit/Cargo.toml index c04266a3b71..290af1428fe 100644 --- a/jit/Cargo.toml +++ b/jit/Cargo.toml @@ -12,12 +12,13 @@ autotests = false [dependencies] rustpython-compiler-core = { path = "../compiler/core", version = "0.2.0" } +num-traits = { workspace = true } +thiserror = { workspace = true } + cranelift = "0.88.0" cranelift-jit = "0.88.0" cranelift-module = "0.88.0" -libffi = "2.0.0" -num-traits = "0.2" -thiserror = "1.0" +libffi = "3.1.0" [dev-dependencies] rustpython-derive = { path = "../derive", version = "0.2.0" } diff --git a/pylib/Cargo.toml b/pylib/Cargo.toml index ce08e1ebee2..dba74d6c3bc 100644 --- a/pylib/Cargo.toml +++ b/pylib/Cargo.toml @@ -16,4 +16,4 @@ rustpython-compiler-core = { version = "0.2.0", path = "../compiler/core" } rustpython-derive = { version = "0.2.0", path = "../derive" } [build-dependencies] -glob = "0.3" +glob = { workspace = true } diff --git a/src/settings.rs b/src/settings.rs index 711ae00f3b3..836bb14012a 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -245,6 +245,9 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { if name == "warn_default_encoding" { warn_default_encoding = true } + if name == "no_sig_int" { + settings.no_sig_int = true; + } let value = parts.next().map(ToOwned::to_owned); (name, value) })); diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 8164949600a..88b896c60e2 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -18,8 +18,32 @@ rustpython-derive = { path = "../derive" } rustpython-vm = { path = "../vm" } rustpython-common = { path = "../common" } +ahash = { workspace = true } +ascii = { workspace = true } +cfg-if = { workspace = true } +crossbeam-utils = { workspace = true } +hex = { workspace = true } +itertools = { workspace = true } +libc = { workspace = true } +nix = { workspace = true } +num-complex = { workspace = true } +num-bigint = { workspace = true } +num-integer = { workspace = true } +num-rational = { workspace = true } +num-traits = { workspace = true } +num_enum = { workspace = true } +once_cell = { workspace = true } +parking_lot = { workspace = true } + +memchr = "2.4.1" +base64 = "0.13.0" +csv-core = "0.1.10" +libz-sys = { version = "1.1.5", optional = true } +puruspe = "0.1.5" +xml-rs = "0.8.4" + # random -rand = "0.8.5" +rand = { workspace = true } rand_core = "0.6.3" mt19937 = "2.0.1" @@ -50,31 +74,6 @@ crc32fast = "1.3.2" flate2 = "1.0.23" bzip2 = { version = "0.4", optional = true } -num-complex = "0.4.0" -num-bigint = "0.4.3" -num-integer = "0.1.44" -num-rational = "0.4.1" - -crossbeam-utils = "0.8.9" -itertools = "0.10.3" -lexical-parse-float = "0.8.3" -num-traits = "0.2.14" -memchr = "2.4.1" -base64 = "0.13.0" -csv-core = "0.1.10" -hex = "0.4.3" -puruspe = "0.1.5" -nix = "0.24" -xml-rs = "0.8.4" -libc = "0.2.133" -cfg-if = "1.0.0" -ahash = "0.7.6" -libz-sys = { version = "1.1.5", optional = true } -num_enum = "0.5.7" -ascii = "1.0.0" -parking_lot = "0.12.0" -once_cell = "1.13.0" - # uuid [target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32")))'.dependencies] mac_address = "1.1.3" @@ -92,8 +91,8 @@ termios = "0.3.3" gethostname = "0.2.3" socket2 = { version = "0.4.4", features = ["all"] } dns-lookup = "1.0.8" -openssl = { version = "0.10.43", optional = true } -openssl-sys = { version = "0.9.72", optional = true } +openssl = { version = "0.10.45", optional = true } +openssl-sys = { version = "0.9.80", optional = true } openssl-probe = { version = "0.1.5", optional = true } foreign-types-shared = { version = "0.1.1", optional = true } @@ -101,9 +100,9 @@ foreign-types-shared = { version = "0.1.1", optional = true } libsqlite3-sys = { version = "0.25", features = ["min_sqlite_version_3_7_16", "bundled"] } [target.'cfg(windows)'.dependencies] -schannel = "0.1.19" -widestring = "0.5.1" -paste = "1.0.7" +paste = { workspace = true } +schannel = { workspace = true } +widestring = { workspace = true } [target.'cfg(windows)'.dependencies.winapi] version = "0.3.9" diff --git a/stdlib/src/posixsubprocess.rs b/stdlib/src/posixsubprocess.rs index a1ea0c1347f..ad3d89227c9 100644 --- a/stdlib/src/posixsubprocess.rs +++ b/stdlib/src/posixsubprocess.rs @@ -177,6 +177,8 @@ fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result { let mut first_err = None; for exec in args.exec_list.as_slice() { + // not using nix's versions of these functions because those allocate the char-ptr array, + // and we can't allocate if let Some(envp) = procargs.envp { unsafe { libc::execve(exec.s.as_ptr(), procargs.argv.as_ptr(), envp.as_ptr()) }; } else { @@ -195,9 +197,8 @@ fn close_fds(above: i32, keep: &[i32]) -> nix::Result<()> { use nix::{dir::Dir, fcntl::OFlag}; // TODO: close fds by brute force if readdir doesn't work: // https://github.com/python/cpython/blob/3.8/Modules/_posixsubprocess.c#L220 - let path = unsafe { CStr::from_bytes_with_nul_unchecked(FD_DIR_NAME) }; let mut dir = Dir::open( - path, + FD_DIR_NAME, OFlag::O_RDONLY | OFlag::O_DIRECTORY, nix::sys::stat::Mode::empty(), )?; @@ -219,10 +220,10 @@ fn close_fds(above: i32, keep: &[i32]) -> nix::Result<()> { target_os = "openbsd", target_vendor = "apple", ))] -const FD_DIR_NAME: &[u8] = b"/dev/fd\0"; +const FD_DIR_NAME: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"/dev/fd\0") }; #[cfg(any(target_os = "linux", target_os = "android"))] -const FD_DIR_NAME: &[u8] = b"/proc/self/fd\0"; +const FD_DIR_NAME: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"/proc/self/fd\0") }; #[cfg(not(target_os = "redox"))] fn pos_int_from_ascii(name: &CStr) -> Option { diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs index 8b9d4fb0c68..d513b93e2ff 100644 --- a/stdlib/src/socket.rs +++ b/stdlib/src/socket.rs @@ -891,53 +891,8 @@ mod _socket { use std::os::unix::ffi::OsStrExt; let buf = crate::vm::function::ArgStrOrBytesLike::try_from_object(vm, addr)?; let path = &*buf.borrow_bytes(); - if cfg!(any(target_os = "linux", target_os = "android")) - && path.first() == Some(&0) - { - use libc::{sa_family_t, socklen_t}; - use {socket2::SockAddr, std::ptr}; - unsafe { - // based on SockAddr::unix - // TODO: upstream or fix socklen check for SockAddr::unix()? - SockAddr::init(|storage, len| { - // Safety: `SockAddr::init` zeros the address, which is a valid - // representation. - let storage: &mut libc::sockaddr_un = &mut *storage.cast(); - let len: &mut socklen_t = &mut *len; - - let bytes = path; - if bytes.len() > storage.sun_path.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "path must be shorter than SUN_LEN", - )); - } - - storage.sun_family = libc::AF_UNIX as sa_family_t; - // Safety: `bytes` and `addr.sun_path` are not overlapping and - // both point to valid memory. - // `SockAddr::init` zeroes the memory, so the path is already - // null terminated. - ptr::copy_nonoverlapping( - bytes.as_ptr(), - storage.sun_path.as_mut_ptr() as *mut u8, - bytes.len(), - ); - - let base = storage as *const _ as usize; - let path = &storage.sun_path as *const _ as usize; - let sun_path_offset = path - base; - let length = sun_path_offset + bytes.len(); - *len = length as socklen_t; - - Ok(()) - }) - } - .map(|(_, addr)| addr) - } else { - socket2::SockAddr::unix(ffi::OsStr::from_bytes(path)) - } - .map_err(|_| vm.new_os_error("AF_UNIX path too long".to_owned()).into()) + socket2::SockAddr::unix(ffi::OsStr::from_bytes(path)) + .map_err(|_| vm.new_os_error("AF_UNIX path too long".to_owned()).into()) } c::AF_INET => { let tuple: PyTupleRef = addr.downcast().map_err(|obj| { @@ -1087,20 +1042,7 @@ mod _socket { _ => {} } if socket_kind == -1 { - // TODO: when socket2 cuts a new release, type will be available on all os - // socket_kind = sock.r#type().map_err(|e| e.into_pyexception(vm))?.into(); - let res = unsafe { - c::getsockopt( - sock_fileno(&sock) as _, - c::SOL_SOCKET, - c::SO_TYPE, - &mut socket_kind as *mut libc::c_int as *mut _, - &mut (std::mem::size_of::() as _), - ) - }; - if res < 0 { - return Err(crate::common::os::errno().into()); - } + socket_kind = sock.r#type().map_err(|e| e.into_pyexception(vm))?.into(); } cfg_if::cfg_if! { if #[cfg(any( @@ -1601,30 +1543,23 @@ mod _socket { if let Some(addr) = addr.as_socket() { return get_ip_addr_tuple(&addr, vm); } - match addr.family() as i32 { - #[cfg(unix)] - libc::AF_UNIX => { - let addr_len = addr.len() as usize; - let unix_addr = unsafe { &*(addr.as_ptr() as *const libc::sockaddr_un) }; - let path_u8 = unsafe { &*(&unix_addr.sun_path[..] as *const [_] as *const [u8]) }; - let sun_path_offset = - &unix_addr.sun_path as *const _ as usize - unix_addr as *const _ as usize; - if cfg!(any(target_os = "linux", target_os = "android")) - && addr_len > sun_path_offset - && unix_addr.sun_path[0] == 0 - { - let abstractaddrlen = addr_len - sun_path_offset; - let abstractpath = &path_u8[..abstractaddrlen]; - vm.ctx.new_bytes(abstractpath.to_vec()).into() - } else { - let len = memchr::memchr(b'\0', path_u8).unwrap_or(path_u8.len()); - let path = &path_u8[..len]; - vm.ctx.new_str(String::from_utf8_lossy(path)).into() - } + #[cfg(unix)] + use nix::sys::socket::{SockaddrLike, UnixAddr}; + #[cfg(unix)] + if let Some(unix_addr) = unsafe { UnixAddr::from_raw(addr.as_ptr(), Some(addr.len())) } { + use std::os::unix::ffi::OsStrExt; + #[cfg(any(target_os = "android", target_os = "linux"))] + if let Some(abstractpath) = unix_addr.as_abstract() { + return vm.ctx.new_bytes([b"\0", abstractpath].concat()).into(); } - // TODO: support more address families - _ => (String::new(), 0).to_pyobject(vm), + // necessary on macos + let path = ffi::OsStr::as_bytes(unix_addr.path().unwrap_or("".as_ref()).as_ref()); + let nul_pos = memchr::memchr(b'\0', path).unwrap_or(path.len()); + let path = ffi::OsStr::from_bytes(&path[..nul_pos]); + return vm.ctx.new_str(path.to_string_lossy()).into(); } + // TODO: support more address families + (String::new(), 0).to_pyobject(vm) } #[pyfunction] @@ -1764,25 +1699,19 @@ mod _socket { let fd = sock_fileno(sock); #[cfg(unix)] { - let mut pollfd = libc::pollfd { - fd, - events: match kind { - SelectKind::Read => libc::POLLIN, - SelectKind::Write => libc::POLLOUT, - SelectKind::Connect => libc::POLLOUT | libc::POLLERR, - }, - revents: 0, + use nix::poll::*; + let events = match kind { + SelectKind::Read => PollFlags::POLLIN, + SelectKind::Write => PollFlags::POLLOUT, + SelectKind::Connect => PollFlags::POLLOUT | PollFlags::POLLERR, }; + let mut pollfd = [PollFd::new(fd, events)]; let timeout = match interval { Some(d) => d.as_millis() as _, None => -1, }; - let ret = unsafe { libc::poll(&mut pollfd, 1, timeout) }; - if ret < 0 { - Err(io::Error::last_os_error()) - } else { - Ok(ret == 0) - } + let ret = poll(&mut pollfd, timeout)?; + Ok(ret == 0) } #[cfg(windows)] { diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs index ed3108ce1f9..a828e720121 100644 --- a/stdlib/src/sqlite.rs +++ b/stdlib/src/sqlite.rs @@ -493,10 +493,10 @@ mod _sqlite { unsafe extern "C" fn authorizer_callback( data: *mut c_void, action: c_int, - arg1: *const i8, - arg2: *const i8, - db_name: *const i8, - access: *const i8, + arg1: *const libc::c_char, + arg2: *const libc::c_char, + db_name: *const libc::c_char, + access: *const libc::c_char, ) -> c_int { let (callable, vm) = (*data.cast::()).retrive(); let f = || -> PyResult { @@ -557,13 +557,13 @@ mod _sqlite { context.result_exception( vm, exc, - &format!("user-defined aggregate's '{}' method not defined\0", name), + &format!("user-defined aggregate's '{name}' method not defined\0"), ) } else { context.result_exception( vm, exc, - &format!("user-defined aggregate's '{}' method raised error\0", name), + &format!("user-defined aggregate's '{name}' method raised error\0"), ) } } @@ -591,13 +591,13 @@ mod _sqlite { context.result_exception( vm, exc, - &format!("user-defined aggregate's '{}' method not defined\0", name), + &format!("user-defined aggregate's '{name}' method not defined\0"), ) } else { context.result_exception( vm, exc, - &format!("user-defined aggregate's '{}' method raised error\0", name), + &format!("user-defined aggregate's '{name}' method raised error\0"), ) } } @@ -1739,8 +1739,7 @@ mod _sqlite { } _ => { return Err(vm.new_not_implemented_error(format!( - "unknown column type: {}", - col_type + "unknown column type: {col_type}" ))); } } @@ -2305,7 +2304,7 @@ mod _sqlite { raise_exception(typ.to_owned(), extended_errcode, errmsg, vm) } - fn open(path: *const i8, uri: bool, vm: &VirtualMachine) -> PyResult { + fn open(path: *const libc::c_char, uri: bool, vm: &VirtualMachine) -> PyResult { let mut db = null_mut(); let ret = unsafe { sqlite3_open_v2( @@ -2329,8 +2328,8 @@ mod _sqlite { fn prepare( self, - sql: *const i8, - tail: *mut *const i8, + sql: *const libc::c_char, + tail: *mut *const libc::c_char, vm: &VirtualMachine, ) -> PyResult> { let mut st = null_mut(); @@ -2413,7 +2412,7 @@ mod _sqlite { #[allow(clippy::too_many_arguments)] fn create_function( self, - name: *const i8, + name: *const libc::c_char, narg: c_int, flags: c_int, data: *mut c_void, @@ -2632,7 +2631,7 @@ mod _sqlite { unsafe { sqlite3_column_text(self.st, pos) } } - fn column_decltype(self, pos: c_int) -> *const i8 { + fn column_decltype(self, pos: c_int) -> *const libc::c_char { unsafe { sqlite3_column_decltype(self.st, pos) } } @@ -2640,7 +2639,7 @@ mod _sqlite { unsafe { sqlite3_column_bytes(self.st, pos) } } - fn column_name(self, pos: c_int) -> *const i8 { + fn column_name(self, pos: c_int) -> *const libc::c_char { unsafe { sqlite3_column_name(self.st, pos) } } @@ -2804,7 +2803,7 @@ mod _sqlite { Ok(obj) } - fn ptr_to_str<'a>(p: *const i8, vm: &VirtualMachine) -> PyResult<&'a str> { + fn ptr_to_str<'a>(p: *const libc::c_char, vm: &VirtualMachine) -> PyResult<&'a str> { if p.is_null() { return Err(vm.new_memory_error("string pointer is null".to_owned())); } @@ -2841,7 +2840,7 @@ mod _sqlite { } } - fn str_to_ptr_len(s: &PyStr, vm: &VirtualMachine) -> PyResult<(*const i8, i32)> { + fn str_to_ptr_len(s: &PyStr, vm: &VirtualMachine) -> PyResult<(*const libc::c_char, i32)> { let len = c_int::try_from(s.byte_len()) .map_err(|_| vm.new_overflow_error("TEXT longer than INT_MAX bytes".to_owned()))?; let ptr = s.as_str().as_ptr().cast(); @@ -2910,7 +2909,7 @@ mod _sqlite { fn begin_statement_ptr_from_isolation_level( s: &PyStr, vm: &VirtualMachine, - ) -> PyResult<*const i8> { + ) -> PyResult<*const libc::c_char> { BEGIN_STATEMENTS .iter() .find(|&&x| x[6..].eq_ignore_ascii_case(s.as_str().as_bytes())) diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index f964c443be0..c7726203cd4 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -26,7 +26,9 @@ mod _ssl { use crate::{ common::{ ascii, - lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, + lock::{ + PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, + }, }, socket::{self, PySocket}, vm::{ @@ -504,12 +506,8 @@ mod _ssl { fn builder(&self) -> PyRwLockWriteGuard<'_, SslContextBuilder> { self.ctx.write() } - fn exec_ctx(&self, func: F) -> R - where - F: Fn(&ssl::SslContextRef) -> R, - { - let c = self.ctx.read(); - func(builder_as_ctx(&c)) + fn ctx(&self) -> PyMappedRwLockReadGuard<'_, ssl::SslContextRef> { + PyRwLockReadGuard::map(self.ctx.read(), builder_as_ctx) } #[pygetset] @@ -554,7 +552,7 @@ mod _ssl { } #[pygetset] fn verify_mode(&self) -> i32 { - let mode = self.exec_ctx(|ctx| ctx.verify_mode()); + let mode = self.ctx().verify_mode(); if mode == SslVerifyMode::NONE { CertRequirements::None.into() } else if mode == SslVerifyMode::PEER { @@ -703,16 +701,15 @@ mod _ssl { vm: &VirtualMachine, ) -> PyResult> { let binary_form = binary_form.unwrap_or(false); - self.exec_ctx(|ctx| { - let certs = ctx - .cert_store() - .objects() - .iter() - .filter_map(|obj| obj.x509()) - .map(|cert| cert_to_py(vm, cert, binary_form)) - .collect::, _>>()?; - Ok(certs) - }) + let certs = self + .ctx() + .cert_store() + .objects() + .iter() + .filter_map(|obj| obj.x509()) + .map(|cert| cert_to_py(vm, cert, binary_form)) + .collect::, _>>()?; + Ok(certs) } #[pymethod] @@ -746,9 +743,7 @@ mod _ssl { args: WrapSocketArgs, vm: &VirtualMachine, ) -> PyResult { - let mut ssl = zelf - .exec_ctx(ssl::Ssl::new) - .map_err(|e| convert_openssl_error(vm, e))?; + let mut ssl = ssl::Ssl::new(&zelf.ctx()).map_err(|e| convert_openssl_error(vm, e))?; let socket_type = if args.server_side { ssl.set_accept_state(); diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 437d2d77ca5..487b3eec899 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -32,45 +32,46 @@ rustpython-common = { path = "../common" } rustpython-derive = { path = "../derive", version = "0.2.0" } rustpython-jit = { path = "../jit", optional = true, version = "0.2.0" } -num-complex = { version = "0.4.0", features = ["serde"] } -num-bigint = { version = "0.4.3", features = ["serde"] } -num-traits = "0.2.14" -num-integer = "0.1.44" -num-rational = "0.4.0" -rand = "0.8.5" -getrandom = { version = "0.2.6", features = ["js"] } -log = "0.4.16" -serde = { version = "1.0.136", features = ["derive"] } +ascii = { workspace = true } +ahash = { workspace = true } +atty = { workspace = true } +bitflags = { workspace = true } +bstr = { workspace = true } +cfg-if = { workspace = true } +crossbeam-utils = { workspace = true } +chrono = { workspace = true, features = ["wasmbind"] } +flame = { workspace = true, optional = true } +hex = { workspace = true } +indexmap = { workspace = true } +itertools = { workspace = true } +libc = { workspace = true } +log = { workspace = true } +nix = { workspace = true } +num-bigint = { workspace = true, features = ["serde"] } +num-complex = { workspace = true, features = ["serde"] } +num-integer = { workspace = true } +num-rational = { workspace = true } +num-traits = { workspace = true } +num_enum = { workspace = true } +once_cell = { workspace = true } +parking_lot = { workspace = true } +paste = { workspace = true } +rand = { workspace = true } +serde = { workspace = true, features = ["derive"] } +static_assertions = { workspace = true } +thiserror = { workspace = true } +thread_local = { workspace = true } + caseless = "0.2.1" -chrono = { version = "0.4.19", features = ["wasmbind"] } -itertools = "0.10.3" -hex = "0.4.3" -hexf-parse = "0.2.1" -indexmap = "1.8.1" -ahash = "0.7.6" -bitflags = "1.3.2" -libc = "0.2.133" -nix = "0.24.2" -paste = "1.0.7" -is-macro = "0.2.0" -result-like = "0.4.5" -num_enum = "0.5.7" -bstr = "0.2.17" -crossbeam-utils = "0.8.9" -parking_lot = "0.12.0" -thread_local = "1.1.4" -cfg-if = "1.0.0" -timsort = "0.1.2" -thiserror = "1.0" -atty = "0.2.14" -static_assertions = "1.1.0" +getrandom = { version = "0.2.6", features = ["js"] } +flamer = { version = "0.4", optional = true } half = "1.8.2" +is-macro = "0.2.0" memchr = "2.4.1" -adler32 = "1.2.0" -flate2 = "1.0.23" -once_cell = "1.10.0" memoffset = "0.6.5" optional = "0.5.0" +result-like = "0.4.5" +timsort = "0.1.2" # RustPython crates implementing functionality based on CPython sre-engine = "0.4.1" @@ -89,11 +90,6 @@ unic-ucd-bidi = "0.9.0" unic-ucd-category = "0.9.0" unic-ucd-ident = "0.9.0" -flame = { version = "0.2", optional = true } -flamer = { version = "0.4", optional = true } - -ascii = "1.0.0" - [target.'cfg(unix)'.dependencies] exitcode = "1.1.2" uname = "0.1.1" @@ -101,16 +97,16 @@ strum = "0.24.0" strum_macros = "0.24.0" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -rustyline = "10.0.0" +rustyline = { workspace = true } which = "4.2.5" [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1.13.1" [target.'cfg(windows)'.dependencies] +schannel = { workspace = true } +widestring = { workspace = true } winreg = "0.10.1" -schannel = "0.1.19" -widestring = "0.5.1" [target.'cfg(windows)'.dependencies.windows] version = "0.39.0" @@ -130,6 +126,6 @@ features = [ wasm-bindgen = "0.2.80" [build-dependencies] -itertools = "0.10.3" +glob = { workspace = true } +itertools = { workspace = true } rustc_version = "0.4.0" -glob = "0.3" diff --git a/vm/build.rs b/vm/build.rs index c744c474408..b04482f8dee 100644 --- a/vm/build.rs +++ b/vm/build.rs @@ -5,7 +5,7 @@ fn main() { #[cfg(feature = "freeze-stdlib")] for entry in glob::glob("Lib/*/*.py").expect("Lib/ exists?").flatten() { let display = entry.display(); - println!("cargo:rerun-if-changed={}", display); + println!("cargo:rerun-if-changed={display}"); } println!("cargo:rustc-env=RUSTPYTHON_GIT_HASH={}", git_hash()); diff --git a/vm/src/builtins/bytearray.rs b/vm/src/builtins/bytearray.rs index c1f42b61db8..3ca0b18730d 100644 --- a/vm/src/builtins/bytearray.rs +++ b/vm/src/builtins/bytearray.rs @@ -442,7 +442,7 @@ impl PyByteArray { affix, "endswith", "bytes", - |s, x: &PyBytesInner| s.ends_with(&x.elements[..]), + |s, x: &PyBytesInner| s.ends_with(x.as_bytes()), vm, ) } @@ -463,7 +463,7 @@ impl PyByteArray { affix, "startswith", "bytes", - |s, x: &PyBytesInner| s.starts_with(&x.elements[..]), + |s, x: &PyBytesInner| s.starts_with(x.as_bytes()), vm, ) } @@ -507,13 +507,37 @@ impl PyByteArray { } #[pymethod] - fn lstrip(&self, chars: OptionalOption) -> Self { - self.inner().lstrip(chars).into() + fn lstrip( + zelf: PyRef, + chars: OptionalOption, + vm: &VirtualMachine, + ) -> PyRef { + let inner = zelf.inner(); + let stripped = inner.lstrip(chars); + let elements = &inner.elements; + if stripped == elements { + drop(inner); + zelf + } else { + vm.new_pyref(PyByteArray::from(stripped.to_vec())) + } } #[pymethod] - fn rstrip(&self, chars: OptionalOption) -> Self { - self.inner().rstrip(chars).into() + fn rstrip( + zelf: PyRef, + chars: OptionalOption, + vm: &VirtualMachine, + ) -> PyRef { + let inner = zelf.inner(); + let stripped = inner.rstrip(chars); + let elements = &inner.elements; + if stripped == elements { + drop(inner); + zelf + } else { + vm.new_pyref(PyByteArray::from(stripped.to_vec())) + } } /// removeprefix($self, prefix, /) diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index 12e968926c2..474d3fa236e 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -61,18 +61,18 @@ impl Deref for PyBytes { type Target = [u8]; fn deref(&self) -> &[u8] { - &self.inner.elements + self.as_bytes() } } impl AsRef<[u8]> for PyBytes { fn as_ref(&self) -> &[u8] { - &self.inner.elements + self.as_bytes() } } impl AsRef<[u8]> for PyBytesRef { fn as_ref(&self) -> &[u8] { - &self.inner.elements + self.as_bytes() } } @@ -133,7 +133,7 @@ impl PyBytes { #[inline] pub fn as_bytes(&self) -> &[u8] { - &self.inner.elements + self.inner.as_bytes() } #[pymethod(magic)] @@ -147,7 +147,7 @@ impl PyBytes { #[pymethod(magic)] fn sizeof(&self) -> usize { - size_of::() + self.inner.elements.len() * size_of::() + size_of::() + self.len() * size_of::() } #[pymethod(magic)] @@ -172,13 +172,9 @@ impl PyBytes { fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { match SequenceIndex::try_from_borrowed_object(vm, needle, "byte")? { SequenceIndex::Int(i) => self - .inner - .elements .getitem_by_index(vm, i) .map(|x| vm.ctx.new_int(x).into()), SequenceIndex::Slice(slice) => self - .inner - .elements .getitem_by_slice(vm, slice) .map(|x| vm.ctx.new_bytes(x).into()), } @@ -294,7 +290,7 @@ impl PyBytes { #[pymethod] fn endswith(&self, options: anystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult { let (affix, substr) = - match options.prepare(&self.inner.elements[..], self.len(), |s, r| s.get_bytes(r)) { + match options.prepare(self.as_bytes(), self.len(), |s, r| s.get_bytes(r)) { Some(x) => x, None => return Ok(false), }; @@ -302,7 +298,7 @@ impl PyBytes { affix, "endswith", "bytes", - |s, x: &PyBytesInner| s.ends_with(&x.elements[..]), + |s, x: &PyBytesInner| s.ends_with(x.as_bytes()), vm, ) } @@ -314,7 +310,7 @@ impl PyBytes { vm: &VirtualMachine, ) -> PyResult { let (affix, substr) = - match options.prepare(&self.inner.elements[..], self.len(), |s, r| s.get_bytes(r)) { + match options.prepare(self.as_bytes(), self.len(), |s, r| s.get_bytes(r)) { Some(x) => x, None => return Ok(false), }; @@ -322,7 +318,7 @@ impl PyBytes { affix, "startswith", "bytes", - |s, x: &PyBytesInner| s.starts_with(&x.elements[..]), + |s, x: &PyBytesInner| s.starts_with(x.as_bytes()), vm, ) } @@ -366,13 +362,31 @@ impl PyBytes { } #[pymethod] - fn lstrip(&self, chars: OptionalOption) -> Self { - self.inner.lstrip(chars).into() + fn lstrip( + zelf: PyRef, + chars: OptionalOption, + vm: &VirtualMachine, + ) -> PyRef { + let stripped = zelf.inner.lstrip(chars); + if stripped == zelf.as_bytes() { + zelf + } else { + vm.ctx.new_bytes(stripped.to_vec()) + } } #[pymethod] - fn rstrip(&self, chars: OptionalOption) -> Self { - self.inner.rstrip(chars).into() + fn rstrip( + zelf: PyRef, + chars: OptionalOption, + vm: &VirtualMachine, + ) -> PyRef { + let stripped = zelf.inner.rstrip(chars); + if stripped == zelf.as_bytes() { + zelf + } else { + vm.ctx.new_bytes(stripped.to_vec()) + } } /// removeprefix($self, prefix, /) @@ -521,12 +535,7 @@ impl PyBytes { #[pymethod(magic)] fn getnewargs(&self, vm: &VirtualMachine) -> PyTupleRef { - let param: Vec = self - .inner - .elements - .iter() - .map(|x| x.to_pyobject(vm)) - .collect(); + let param: Vec = self.elements().map(|x| x.to_pyobject(vm)).collect(); PyTuple::new_ref(param, &vm.ctx) } @@ -544,7 +553,7 @@ impl PyBytes { zelf: PyRef, vm: &VirtualMachine, ) -> (PyTypeRef, PyTupleRef, Option) { - let bytes = PyBytes::from(zelf.inner.elements.clone()).to_pyobject(vm); + let bytes = PyBytes::from(zelf.to_vec()).to_pyobject(vm); ( zelf.class().to_owned(), PyTuple::new_ref(vec![bytes], &vm.ctx), @@ -602,8 +611,7 @@ impl AsSequence for PyBytes { }), item: atomic_func!(|seq, i, vm| { PyBytes::sequence_downcast(seq) - .inner - .elements + .as_bytes() .getitem_by_index(vm, i) .map(|x| vm.ctx.new_bytes(vec![x]).into()) }), diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs index 24cb9700775..19d586bd0dc 100644 --- a/vm/src/builtins/code.rs +++ b/vm/src/builtins/code.rs @@ -218,6 +218,18 @@ impl PyRef { self.code.source_path.to_owned() } + #[pygetset] + pub fn co_cellvars(self, vm: &VirtualMachine) -> PyTupleRef { + let cellvars = self + .code + .cellvars + .deref() + .iter() + .map(|name| name.to_pyobject(vm)) + .collect(); + vm.ctx.new_tuple(cellvars) + } + #[pygetset] fn co_firstlineno(self) -> usize { self.code.first_line_number @@ -262,6 +274,18 @@ impl PyRef { vm.ctx.new_tuple(varnames) } + #[pygetset] + pub fn co_freevars(self, vm: &VirtualMachine) -> PyTupleRef { + let names = self + .code + .freevars + .deref() + .iter() + .map(|name| name.to_pyobject(vm)) + .collect(); + vm.ctx.new_tuple(names) + } + #[pymethod] pub fn replace(self, args: ReplaceArgs, vm: &VirtualMachine) -> PyResult { let posonlyarg_count = match args.co_posonlyargcount { diff --git a/vm/src/builtins/complex.rs b/vm/src/builtins/complex.rs index df81122fd24..3f86b48d196 100644 --- a/vm/src/builtins/complex.rs +++ b/vm/src/builtins/complex.rs @@ -128,7 +128,7 @@ impl Constructor for PyComplex { let val = if cls.is(vm.ctx.types.complex_type) && imag_missing { match val.downcast_exact::(vm) { Ok(c) => { - return Ok(c.into()); + return Ok(c.into_pyref().into()); } Err(val) => val, } diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs index ba747bbf250..7fd8cac5d1f 100644 --- a/vm/src/builtins/dict.rs +++ b/vm/src/builtins/dict.rs @@ -1,6 +1,6 @@ use super::{ set::PySetInner, IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, - PyMappingProxy, PySet, PyStrRef, PyTupleRef, PyType, PyTypeRef, + PyMappingProxy, PySet, PyStr, PyTupleRef, PyType, PyTypeRef, }; use crate::{ atomic_func, @@ -11,7 +11,6 @@ use crate::{ }, class::{PyClassDef, PyClassImpl}, common::ascii, - convert::ToPyObject, dictdatatype::{self, DictKey}, function::{ ArgIterable, FuncArgs, KwArgs, OptionalArg, PyArithmeticValue::*, PyComparisonValue, @@ -24,7 +23,8 @@ use crate::{ IterNextIterable, Iterable, PyComparisonOp, Unconstructible, Unhashable, }, vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromObject, }; use once_cell::sync::Lazy; use rustpython_common::lock::PyMutex; @@ -65,8 +65,9 @@ impl PyDict { // Used in update and ior. pub(crate) fn merge_object(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let other = match other.downcast_exact(vm) { - Ok(dict_other) => return self.merge_dict(dict_other, vm), + let casted: Result, _> = other.downcast_exact(vm); + let other = match casted { + Ok(dict_other) => return self.merge_dict(dict_other.into_pyref(), vm), Err(other) => other, }; let dict = &self.entries; @@ -227,7 +228,7 @@ impl PyDict { for key in iterable.iter(vm)? { pydict.setitem(key?, value.clone(), vm)?; } - Ok(pydict.to_pyobject(vm)) + Ok(pydict.into_pyref().into()) } Err(pyobj) => { for key in iterable.iter(vm)? { @@ -533,9 +534,8 @@ impl Py { pub fn to_attributes(&self, vm: &VirtualMachine) -> PyAttributes { let mut attrs = PyAttributes::default(); for (key, value) in self { - // TODO: use PyRefExact for interning - let key: PyStrRef = key.downcast().expect("dict has non-string keys"); - attrs.insert(vm.ctx.intern_str(key.as_str()), value); + let key: PyRefExact = key.downcast_exact(vm).expect("dict has non-string keys"); + attrs.insert(vm.ctx.intern_str(key), value); } attrs } diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs index 3ad2ec6f5cc..8808eae9876 100644 --- a/vm/src/builtins/function.rs +++ b/vm/src/builtins/function.rs @@ -101,7 +101,9 @@ impl PyFunction { if nargs > nexpected_args { return Err(vm.new_type_error(format!( "{}() takes {} positional arguments but {} were given", - &self.code.obj_name, nexpected_args, nargs + self.qualname(), + nexpected_args, + nargs ))); } } @@ -132,9 +134,11 @@ impl PyFunction { if let Some(pos) = argpos(code.posonlyarg_count..total_args, &name) { let slot = &mut fastlocals[pos]; if slot.is_some() { - return Err( - vm.new_type_error(format!("Got multiple values for argument '{name}'")) - ); + return Err(vm.new_type_error(format!( + "{}() got multiple values for argument '{}'", + self.qualname(), + name + ))); } *slot = Some(value); } else if let Some(kwargs) = kwargs.as_ref() { @@ -142,15 +146,17 @@ impl PyFunction { } else if argpos(0..code.posonlyarg_count, &name).is_some() { posonly_passed_as_kwarg.push(name); } else { - return Err( - vm.new_type_error(format!("got an unexpected keyword argument '{name}'")) - ); + return Err(vm.new_type_error(format!( + "{}() got an unexpected keyword argument '{}'", + self.qualname(), + name + ))); } } if !posonly_passed_as_kwarg.is_empty() { return Err(vm.new_type_error(format!( "{}() got some positional-only arguments passed as keyword arguments: '{}'", - &self.code.obj_name, + self.qualname(), posonly_passed_as_kwarg.into_iter().format(", "), ))); } @@ -207,7 +213,7 @@ impl PyFunction { return Err(vm.new_type_error(format!( "{}() missing {} required positional argument{}: '{}{}{}'", - &self.code.obj_name, + self.qualname(), missing_args_len, if missing_args_len == 1 { "" } else { "s" }, missing.iter().join("', '"), diff --git a/vm/src/builtins/function/jitfunc.rs b/vm/src/builtins/function/jitfunc.rs index 706eb95a7bd..4c4bf09765b 100644 --- a/vm/src/builtins/function/jitfunc.rs +++ b/vm/src/builtins/function/jitfunc.rs @@ -58,7 +58,7 @@ fn get_jit_arg_type(dict: &PyDictRef, name: &str, vm: &VirtualMachine) -> PyResu } } else { Err(new_jit_error( - format!("argument {} needs annotation", name), + format!("argument {name} needs annotation"), vm, )) } diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs index d7fd2f8de08..07c53a8b1b9 100644 --- a/vm/src/builtins/int.rs +++ b/vm/src/builtins/int.rs @@ -231,7 +231,7 @@ impl Constructor for PyInt { let val = if cls.is(vm.ctx.types.int_type) { match val.downcast_exact::(vm) { Ok(i) => { - return Ok(i.to_pyobject(vm)); + return Ok(i.into_pyref().into()); } Err(val) => val, } @@ -606,11 +606,11 @@ impl PyInt { ) -> PyResult> { let signed = args.signed.map_or(false, Into::into); let value = match (args.byteorder, signed) { - (ArgByteOrder::Big, true) => BigInt::from_signed_bytes_be(&args.bytes.elements), - (ArgByteOrder::Big, false) => BigInt::from_bytes_be(Sign::Plus, &args.bytes.elements), - (ArgByteOrder::Little, true) => BigInt::from_signed_bytes_le(&args.bytes.elements), + (ArgByteOrder::Big, true) => BigInt::from_signed_bytes_be(args.bytes.as_bytes()), + (ArgByteOrder::Big, false) => BigInt::from_bytes_be(Sign::Plus, args.bytes.as_bytes()), + (ArgByteOrder::Little, true) => BigInt::from_signed_bytes_le(args.bytes.as_bytes()), (ArgByteOrder::Little, false) => { - BigInt::from_bytes_le(Sign::Plus, &args.bytes.elements) + BigInt::from_bytes_le(Sign::Plus, args.bytes.as_bytes()) } }; Self::with_value(cls, value, vm) diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index e7cf19d40f2..039a48d4feb 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -486,7 +486,7 @@ impl PyMemoryView { } } if !is_adjusted { - // no suboffset setted, stride must be positive + // no suboffset set, stride must be positive self.start += stride as usize * range.start; } let newlen = range.len(); @@ -507,7 +507,7 @@ impl PyMemoryView { } } if !is_adjusted_suboffset { - // no suboffset setted, stride must be positive + // no suboffset set, stride must be positive self.start += stride as usize * if step.is_negative() { range.end - 1 @@ -521,7 +521,7 @@ impl PyMemoryView { Ok(()) } - /// return the length of the first dimention + /// return the length of the first dimension #[pymethod(magic)] fn len(&self, vm: &VirtualMachine) -> PyResult { self.try_not_released(vm)?; diff --git a/vm/src/builtins/range.rs b/vm/src/builtins/range.rs index 70426dd54bd..fcc3a8fa5c2 100644 --- a/vm/src/builtins/range.rs +++ b/vm/src/builtins/range.rs @@ -295,12 +295,12 @@ impl PyRange { #[pymethod(magic)] fn reduce(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef) { - let range_paramters: Vec = vec![&self.start, &self.stop, &self.step] + let range_parameters: Vec = vec![&self.start, &self.stop, &self.step] .iter() .map(|x| x.as_object().to_owned()) .collect(); - let range_paramters_tuple = vm.ctx.new_tuple(range_paramters); - (vm.ctx.types.range_type.to_owned(), range_paramters_tuple) + let range_parameters_tuple = vm.ctx.new_tuple(range_parameters); + (vm.ctx.types.range_type.to_owned(), range_parameters_tuple) } #[pymethod] diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs index c2ba6781f7d..5b5db309050 100644 --- a/vm/src/builtins/set.rs +++ b/vm/src/builtins/set.rs @@ -354,7 +354,7 @@ impl PySetInner { self.merge_set(any_set, vm) // check Dict } else if let Ok(dict) = iterable.to_owned().downcast_exact::(vm) { - self.merge_dict(dict, vm) + self.merge_dict(dict.into_pyref(), vm) } else { // add iterable that is not AnySet or Dict for item in iterable.try_into_value::(vm)?.iter(vm)? { @@ -799,7 +799,7 @@ impl Constructor for PyFrozenSet { let elements = if let OptionalArg::Present(iterable) = iterable { let iterable = if cls.is(vm.ctx.types.frozenset_type) { match iterable.downcast_exact::(vm) { - Ok(fs) => return Ok(fs.into()), + Ok(fs) => return Ok(fs.into_pyref().into()), Err(iterable) => iterable, } } else { diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index bf0d17276ab..956c2fe901c 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -592,25 +592,41 @@ impl PyStr { } #[pymethod] - fn lstrip(&self, chars: OptionalOption) -> String { - self.as_str() - .py_strip( - chars, - |s, chars| s.trim_start_matches(|c| chars.contains(c)), - |s| s.trim_start(), - ) - .to_owned() + fn lstrip( + zelf: PyRef, + chars: OptionalOption, + vm: &VirtualMachine, + ) -> PyRef { + let s = zelf.as_str(); + let stripped = s.py_strip( + chars, + |s, chars| s.trim_start_matches(|c| chars.contains(c)), + |s| s.trim_start(), + ); + if s == stripped { + zelf + } else { + stripped.into_pystr_ref(vm) + } } #[pymethod] - fn rstrip(&self, chars: OptionalOption) -> String { - self.as_str() - .py_strip( - chars, - |s, chars| s.trim_end_matches(|c| chars.contains(c)), - |s| s.trim_end(), - ) - .to_owned() + fn rstrip( + zelf: PyRef, + chars: OptionalOption, + vm: &VirtualMachine, + ) -> PyRef { + let s = zelf.as_str(); + let stripped = s.py_strip( + chars, + |s, chars| s.trim_end_matches(|c| chars.contains(c)), + |s| s.trim_end(), + ); + if s == stripped { + zelf + } else { + stripped.into_pystr_ref(vm) + } } #[pymethod] @@ -860,13 +876,24 @@ impl PyStr { } #[pymethod] - fn join(&self, iterable: ArgIterable, vm: &VirtualMachine) -> PyResult { + fn join( + zelf: PyRef, + iterable: ArgIterable, + vm: &VirtualMachine, + ) -> PyResult { let iter = iterable.iter(vm)?; - - match iter.exactly_one() { - Ok(first) => first, - Err(iter) => Ok(vm.ctx.new_str(self.as_str().py_join(iter)?)), - } + let joined = match iter.exactly_one() { + Ok(first) => { + let first = first?; + if first.as_object().class().is(vm.ctx.types.str_type) { + return Ok(first); + } else { + first.as_str().to_owned() + } + } + Err(iter) => zelf.as_str().py_join(iter)?, + }; + Ok(joined.into_pystr_ref(vm)) } // FIXME: two traversals of str is expensive diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index b5472f6998a..750527384f2 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -95,7 +95,7 @@ impl Constructor for PyTuple { let elements = if let OptionalArg::Present(iterable) = iterable { let iterable = if cls.is(vm.ctx.types.tuple_type) { match iterable.downcast_exact::(vm) { - Ok(tuple) => return Ok(tuple.into()), + Ok(tuple) => return Ok(tuple.into_pyref().into()), Err(iterable) => iterable, } } else { diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index 8bd4a4fe290..5689d395a20 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -20,7 +20,7 @@ use rustpython_common::hash; #[derive(Debug, Default, Clone)] pub struct PyBytesInner { - pub(crate) elements: Vec, + pub(super) elements: Vec, } impl From> for PyBytesInner { @@ -79,9 +79,7 @@ impl ByteInnerNewOptions { // construct an exact bytes from an exact bytes do not clone let obj = if cls.is(PyBytes::class(vm)) { match obj.downcast_exact::(vm) { - Ok(b) => { - return Ok(b); - } + Ok(b) => return Ok(b.into_pyref()), Err(obj) => obj, } } else { @@ -243,6 +241,11 @@ impl ByteInnerTranslateOptions { pub type ByteInnerSplitOptions<'a> = anystr::SplitArgs<'a, PyBytesInner>; impl PyBytesInner { + #[inline] + pub fn as_bytes(&self) -> &[u8] { + &self.elements + } + pub fn repr(&self, class_name: Option<&str>) -> String { if let Some(class_name) = class_name { rustpython_common::bytes::repr_with(&self.elements, &[class_name, "("], ")") @@ -578,24 +581,20 @@ impl PyBytesInner { .to_vec() } - pub fn lstrip(&self, chars: OptionalOption) -> Vec { - self.elements - .py_strip( - chars, - |s, chars| s.trim_start_with(|c| chars.contains(&(c as u8))), - |s| s.trim_start(), - ) - .to_vec() + pub fn lstrip(&self, chars: OptionalOption) -> &[u8] { + self.elements.py_strip( + chars, + |s, chars| s.trim_start_with(|c| chars.contains(&(c as u8))), + |s| s.trim_start(), + ) } - pub fn rstrip(&self, chars: OptionalOption) -> Vec { - self.elements - .py_strip( - chars, - |s, chars| s.trim_end_with(|c| chars.contains(&(c as u8))), - |s| s.trim_end(), - ) - .to_vec() + pub fn rstrip(&self, chars: OptionalOption) -> &[u8] { + self.elements.py_strip( + chars, + |s, chars| s.trim_end_with(|c| chars.contains(&(c as u8))), + |s| s.trim_end(), + ) } // new in Python 3.9 diff --git a/vm/src/format.rs b/vm/src/format.rs index 8b51fa0aed2..d13dbd9384e 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -18,18 +18,18 @@ impl IntoPyException for FormatSpecError { vm.new_value_error("Invalid format specifier".to_owned()) } FormatSpecError::UnspecifiedFormat(c1, c2) => { - let msg = format!("Cannot specify '{}' with '{}'.", c1, c2); + let msg = format!("Cannot specify '{c1}' with '{c2}'."); vm.new_value_error(msg) } FormatSpecError::UnknownFormatCode(c, s) => { - let msg = format!("Unknown format code '{}' for object of type '{}'", c, s); + let msg = format!("Unknown format code '{c}' for object of type '{s}'"); vm.new_value_error(msg) } FormatSpecError::PrecisionNotAllowed => { vm.new_value_error("Precision not allowed in integer format specifier".to_owned()) } FormatSpecError::NotAllowed(s) => { - let msg = format!("{} not allowed with integer format specifier 'c'", s); + let msg = format!("{s} not allowed with integer format specifier 'c'"); vm.new_value_error(msg) } FormatSpecError::UnableToConvert => { @@ -39,10 +39,7 @@ impl IntoPyException for FormatSpecError { vm.new_overflow_error("%c arg not in range(0x110000)".to_owned()) } FormatSpecError::NotImplemented(c, s) => { - let msg = format!( - "Format code '{}' for object of type '{}' not implemented yet", - c, s - ); + let msg = format!("Format code '{c}' for object of type '{s}' not implemented yet"); vm.new_value_error(msg) } } diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index 19055df25c7..c8cdd792923 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -12,7 +12,7 @@ //! but not to do to remember it is a PyRef object. use super::{ - ext::{AsObject, PyResult}, + ext::{AsObject, PyRefExact, PyResult}, payload::PyObjectPayload, PyAtomicRef, }; @@ -564,7 +564,7 @@ impl PyObjectRef { pub fn downcast_exact( self, vm: &VirtualMachine, - ) -> Result, Self> { + ) -> Result, Self> { if self.class().is(T::class(vm)) { // TODO: is this always true? assert!( @@ -572,7 +572,7 @@ impl PyObjectRef { "obj.__class__ is T::class() but payload is not T" ); // SAFETY: just asserted that payload_is::() - Ok(unsafe { PyRef::from_obj_unchecked(self) }) + Ok(unsafe { PyRefExact::new_unchecked(PyRef::from_obj_unchecked(self)) }) } else { Err(self) } @@ -1077,7 +1077,7 @@ impl PyWeakRef { } } -/// Paritally initialize a struct, ensuring that all fields are +/// Partially initialize a struct, ensuring that all fields are /// either given values or explicitly left uninitialized macro_rules! partially_init { ( diff --git a/vm/src/object/ext.rs b/vm/src/object/ext.rs index 5bd0779ee2c..4e74a4b58be 100644 --- a/vm/src/object/ext.rs +++ b/vm/src/object/ext.rs @@ -107,6 +107,7 @@ impl std::borrow::ToOwned for PyExact { } } +/// PyRef but guaranteed not to be a subtype instance #[derive(Debug)] #[repr(transparent)] pub struct PyRefExact { diff --git a/vm/src/sliceable.rs b/vm/src/sliceable.rs index 8f1288cc49e..7a4a3d0b5a1 100644 --- a/vm/src/sliceable.rs +++ b/vm/src/sliceable.rs @@ -308,14 +308,15 @@ impl SequenceIndexOp for isize { } fn wrapped_at(&self, len: usize) -> Option { - let neg = self.is_negative(); - let p = self.unsigned_abs(); - if neg { - len.checked_sub(p) - } else if p >= len { + let mut p = *self; + if p < 0 { + // casting to isize is ok because it is used by wrapping_add + p = p.wrapping_add(len as isize); + } + if p < 0 || (p as usize) >= len { None } else { - Some(p) + Some(p as usize) } } } diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 446a197a7f2..f9832c9f71c 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -872,7 +872,11 @@ mod builtins { // Use downcast_exact to keep ref to old object on error. let metaclass = kwargs .pop_kwarg("metaclass") - .map(|metaclass| metaclass.downcast_exact::(vm)) + .map(|metaclass| { + metaclass + .downcast_exact::(vm) + .map(|m| m.into_pyref()) + }) .unwrap_or_else(|| Ok(vm.ctx.types.type_type.to_owned())); let (metaclass, meta_name) = match metaclass { @@ -996,7 +1000,7 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "NotImplemented" => ctx.not_implemented(), "Ellipsis" => vm.ctx.ellipsis.clone(), - // ordered by exception_hierarachy.txt + // ordered by exception_hierarchy.txt // Exceptions: "BaseException" => ctx.exceptions.base_exception_type.to_owned(), "SystemExit" => ctx.exceptions.system_exit.to_owned(), diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index a5bff75ac40..c7e5b4f3ebb 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1552,7 +1552,7 @@ mod _io { } let flush_res = data.flush(vm); let close_res = vm.call_method(data.raw.as_ref().unwrap(), "close", ()); - exeption_chain(flush_res, close_res) + exception_chain(flush_res, close_res) } #[pymethod] @@ -1568,7 +1568,7 @@ mod _io { let data = zelf.lock(vm)?; let raw = data.raw.as_ref().unwrap(); let close_res = vm.call_method(raw, "close", ()); - exeption_chain(flush_res, close_res) + exception_chain(flush_res, close_res) } #[pymethod] @@ -1657,7 +1657,7 @@ mod _io { } } - fn exeption_chain(e1: PyResult<()>, e2: PyResult) -> PyResult { + fn exception_chain(e1: PyResult<()>, e2: PyResult) -> PyResult { match (e1, e2) { (Err(e1), Err(e)) => { e.set_context(Some(e1)); @@ -1858,7 +1858,7 @@ mod _io { fn close(&self, vm: &VirtualMachine) -> PyResult { let write_res = self.write.close_strict(vm).map(drop); let read_res = self.read.close_strict(vm); - exeption_chain(write_res, read_res) + exception_chain(write_res, read_res) } } @@ -2897,7 +2897,7 @@ mod _io { } let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop); let close_res = vm.call_method(&buffer, "close", ()).map(drop); - exeption_chain(flush_res, close_res) + exception_chain(flush_res, close_res) } #[pygetset] fn closed(&self, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 392bf05c8ea..7804a4a2aff 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1673,7 +1673,7 @@ pub(super) mod _os { Err(vm.new_value_error(format!("Invalid wait status: {status}"))) } else { i32::try_from(status.rotate_right(8)) - .map_err(|_| vm.new_value_error(format!("invalid wait status: {}", status))) + .map_err(|_| vm.new_value_error(format!("invalid wait status: {status}"))) } } } @@ -1694,7 +1694,7 @@ pub(super) mod _os { _ => 0, }; - Ok(Some(format!("cp{}", cp))) + Ok(Some(format!("cp{cp}"))) } else { let encoding = unsafe { let encoding = libc::nl_langinfo(libc::CODESET); diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index b4df5eaa4ba..7f14b2db562 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -571,10 +571,7 @@ pub mod module { let priority_type = priority_class.name(); let priority = self.sched_priority.clone(); let value = priority.downcast::().map_err(|_| { - vm.new_type_error(format!( - "an integer is required (got type {})", - priority_type - )) + vm.new_type_error(format!("an integer is required (got type {priority_type})")) })?; let sched_priority = value.try_to_primitive(vm)?; Ok(libc::sched_param { sched_priority }) @@ -1094,13 +1091,9 @@ pub mod module { #[pyfunction] fn ttyname(fd: i32, vm: &VirtualMachine) -> PyResult { - let name = unsafe { libc::ttyname(fd) }; - if name.is_null() { - Err(errno_err(vm)) - } else { - let name = unsafe { CStr::from_ptr(name) }.to_str().unwrap(); - Ok(vm.ctx.new_str(name).into()) - } + let name = unistd::ttyname(fd).map_err(|e| e.into_pyexception(vm))?; + let name = name.into_os_string().into_string().unwrap(); + Ok(vm.ctx.new_str(name).into()) } #[pyfunction] @@ -1132,30 +1125,24 @@ pub mod module { #[cfg(any(target_os = "android", target_os = "linux", target_os = "openbsd"))] #[pyfunction] fn getresuid(vm: &VirtualMachine) -> PyResult<(u32, u32, u32)> { - let mut ruid = 0; - let mut euid = 0; - let mut suid = 0; - let ret = unsafe { libc::getresuid(&mut ruid, &mut euid, &mut suid) }; - if ret == 0 { - Ok((ruid, euid, suid)) - } else { - Err(errno_err(vm)) - } + let ret = unistd::getresuid().map_err(|e| e.into_pyexception(vm))?; + Ok(( + ret.real.as_raw(), + ret.effective.as_raw(), + ret.saved.as_raw(), + )) } // cfg from nix #[cfg(any(target_os = "android", target_os = "linux", target_os = "openbsd"))] #[pyfunction] fn getresgid(vm: &VirtualMachine) -> PyResult<(u32, u32, u32)> { - let mut rgid = 0; - let mut egid = 0; - let mut sgid = 0; - let ret = unsafe { libc::getresgid(&mut rgid, &mut egid, &mut sgid) }; - if ret == 0 { - Ok((rgid, egid, sgid)) - } else { - Err(errno_err(vm)) - } + let ret = unistd::getresgid().map_err(|e| e.into_pyexception(vm))?; + Ok(( + ret.real.as_raw(), + ret.effective.as_raw(), + ret.saved.as_raw(), + )) } // cfg from nix @@ -1991,7 +1978,7 @@ pub mod module { #[pyfunction] fn getrandom(size: isize, flags: OptionalArg, vm: &VirtualMachine) -> PyResult> { let size = usize::try_from(size) - .map_err(|_| vm.new_os_error(format!("Invalid argument for size: {}", size)))?; + .map_err(|_| vm.new_os_error(format!("Invalid argument for size: {size}")))?; let mut buf = Vec::with_capacity(size); unsafe { let len = sys_getrandom( diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index c4341082843..cb9b205edf5 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -103,7 +103,9 @@ pub(crate) mod _signal { .clone() .get_attr("default_int_handler", vm) .expect("_signal does not have this attr?"); - signal(libc::SIGINT, int_handler, vm).expect("Failed to set sigint handler"); + if !vm.state.settings.no_sig_int { + signal(libc::SIGINT, int_handler, vm).expect("Failed to set sigint handler"); + } } #[pyfunction] diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index a9ab5417414..474d51d75dd 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -207,8 +207,18 @@ mod time { #[pyfunction] fn strftime(format: PyStrRef, t: OptionalArg, vm: &VirtualMachine) -> PyResult { + use std::fmt::Write; + let instant = t.naive_or_local(vm)?; - let formatted_time = instant.format(format.as_str()).to_string(); + let mut formatted_time = String::new(); + + /* + * chrono doesn't support all formats and it + * raises an error if unsupported format is supplied. + * If error happens, we set result as input arg. + */ + write!(&mut formatted_time, "{}", instant.format(format.as_str())) + .unwrap_or_else(|_| formatted_time = format.to_string()); Ok(vm.ctx.new_str(formatted_time).into()) } @@ -389,11 +399,11 @@ mod unix { use super::{SEC_TO_NS, US_TO_NS}; #[cfg_attr(target_os = "macos", allow(unused_imports))] use crate::{ - builtins::{try_bigint_to_f64, PyFloat, PyIntRef, PyNamespace, PyStrRef}, - function::Either, - stdlib::os, - PyRef, PyResult, VirtualMachine, + builtins::{PyNamespace, PyStrRef}, + convert::IntoPyException, + PyObject, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, }; + use nix::{sys::time::TimeSpec, time::ClockId}; use std::time::Duration; #[cfg(target_os = "solaris")] @@ -425,83 +435,62 @@ mod unix { #[pyattr] use libc::{CLOCK_PROF, CLOCK_UPTIME}; - fn get_clock_time(clk_id: PyIntRef, vm: &VirtualMachine) -> PyResult { - let mut timespec = std::mem::MaybeUninit::uninit(); - let ts: libc::timespec = unsafe { - if libc::clock_gettime(clk_id.try_to_primitive(vm)?, timespec.as_mut_ptr()) == -1 { - return Err(os::errno_err(vm)); - } - timespec.assume_init() - }; - Ok(Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32)) + impl TryFromBorrowedObject for ClockId { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult { + obj.try_to_value(vm).map(ClockId::from_raw) + } + } + + fn get_clock_time(clk_id: ClockId, vm: &VirtualMachine) -> PyResult { + let ts = nix::time::clock_gettime(clk_id).map_err(|e| e.into_pyexception(vm))?; + Ok(ts.into()) } #[pyfunction] - fn clock_gettime(clk_id: PyIntRef, vm: &VirtualMachine) -> PyResult { + fn clock_gettime(clk_id: ClockId, vm: &VirtualMachine) -> PyResult { get_clock_time(clk_id, vm).map(|d| d.as_secs_f64()) } #[pyfunction] - fn clock_gettime_ns(clk_id: PyIntRef, vm: &VirtualMachine) -> PyResult { + fn clock_gettime_ns(clk_id: ClockId, vm: &VirtualMachine) -> PyResult { get_clock_time(clk_id, vm).map(|d| d.as_nanos()) } #[cfg(not(target_os = "redox"))] #[pyfunction] - fn clock_getres(clk_id: PyIntRef, vm: &VirtualMachine) -> PyResult { - let mut timespec = std::mem::MaybeUninit::uninit(); - let ts: libc::timespec = unsafe { - if libc::clock_getres(clk_id.try_to_primitive(vm)?, timespec.as_mut_ptr()) == -1 { - return Err(os::errno_err(vm)); - } - timespec.assume_init() - }; - Ok(Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32).as_secs_f64()) + fn clock_getres(clk_id: ClockId, vm: &VirtualMachine) -> PyResult { + let ts = nix::time::clock_getres(clk_id).map_err(|e| e.into_pyexception(vm))?; + Ok(Duration::from(ts).as_secs_f64()) } #[cfg(not(target_os = "redox"))] - #[cfg(any(not(target_vendor = "apple"), target_os = "macos"))] - fn set_clock_time( - clk_id: PyIntRef, - timespec: libc::timespec, - vm: &VirtualMachine, - ) -> PyResult<()> { - let res = unsafe { libc::clock_settime(clk_id.try_to_primitive(vm)?, ×pec) }; - if res == -1 { - return Err(os::errno_err(vm)); - } - Ok(()) + #[cfg(not(target_vendor = "apple"))] + fn set_clock_time(clk_id: ClockId, timespec: TimeSpec, vm: &VirtualMachine) -> PyResult<()> { + nix::time::clock_settime(clk_id, timespec).map_err(|e| e.into_pyexception(vm)) + } + + #[cfg(not(target_os = "redox"))] + #[cfg(target_os = "macos")] + fn set_clock_time(clk_id: ClockId, timespec: TimeSpec, vm: &VirtualMachine) -> PyResult<()> { + // idk why nix disables clock_settime on macos + let ret = unsafe { libc::clock_settime(clk_id.as_raw(), timespec.as_ref()) }; + nix::Error::result(ret) + .map(drop) + .map_err(|e| e.into_pyexception(vm)) } #[cfg(not(target_os = "redox"))] #[cfg(any(not(target_vendor = "apple"), target_os = "macos"))] #[pyfunction] - fn clock_settime( - clk_id: PyIntRef, - time: Either, PyIntRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let time = match time { - Either::A(f) => f.to_f64(), - Either::B(z) => try_bigint_to_f64(z.as_bigint(), vm)?, - }; - let nanos = time.fract() * (SEC_TO_NS as f64); - let ts = libc::timespec { - tv_sec: time.floor() as libc::time_t, - tv_nsec: nanos as _, - }; - set_clock_time(clk_id, ts, vm) + fn clock_settime(clk_id: ClockId, time: Duration, vm: &VirtualMachine) -> PyResult<()> { + set_clock_time(clk_id, time.into(), vm) } #[cfg(not(target_os = "redox"))] #[cfg(any(not(target_vendor = "apple"), target_os = "macos"))] #[pyfunction] - fn clock_settime_ns(clk_id: PyIntRef, time: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { - let time: libc::time_t = time.try_to_primitive(vm)?; - let ts = libc::timespec { - tv_sec: time / (SEC_TO_NS as libc::time_t), - tv_nsec: time.rem_euclid(SEC_TO_NS as libc::time_t) as _, - }; + fn clock_settime_ns(clk_id: ClockId, time: libc::time_t, vm: &VirtualMachine) -> PyResult<()> { + let ts = Duration::from_nanos(time as _).into(); set_clock_time(clk_id, ts, vm) } @@ -522,25 +511,25 @@ mod unix { false, "time.clock_gettime(CLOCK_MONOTONIC)", true, - clock_getres(vm.ctx.new_int(CLOCK_MONOTONIC), vm)?, + clock_getres(ClockId::CLOCK_MONOTONIC, vm)?, ), "process_time" => ( false, "time.clock_gettime(CLOCK_PROCESS_CPUTIME_ID)", true, - clock_getres(vm.ctx.new_int(CLOCK_PROCESS_CPUTIME_ID), vm)?, + clock_getres(ClockId::CLOCK_PROCESS_CPUTIME_ID, vm)?, ), "thread_time" => ( false, "time.clock_gettime(CLOCK_THREAD_CPUTIME_ID)", true, - clock_getres(vm.ctx.new_int(CLOCK_THREAD_CPUTIME_ID), vm)?, + clock_getres(ClockId::CLOCK_THREAD_CPUTIME_ID, vm)?, ), "time" => ( true, "time.clock_gettime(CLOCK_REALTIME)", false, - clock_getres(vm.ctx.new_int(CLOCK_REALTIME), vm)?, + clock_getres(ClockId::CLOCK_REALTIME, vm)?, ), _ => return Err(vm.new_value_error("unknown clock".to_owned())), }; @@ -568,22 +557,19 @@ mod unix { } pub(super) fn get_monotonic_time(vm: &VirtualMachine) -> PyResult { - get_clock_time(vm.ctx.new_int(CLOCK_MONOTONIC), vm) + get_clock_time(ClockId::CLOCK_MONOTONIC, vm) } pub(super) fn get_perf_time(vm: &VirtualMachine) -> PyResult { - get_clock_time(vm.ctx.new_int(CLOCK_MONOTONIC), vm) + get_clock_time(ClockId::CLOCK_MONOTONIC, vm) } #[pyfunction] fn sleep(dur: Duration, vm: &VirtualMachine) -> PyResult<()> { // this is basically std::thread::sleep, but that catches interrupts and we don't want to; - let mut ts = libc::timespec { - tv_sec: std::cmp::min(libc::time_t::max_value() as u64, dur.as_secs()) as libc::time_t, - tv_nsec: dur.subsec_nanos() as _, - }; - let res = unsafe { libc::nanosleep(&ts, &mut ts) }; + let ts = TimeSpec::from(dur); + let res = unsafe { libc::nanosleep(ts.as_ref(), std::ptr::null_mut()) }; let interrupted = res == -1 && nix::errno::errno() == libc::EINTR; if interrupted { @@ -600,14 +586,7 @@ mod unix { target_os = "redox" )))] pub(super) fn get_thread_time(vm: &VirtualMachine) -> PyResult { - let time: libc::timespec = unsafe { - let mut time = std::mem::MaybeUninit::uninit(); - if libc::clock_gettime(CLOCK_THREAD_CPUTIME_ID, time.as_mut_ptr()) == -1 { - return Err(vm.new_os_error("Failed to get clock time".to_owned())); - } - time.assume_init() - }; - Ok(Duration::new(time.tv_sec as u64, time.tv_nsec as u32)) + get_clock_time(ClockId::CLOCK_THREAD_CPUTIME_ID, vm) } #[cfg(target_os = "solaris")] @@ -622,14 +601,7 @@ mod unix { target_os = "openbsd", )))] pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult { - let time: libc::timespec = unsafe { - let mut time = std::mem::MaybeUninit::uninit(); - if libc::clock_gettime(CLOCK_PROCESS_CPUTIME_ID, time.as_mut_ptr()) == -1 { - return Err(vm.new_os_error("Failed to get clock time".to_owned())); - } - time.assume_init() - }; - Ok(Duration::new(time.tv_sec as u64, time.tv_nsec as u32)) + get_clock_time(ClockId::CLOCK_PROCESS_CPUTIME_ID, vm) } #[cfg(any( @@ -639,6 +611,7 @@ mod unix { target_os = "openbsd", ))] pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult { + use nix::sys::resource::{getrusage, UsageWho}; fn from_timeval(tv: libc::timeval, vm: &VirtualMachine) -> PyResult { (|tv: libc::timeval| { let t = tv.tv_sec.checked_mul(SEC_TO_NS)?; @@ -649,15 +622,9 @@ mod unix { vm.new_overflow_error("timestamp too large to convert to i64".to_owned()) }) } - let ru: libc::rusage = unsafe { - let mut ru = std::mem::MaybeUninit::uninit(); - if libc::getrusage(libc::RUSAGE_SELF, ru.as_mut_ptr()) == -1 { - return Err(vm.new_os_error("Failed to get clock time".to_owned())); - } - ru.assume_init() - }; - let utime = from_timeval(ru.ru_utime, vm)?; - let stime = from_timeval(ru.ru_stime, vm)?; + let ru = getrusage(UsageWho::RUSAGE_SELF).map_err(|e| e.into_pyexception(vm))?; + let utime = from_timeval(ru.user_time().into(), vm)?; + let stime = from_timeval(ru.system_time().into(), vm)?; Ok(Duration::from_nanos((utime + stime) as u64)) } diff --git a/vm/src/vm/setting.rs b/vm/src/vm/setting.rs index 53a4c59a38a..362b47a5cd6 100644 --- a/vm/src/vm/setting.rs +++ b/vm/src/vm/setting.rs @@ -16,6 +16,9 @@ pub struct Settings { /// -O optimization switch counter pub optimize: u8, + /// Not set SIGINT handler(i.e. for embedded mode) + pub no_sig_int: bool, + /// -s pub no_user_site: bool, @@ -85,6 +88,7 @@ impl Default for Settings { inspect: false, interactive: false, optimize: 0, + no_sig_int: false, no_user_site: false, no_site: false, ignore_environment: false, diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index cf77cd1d05a..8a28f903f8d 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -23,13 +23,12 @@ rustpython-stdlib = { path = "../../stdlib", default-features = false, optional # make sure no threading! otherwise wasm build will fail rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] } +serde = { workspace = true } + console_error_panic_hook = "0.1" js-sys = "0.3" -# make parking_lot use wasm-bingden for instant -parking_lot = { version = "0.12.0" } -serde = "1.0" serde-wasm-bindgen = "0.3.1" -wasm-bindgen = "0.2" +wasm-bindgen = "0.2.80" wasm-bindgen-futures = "0.4" web-sys = { version = "0.3", features = [ "console", diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 144f67e3feb..9f302440cc8 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -29,7 +29,7 @@ mod _browser { "json" => Ok(FetchResponseFormat::Json), "text" => Ok(FetchResponseFormat::Text), "array_buffer" => Ok(FetchResponseFormat::ArrayBuffer), - _ => Err(vm.new_type_error("Unkown fetch response_format".into())), + _ => Err(vm.new_type_error("Unknown fetch response_format".into())), } } fn get_response(&self, response: &web_sys::Response) -> Result { diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index b016cb0ac33..f5be91a46fa 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -72,7 +72,7 @@ pub fn js_err_to_py_err(vm: &VirtualMachine, js_err: &JsValue) -> PyBaseExceptio } None => vm.new_exception_msg( vm.ctx.exceptions.exception_type.to_owned(), - format!("{:?}", js_err), + format!("{js_err:?}"), ), } } @@ -246,7 +246,7 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { } pub fn syntax_err(err: CompileError) -> SyntaxError { - let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err)); + let js_err = SyntaxError::new(&format!("Error parsing Python code: {err}")); let _ = Reflect::set(&js_err, &"row".into(), &(err.location.row() as u32).into()); let _ = Reflect::set( &js_err, diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs index 9e470532b2e..a707b6f4933 100644 --- a/wasm/lib/src/js_module.rs +++ b/wasm/lib/src/js_module.rs @@ -160,7 +160,7 @@ mod _js { .map(PyJsValue::new) .map_err(|err| new_js_error(vm, err)) } else { - Err(vm.new_attribute_error(format!("No attribute {:?} on JS value", name))) + Err(vm.new_attribute_error(format!("No attribute {name:?} on JS value"))) } } diff --git a/wasm/notebook/src/index.js b/wasm/notebook/src/index.js index f91907a5361..799c49ae723 100644 --- a/wasm/notebook/src/index.js +++ b/wasm/notebook/src/index.js @@ -229,7 +229,7 @@ async function executeNotebook() { */ let parsedCode = iomdParser(mainCode); for (const chunk of parsedCode) { - // For each type of chunk, do somthing + // For each type of chunk, do something // so far have py for python, md for markdown and math for math ;p let content = chunk.chunkContent; switch (chunk.chunkType) { diff --git a/wasm/notebook/src/process.js b/wasm/notebook/src/process.js index d0d633374a0..fc6eb6cd247 100644 --- a/wasm/notebook/src/process.js +++ b/wasm/notebook/src/process.js @@ -19,7 +19,7 @@ function renderMarkdown(md) { // Render Math with Katex function renderMath(math) { - // TODO: definetly add error handling. + // TODO: definitely add error handling. return katex.renderToString(math, { macros: { '\\f': '#1f(#2)' }, }); diff --git a/whats_left.py b/whats_left.py index ef97a602b37..46517468dbb 100755 --- a/whats_left.py +++ b/whats_left.py @@ -345,7 +345,7 @@ def compare(): import json import platform - def method_incompatability_reason(typ, method_name, real_method_value): + def method_incompatibility_reason(typ, method_name, real_method_value): has_method = hasattr(typ, method_name) if not has_method: return "" @@ -364,7 +364,7 @@ def method_incompatability_reason(typ, method_name, real_method_value): for name, (typ, real_value, methods) in expected_methods.items(): missing_methods = {} for method, real_method_value in methods: - reason = method_incompatability_reason(typ, method, real_method_value) + reason = method_incompatibility_reason(typ, method, real_method_value) if reason is not None: missing_methods[method] = reason if missing_methods: