From 35cf81d3893a7e2f0baf590c7efb686d020dbea5 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 2 Jul 2020 15:47:56 +0000 Subject: [PATCH] Bug 1648885 and Bug 1649432: vendor latest Cranelift to get Spectre mitigations and fix fuzzbug. r=bbouvier This patch pulls in Cranelift revision 47a218f908e6bdeb7a0fb65ed74e58a0b608080d, which incorporates several relevant changes: - It includes the Spectre mitigation for explicit heap bounds checks merged in PR bytecodealliance/wasmtime#1930, resolving Bug 1648885. - It includes the fix for an out-of-bounds subtraction on large shift amounts merged in PR bytecodealliance/wasmtime#1954, resolving Bug 1649432. We need to temporarily disable the `wasm/limits.js` jit-test on Cranelift configurations because it now needs shared memory to work, and the Cranelift backend does not support this yet. Given that this should be ready in the next month at most (requires atomics support on AArch64, which is currently being examined), it seems simpler to temporarily disable the test on aarch64 than to try to disentangle the bits that depend on shared memories explicitly. This patch also edits the `regexp/bug1445907.js` jit-test to run only if Wasm debugging is supported. This is needed for the test not to fail with `--wasm-compiler=cranelift` (which disables Baseline, the only Wasm compiler that supports debugging). Differential Revision: https://phabricator.services.mozilla.com/D81936 --- .cargo/config.in | 2 +- Cargo.lock | 20 +- Cargo.toml | 4 +- js/src/jit-test/tests/regexp/bug1445907.js | 2 +- js/src/jit-test/tests/wasm/limits.js | 3 + js/src/wasm/cranelift/src/wasm2clif.rs | 21 +- .../.cargo-checksum.json | 2 +- .../src/isa/x86/encodings.rs | 73 +- .../src/isa/x86/legalize.rs | 8 +- .../src/isa/x86/opcodes.rs | 15 + .../src/shared/instructions.rs | 74 +- .../src/shared/settings.rs | 18 + .../cranelift-codegen/.cargo-checksum.json | 2 +- .../src/binemit/relaxation.rs | 6 +- .../cranelift-codegen/src/binemit/stackmap.rs | 2 +- .../rust/cranelift-codegen/src/ir/extfunc.rs | 2 +- .../cranelift-codegen/src/ir/globalvalue.rs | 2 +- .../rust/cranelift-codegen/src/ir/valueloc.rs | 4 +- .../cranelift-codegen/src/isa/aarch64/abi.rs | 6 +- .../src/isa/aarch64/inst/emit.rs | 49 +- .../src/isa/aarch64/inst/emit_tests.rs | 105 ++ .../src/isa/aarch64/inst/imms.rs | 8 + .../src/isa/aarch64/inst/mod.rs | 59 +- .../src/isa/aarch64/inst/regs.rs | 11 +- .../src/isa/aarch64/lower.rs | 132 +- .../src/isa/aarch64/lower_inst.rs | 385 ++--- .../rust/cranelift-codegen/src/isa/mod.rs | 1 + .../rust/cranelift-codegen/src/isa/x64/abi.rs | 881 +++++++++--- .../src/isa/x64/inst/args.rs | 548 ++++--- .../src/isa/x64/inst/emit.rs | 1184 ++++++++------- .../src/isa/x64/inst/emit_tests.rs | 1265 +++++++++++------ .../cranelift-codegen/src/isa/x64/inst/mod.rs | 674 +++++---- .../cranelift-codegen/src/isa/x64/lower.rs | 693 +++++++-- .../rust/cranelift-codegen/src/isa/x64/mod.rs | 2 +- .../src/isa/x86/enc_tables.rs | 149 ++ .../cranelift-codegen/src/legalizer/heap.rs | 79 +- .../cranelift-codegen/src/machinst/buffer.rs | 18 +- .../cranelift-codegen/src/machinst/lower.rs | 6 +- .../cranelift-codegen/src/machinst/vcode.rs | 9 +- .../rust/cranelift-codegen/src/postopt.rs | 6 +- .../cranelift-codegen/src/preopt.serialized | Bin 5536 -> 5438 bytes .../src/redundant_reload_remover.rs | 6 +- .../rust/cranelift-codegen/src/result.rs | 2 +- .../rust/cranelift-codegen/src/settings.rs | 1 + .../cranelift-frontend/.cargo-checksum.json | 2 +- .../rust/cranelift-frontend/src/frontend.rs | 10 + .../rust/cranelift-wasm/.cargo-checksum.json | 2 +- third_party/rust/cranelift-wasm/Cargo.toml | 2 +- third_party/rust/cranelift-wasm/README.md | 2 +- .../cranelift-wasm/src/code_translator.rs | 72 +- .../rust/cranelift-wasm/src/environ/dummy.rs | 20 +- .../rust/cranelift-wasm/src/environ/spec.rs | 57 +- .../cranelift-wasm/src/func_translator.rs | 3 +- third_party/rust/cranelift-wasm/src/lib.rs | 1 + .../cranelift-wasm/src/module_translator.rs | 5 + .../cranelift-wasm/src/sections_translator.rs | 54 +- .../cranelift-wasm/src/state/func_state.rs | 4 +- .../cranelift-wasm/src/translation_utils.rs | 24 +- .../rust/wasmparser/.cargo-checksum.json | 2 +- third_party/rust/wasmparser/Cargo.lock | 151 +- third_party/rust/wasmparser/Cargo.toml | 10 +- .../rust/wasmparser/benches/benchmark.rs | 189 ++- third_party/rust/wasmparser/compare-master.sh | 12 - .../rust/wasmparser/compare-with-main.sh | 10 + third_party/rust/wasmparser/examples/dump.rs | 2 +- .../rust/wasmparser/examples/simple.rs | 8 +- .../rust/wasmparser/src/binary_reader.rs | 95 +- third_party/rust/wasmparser/src/lib.rs | 11 + third_party/rust/wasmparser/src/limits.rs | 6 +- .../rust/wasmparser/src/module_resources.rs | 33 +- .../wasmparser/src/operators_validator.rs | 174 +-- third_party/rust/wasmparser/src/parser.rs | 159 ++- third_party/rust/wasmparser/src/primitives.rs | 62 +- .../wasmparser/src/readers/alias_section.rs | 84 ++ .../wasmparser/src/readers/import_section.rs | 19 +- .../src/readers/instance_section.rs | 147 ++ .../rust/wasmparser/src/readers/mod.rs | 15 +- .../rust/wasmparser/src/readers/module.rs | 85 +- .../src/readers/module_code_section.rs | 93 ++ .../wasmparser/src/readers/module_section.rs | 55 + .../wasmparser/src/readers/name_section.rs | 16 + .../wasmparser/src/readers/type_section.rs | 23 +- third_party/rust/wasmparser/src/validator.rs | 1128 +++++++++++---- 83 files changed, 6679 insertions(+), 2707 deletions(-) delete mode 100755 third_party/rust/wasmparser/compare-master.sh create mode 100755 third_party/rust/wasmparser/compare-with-main.sh create mode 100644 third_party/rust/wasmparser/src/readers/alias_section.rs create mode 100644 third_party/rust/wasmparser/src/readers/instance_section.rs create mode 100644 third_party/rust/wasmparser/src/readers/module_code_section.rs create mode 100644 third_party/rust/wasmparser/src/readers/module_section.rs diff --git a/.cargo/config.in b/.cargo/config.in index 4430a2b8e38c..c87080876cae 100644 --- a/.cargo/config.in +++ b/.cargo/config.in @@ -65,7 +65,7 @@ rev = "3224e2dee65c0726c448484d4c3c43956b9330ec" [source."https://github.com/bytecodealliance/wasmtime"] git = "https://github.com/bytecodealliance/wasmtime" replace-with = "vendored-sources" -rev = "238ae3bf2111847f60089656eb97fc9345295b1f" +rev = "47a218f908e6bdeb7a0fb65ed74e58a0b608080d" [source."https://github.com/badboy/failure"] git = "https://github.com/badboy/failure" diff --git a/Cargo.lock b/Cargo.lock index 4e115471dbe1..fb91aeef2ae7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -734,7 +734,7 @@ dependencies = [ [[package]] name = "cranelift-bforest" version = "0.65.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=238ae3bf2111847f60089656eb97fc9345295b1f#238ae3bf2111847f60089656eb97fc9345295b1f" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=47a218f908e6bdeb7a0fb65ed74e58a0b608080d#47a218f908e6bdeb7a0fb65ed74e58a0b608080d" dependencies = [ "cranelift-entity 0.65.0", ] @@ -742,7 +742,7 @@ dependencies = [ [[package]] name = "cranelift-codegen" version = "0.65.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=238ae3bf2111847f60089656eb97fc9345295b1f#238ae3bf2111847f60089656eb97fc9345295b1f" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=47a218f908e6bdeb7a0fb65ed74e58a0b608080d#47a218f908e6bdeb7a0fb65ed74e58a0b608080d" dependencies = [ "byteorder", "cranelift-bforest", @@ -759,7 +759,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" version = "0.65.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=238ae3bf2111847f60089656eb97fc9345295b1f#238ae3bf2111847f60089656eb97fc9345295b1f" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=47a218f908e6bdeb7a0fb65ed74e58a0b608080d#47a218f908e6bdeb7a0fb65ed74e58a0b608080d" dependencies = [ "cranelift-codegen-shared", "cranelift-entity 0.65.0", @@ -768,7 +768,7 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" version = "0.65.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=238ae3bf2111847f60089656eb97fc9345295b1f#238ae3bf2111847f60089656eb97fc9345295b1f" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=47a218f908e6bdeb7a0fb65ed74e58a0b608080d#47a218f908e6bdeb7a0fb65ed74e58a0b608080d" [[package]] name = "cranelift-entity" @@ -778,12 +778,12 @@ source = "git+https://github.com/PLSysSec/lucet_sandbox_compiler?rev=5e870faf6f9 [[package]] name = "cranelift-entity" version = "0.65.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=238ae3bf2111847f60089656eb97fc9345295b1f#238ae3bf2111847f60089656eb97fc9345295b1f" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=47a218f908e6bdeb7a0fb65ed74e58a0b608080d#47a218f908e6bdeb7a0fb65ed74e58a0b608080d" [[package]] name = "cranelift-frontend" version = "0.65.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=238ae3bf2111847f60089656eb97fc9345295b1f#238ae3bf2111847f60089656eb97fc9345295b1f" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=47a218f908e6bdeb7a0fb65ed74e58a0b608080d#47a218f908e6bdeb7a0fb65ed74e58a0b608080d" dependencies = [ "cranelift-codegen", "log", @@ -794,14 +794,14 @@ dependencies = [ [[package]] name = "cranelift-wasm" version = "0.65.0" -source = "git+https://github.com/bytecodealliance/wasmtime?rev=238ae3bf2111847f60089656eb97fc9345295b1f#238ae3bf2111847f60089656eb97fc9345295b1f" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=47a218f908e6bdeb7a0fb65ed74e58a0b608080d#47a218f908e6bdeb7a0fb65ed74e58a0b608080d" dependencies = [ "cranelift-codegen", "cranelift-entity 0.65.0", "cranelift-frontend", "log", "thiserror", - "wasmparser 0.57.0", + "wasmparser 0.58.0", ] [[package]] @@ -5381,9 +5381,9 @@ checksum = "073da89bf1c84db000dd68ce660c1b4a08e3a2d28fd1e3394ab9e7abdde4a0f8" [[package]] name = "wasmparser" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32fddd575d477c6e9702484139cf9f23dcd554b06d185ed0f56c857dd3a47aa6" +checksum = "721a8d79483738d7aef6397edcf8f04cd862640b1ad5973adf5bb50fc10e86db" [[package]] name = "wast" diff --git a/Cargo.toml b/Cargo.toml index e5ef5701cfc7..09f27bea7288 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,8 +74,8 @@ failure_derive = { git = "https://github.com/badboy/failure", rev = "64af847bc5f [patch.crates-io.cranelift-codegen] git = "https://github.com/bytecodealliance/wasmtime" -rev = "238ae3bf2111847f60089656eb97fc9345295b1f" +rev = "47a218f908e6bdeb7a0fb65ed74e58a0b608080d" [patch.crates-io.cranelift-wasm] git = "https://github.com/bytecodealliance/wasmtime" -rev = "238ae3bf2111847f60089656eb97fc9345295b1f" +rev = "47a218f908e6bdeb7a0fb65ed74e58a0b608080d" diff --git a/js/src/jit-test/tests/regexp/bug1445907.js b/js/src/jit-test/tests/regexp/bug1445907.js index 90df4b8172c4..7f084f046931 100644 --- a/js/src/jit-test/tests/regexp/bug1445907.js +++ b/js/src/jit-test/tests/regexp/bug1445907.js @@ -1,4 +1,4 @@ -// |jit-test| skip-if: !wasmIsSupported() +// |jit-test| skip-if: !wasmDebuggingIsSupported() // On ARM64, we failed to save x28 properly when generating code for the regexp // matcher. diff --git a/js/src/jit-test/tests/wasm/limits.js b/js/src/jit-test/tests/wasm/limits.js index 152f7d25bcf3..08e2145694bd 100644 --- a/js/src/jit-test/tests/wasm/limits.js +++ b/js/src/jit-test/tests/wasm/limits.js @@ -1,3 +1,6 @@ +// |jit-test| skip-if: wasmCompilersPresent().match("cranelift") +// (Reason: the Cranelift backend does not support shared memory yet.) + // Tests of limits of memory and table types const PageSize = 65536; diff --git a/js/src/wasm/cranelift/src/wasm2clif.rs b/js/src/wasm/cranelift/src/wasm2clif.rs index a16057a33a7f..bcbd9143f3c8 100644 --- a/js/src/wasm/cranelift/src/wasm2clif.rs +++ b/js/src/wasm/cranelift/src/wasm2clif.rs @@ -29,8 +29,8 @@ use cranelift_codegen::ir::InstBuilder; use cranelift_codegen::isa::{CallConv, TargetFrontendConfig, TargetIsa}; use cranelift_codegen::packed_option::PackedOption; use cranelift_wasm::{ - FuncEnvironment, FuncIndex, GlobalIndex, GlobalVariable, MemoryIndex, ReturnMode, - SignatureIndex, TableIndex, TargetEnvironment, WasmError, WasmResult, + FuncEnvironment, FuncIndex, FunctionBuilder, GlobalIndex, GlobalVariable, MemoryIndex, + ReturnMode, SignatureIndex, TableIndex, TargetEnvironment, WasmError, WasmResult, }; use crate::bindings::{self, GlobalDesc, SymbolicAddress}; @@ -1085,6 +1085,7 @@ impl<'static_env, 'module_env> FuncEnvironment for TransEnv<'static_env, 'module &mut self, mut pos: FuncCursor, table_index: TableIndex, + _table: ir::Table, delta: ir::Value, init_value: ir::Value, ) -> WasmResult { @@ -1096,10 +1097,14 @@ impl<'static_env, 'module_env> FuncEnvironment for TransEnv<'static_env, 'module fn translate_table_get( &mut self, - mut pos: FuncCursor, + builder: &mut FunctionBuilder, table_index: TableIndex, + _table: ir::Table, index: ir::Value, ) -> WasmResult { + // TODO(bug 1650038): make use of the `FunctionBuilder` here and its + // ability to edit the CFG in order to add a fast-path. + let mut pos = builder.cursor(); let table_index = pos.ins().iconst(ir::types::I32, table_index.index() as i64); Ok(self .instance_call(&mut pos, &FN_TABLE_GET, &[index, table_index]) @@ -1108,11 +1113,15 @@ impl<'static_env, 'module_env> FuncEnvironment for TransEnv<'static_env, 'module fn translate_table_set( &mut self, - mut pos: FuncCursor, + builder: &mut FunctionBuilder, table_index: TableIndex, + _table: ir::Table, value: ir::Value, index: ir::Value, ) -> WasmResult<()> { + // TODO(bug 1650038): make use of the `FunctionBuilder` here and its + // ability to edit the CFG in order to add a fast-path. + let mut pos = builder.cursor(); let table_index = pos.ins().iconst(ir::types::I32, table_index.index() as i64); self.instance_call(&mut pos, &FN_TABLE_SET, &[index, value, table_index]); Ok(()) @@ -1187,9 +1196,9 @@ impl<'static_env, 'module_env> FuncEnvironment for TransEnv<'static_env, 'module fn translate_ref_func( &mut self, mut pos: FuncCursor, - func_index: u32, + func_index: FuncIndex, ) -> WasmResult { - let func_index = pos.ins().iconst(ir::types::I32, func_index as i64); + let func_index = pos.ins().iconst(ir::types::I32, func_index.index() as i64); Ok(self .instance_call(&mut pos, &FN_REF_FUNC, &[func_index]) .unwrap()) diff --git a/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json b/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json index 148166c2adc3..77d87deb8f6e 100644 --- a/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json +++ b/third_party/rust/cranelift-codegen-meta/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"7c01a301a32e60cd9b0edd66f4cf8700e5de1d31607437ea756d4f8b0ae29a54","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"b123f056d0d458396679c5f7f2a16d2762af0258fcda4ac14b6655a95e5a0022","src/cdsl/ast.rs":"84a4b7e3301e3249716958a7aa4ea5ba8c6172e3c02f57ee3880504c4433ff19","src/cdsl/cpu_modes.rs":"996e45b374cfe85ac47c8c86c4459fe4c04b3158102b4c63b6ee434d5eed6a9e","src/cdsl/encodings.rs":"d884a564815a03c23369bcf31d13b122ae5ba84d0c80eda9312f0c0a829bf794","src/cdsl/formats.rs":"63e638305aa3ca6dd409ddf0e5e9605eeac1cc2631103e42fc6cbc87703d9b63","src/cdsl/instructions.rs":"41e1a230501de3f0da3960d8aa375c8bcd60ec62ede94ad61806816acbd8009a","src/cdsl/isa.rs":"ccabd6848b69eb069c10db61c7e7f86080777495714bb53d03e663c40541be94","src/cdsl/mod.rs":"0aa827923bf4c45e5ee2359573bd863e00f474acd532739f49dcd74a27553882","src/cdsl/operands.rs":"1c3411504de9c83112ff48e0ff1cfbb2e4ba5a9a15c1716f411ef31a4df59899","src/cdsl/recipes.rs":"80b7cd87332229b569e38086ceee8d557e679b9a32ad2e50bdb15c33337c3418","src/cdsl/regs.rs":"466a42a43355fc7623fe5d8e8d330622207a3af6a80cb9367bc0f06e224c9ee0","src/cdsl/settings.rs":"e6fd9a31925743b93b11f09c9c8271bab6aa2430aa053a2601957b4487df7d77","src/cdsl/type_inference.rs":"1efca8a095ffc899b7527bda6b9d9378c73d7283f8dceaa4819e8af599f8be21","src/cdsl/types.rs":"ff764c9e9c29a05677bff6164e7bc25a0c32655052d77ae580536abba8b1713b","src/cdsl/typevar.rs":"371ac795added2cb464371443313eb55350c629c62ce8e62e192129b6c41d45e","src/cdsl/xform.rs":"55da0c3f2403147b535ab6ae5d69c623fbe839edecf2a3af1de84420cd58402d","src/default_map.rs":"101bb0282a124f9c921f6bd095f529e8753621450d783c3273b0b0394c2c5c03","src/error.rs":"e9b11b2feb2d867b94c8810fdc5a6c4e0d9131604a0bfa5340ff2639a55100b4","src/gen_binemit.rs":"515e243420b30d1e01f8ea630282d9b6d78a715e1951f3f20392e19a48164442","src/gen_encodings.rs":"f00cded6b68a9b48c9e3cd39a8b6f0ba136f4062c8f8666109158a72c62c3ed1","src/gen_inst.rs":"88532d2e2c9724dde968d6b046927249c33d2037ab3e3fd1bd7ebfa77fe12bc7","src/gen_legalizer.rs":"ea229ab9393cc5ba2242f626e74c624ea59314535e74b26602dafb8e96481a72","src/gen_registers.rs":"a904119ed803c9de24dedd15149a65337ffc168bb1d63df53d7fdebfb5f4b158","src/gen_settings.rs":"f3cc3d31f6cc898f30606caf084f0de220db2d3b1b5e5e4145fa7c9a9a1597e2","src/gen_types.rs":"f6c090e1646a43bf2fe81ae0a7029cc6f7dc6d43285368f56d86c35a21c469a6","src/isa/arm32/mod.rs":"da18cb40c1a0a6b613ddefcc38a5d01d02c95de6f233ebd4ad84fefb992c008b","src/isa/arm64/mod.rs":"3a815eaa478d82b7f8b536b83f9debb6b79ec860f99fea6485f209a836c6939a","src/isa/mod.rs":"136141f99f217ba42b9e3f7f47238ab19cc974bb3bef2e2df7f7b5a683989d46","src/isa/riscv/encodings.rs":"8abb1968d917588bc5fc5f5be6dd66bdec23ac456ba65f8138237c8e891e843c","src/isa/riscv/mod.rs":"a7b461a30bbfbc1e3b33645422ff40d5b1761c30cb5d4a8aa12e9a3b7f7aee51","src/isa/riscv/recipes.rs":"5be3bf7c9ba3c51ece384b7eee75a8f7fa0cbacc6a5babc9d0e1d92a2e54a4c2","src/isa/x86/encodings.rs":"ccb5b5b4cb1861f036835c2ac56aefe2a477c8224fb6b681876461230aab67e5","src/isa/x86/instructions.rs":"1aee81c8bc0215fa1cad83e97a0915b24521ae61d503cd727a2406a25dd60f29","src/isa/x86/legalize.rs":"0809d49dbc49d35f33e027890265179ebfda3c55ed252040f8c4ff35d6ee7b02","src/isa/x86/mod.rs":"2b84474c2b0e272c1ebe32530c57f6b11133127c286c8f82c5ae5b6486386238","src/isa/x86/opcodes.rs":"ed8a0e536e290b2930a88816944692c4baa043684c00eafa51da183cdbb59f7d","src/isa/x86/recipes.rs":"c63469f430e457554acf1534f6fe8f37b41984d38d272e023aa0d93b778dc993","src/isa/x86/registers.rs":"4be0a45d8acd465c31746b7976124025b06b453e3f6d587f93efb5af0e12b1a8","src/isa/x86/settings.rs":"2d3e09ca34638e19621aef2492ca6943b105e6add830bd91bddbdc85277cb680","src/lib.rs":"2491b0e74078914cb89d1778fa8174daf723fe76aaf7fed18741237d68f6df32","src/shared/entities.rs":"90f774a70e1c2a2e9a553c07a5e80e0fe54cf127434bd83e67274bba4e1a19ba","src/shared/formats.rs":"2f8cbb008778a49b60efac4647dffef654d225823e03ca6272af2678666dc423","src/shared/immediates.rs":"e4a57657f6af9853794804eb41c01204a2c13a632f44f55d90e156a4b98c5f65","src/shared/instructions.rs":"06dae423ead6c1934fcf5813ddbd4f9983a4957e8ac9a17c88b014903bf71f41","src/shared/legalize.rs":"e8fd35104c1907c0e9453fb98372373aea20b54af10457156f6abd86929099dc","src/shared/mod.rs":"c219625990bf15507ac1077b349ce20e5312d4e4707426183676d469e78792b7","src/shared/settings.rs":"0b4f903de5f2df19304c44bf4bd456c3a8e165103b38ccb13b6f88ae8a3c7ee8","src/shared/types.rs":"4702df132f4b5d70cc9411ec5221ba0b1bd4479252274e0223ae57b6d0331247","src/srcgen.rs":"dcfc159c8599270f17e6a978c4be255abca51556b5ef0da497faec4a4a1e62ce","src/unique_table.rs":"31aa54330ca4786af772d32e8cb6158b6504b88fa93fe177bf0c6cbe545a8d35"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"7c01a301a32e60cd9b0edd66f4cf8700e5de1d31607437ea756d4f8b0ae29a54","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"b123f056d0d458396679c5f7f2a16d2762af0258fcda4ac14b6655a95e5a0022","src/cdsl/ast.rs":"84a4b7e3301e3249716958a7aa4ea5ba8c6172e3c02f57ee3880504c4433ff19","src/cdsl/cpu_modes.rs":"996e45b374cfe85ac47c8c86c4459fe4c04b3158102b4c63b6ee434d5eed6a9e","src/cdsl/encodings.rs":"d884a564815a03c23369bcf31d13b122ae5ba84d0c80eda9312f0c0a829bf794","src/cdsl/formats.rs":"63e638305aa3ca6dd409ddf0e5e9605eeac1cc2631103e42fc6cbc87703d9b63","src/cdsl/instructions.rs":"41e1a230501de3f0da3960d8aa375c8bcd60ec62ede94ad61806816acbd8009a","src/cdsl/isa.rs":"ccabd6848b69eb069c10db61c7e7f86080777495714bb53d03e663c40541be94","src/cdsl/mod.rs":"0aa827923bf4c45e5ee2359573bd863e00f474acd532739f49dcd74a27553882","src/cdsl/operands.rs":"1c3411504de9c83112ff48e0ff1cfbb2e4ba5a9a15c1716f411ef31a4df59899","src/cdsl/recipes.rs":"80b7cd87332229b569e38086ceee8d557e679b9a32ad2e50bdb15c33337c3418","src/cdsl/regs.rs":"466a42a43355fc7623fe5d8e8d330622207a3af6a80cb9367bc0f06e224c9ee0","src/cdsl/settings.rs":"e6fd9a31925743b93b11f09c9c8271bab6aa2430aa053a2601957b4487df7d77","src/cdsl/type_inference.rs":"1efca8a095ffc899b7527bda6b9d9378c73d7283f8dceaa4819e8af599f8be21","src/cdsl/types.rs":"ff764c9e9c29a05677bff6164e7bc25a0c32655052d77ae580536abba8b1713b","src/cdsl/typevar.rs":"371ac795added2cb464371443313eb55350c629c62ce8e62e192129b6c41d45e","src/cdsl/xform.rs":"55da0c3f2403147b535ab6ae5d69c623fbe839edecf2a3af1de84420cd58402d","src/default_map.rs":"101bb0282a124f9c921f6bd095f529e8753621450d783c3273b0b0394c2c5c03","src/error.rs":"e9b11b2feb2d867b94c8810fdc5a6c4e0d9131604a0bfa5340ff2639a55100b4","src/gen_binemit.rs":"515e243420b30d1e01f8ea630282d9b6d78a715e1951f3f20392e19a48164442","src/gen_encodings.rs":"f00cded6b68a9b48c9e3cd39a8b6f0ba136f4062c8f8666109158a72c62c3ed1","src/gen_inst.rs":"88532d2e2c9724dde968d6b046927249c33d2037ab3e3fd1bd7ebfa77fe12bc7","src/gen_legalizer.rs":"ea229ab9393cc5ba2242f626e74c624ea59314535e74b26602dafb8e96481a72","src/gen_registers.rs":"a904119ed803c9de24dedd15149a65337ffc168bb1d63df53d7fdebfb5f4b158","src/gen_settings.rs":"f3cc3d31f6cc898f30606caf084f0de220db2d3b1b5e5e4145fa7c9a9a1597e2","src/gen_types.rs":"f6c090e1646a43bf2fe81ae0a7029cc6f7dc6d43285368f56d86c35a21c469a6","src/isa/arm32/mod.rs":"da18cb40c1a0a6b613ddefcc38a5d01d02c95de6f233ebd4ad84fefb992c008b","src/isa/arm64/mod.rs":"3a815eaa478d82b7f8b536b83f9debb6b79ec860f99fea6485f209a836c6939a","src/isa/mod.rs":"136141f99f217ba42b9e3f7f47238ab19cc974bb3bef2e2df7f7b5a683989d46","src/isa/riscv/encodings.rs":"8abb1968d917588bc5fc5f5be6dd66bdec23ac456ba65f8138237c8e891e843c","src/isa/riscv/mod.rs":"a7b461a30bbfbc1e3b33645422ff40d5b1761c30cb5d4a8aa12e9a3b7f7aee51","src/isa/riscv/recipes.rs":"5be3bf7c9ba3c51ece384b7eee75a8f7fa0cbacc6a5babc9d0e1d92a2e54a4c2","src/isa/x86/encodings.rs":"2b3c5105e32bce932d2628963cc5c853207e37204a6aec38caace60e52870bbe","src/isa/x86/instructions.rs":"1aee81c8bc0215fa1cad83e97a0915b24521ae61d503cd727a2406a25dd60f29","src/isa/x86/legalize.rs":"ddc834ae8f4a06ca8e3fccf7aef6a097163a2f8d258a7cbc3cc6a8b93c9c0413","src/isa/x86/mod.rs":"2b84474c2b0e272c1ebe32530c57f6b11133127c286c8f82c5ae5b6486386238","src/isa/x86/opcodes.rs":"79d42b71f78119f4ca1dc4fc90bc9efb04c6fc526e01cbe79368aa59f117266a","src/isa/x86/recipes.rs":"c63469f430e457554acf1534f6fe8f37b41984d38d272e023aa0d93b778dc993","src/isa/x86/registers.rs":"4be0a45d8acd465c31746b7976124025b06b453e3f6d587f93efb5af0e12b1a8","src/isa/x86/settings.rs":"2d3e09ca34638e19621aef2492ca6943b105e6add830bd91bddbdc85277cb680","src/lib.rs":"2491b0e74078914cb89d1778fa8174daf723fe76aaf7fed18741237d68f6df32","src/shared/entities.rs":"90f774a70e1c2a2e9a553c07a5e80e0fe54cf127434bd83e67274bba4e1a19ba","src/shared/formats.rs":"2f8cbb008778a49b60efac4647dffef654d225823e03ca6272af2678666dc423","src/shared/immediates.rs":"e4a57657f6af9853794804eb41c01204a2c13a632f44f55d90e156a4b98c5f65","src/shared/instructions.rs":"5ffa26a91b344fb7014a34e0d97b4df90d604a5bd49a49a75c262591deb8e6c4","src/shared/legalize.rs":"e8fd35104c1907c0e9453fb98372373aea20b54af10457156f6abd86929099dc","src/shared/mod.rs":"c219625990bf15507ac1077b349ce20e5312d4e4707426183676d469e78792b7","src/shared/settings.rs":"7800f51d97a95d572310f6c80ded59c1c84cf3ba06f9425f4205f88ac46b4e98","src/shared/types.rs":"4702df132f4b5d70cc9411ec5221ba0b1bd4479252274e0223ae57b6d0331247","src/srcgen.rs":"dcfc159c8599270f17e6a978c4be255abca51556b5ef0da497faec4a4a1e62ce","src/unique_table.rs":"31aa54330ca4786af772d32e8cb6158b6504b88fa93fe177bf0c6cbe545a8d35"},"package":null} \ No newline at end of file diff --git a/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs b/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs index d507abc0f5fb..303b1bfaebb5 100644 --- a/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs +++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/encodings.rs @@ -231,6 +231,32 @@ impl PerCpuModeEncodings { }); } + /// Add encodings for `inst.r32` to X86_32. + /// Add encodings for `inst.r32` to X86_64 with and without REX. + /// Add encodings for `inst.r64` to X86_64 with a REX.W prefix. + fn enc_r32_r64_instp( + &mut self, + inst: &Instruction, + template: Template, + instp: InstructionPredicateNode, + ) { + self.enc32_func(inst.bind(R32), template.nonrex(), |builder| { + builder.inst_predicate(instp.clone()) + }); + + // REX-less encoding must come after REX encoding so we don't use it by default. Otherwise + // reg-alloc would never use r8 and up. + self.enc64_func(inst.bind(R32), template.rex(), |builder| { + builder.inst_predicate(instp.clone()) + }); + self.enc64_func(inst.bind(R32), template.nonrex(), |builder| { + builder.inst_predicate(instp.clone()) + }); + self.enc64_func(inst.bind(R64), template.rex().w(), |builder| { + builder.inst_predicate(instp) + }); + } + /// Add encodings for `inst.r32` to X86_32. /// Add encodings for `inst.r64` to X86_64 with a REX.W prefix. fn enc_r32_r64_rex_only(&mut self, inst: impl Into, template: Template) { @@ -810,6 +836,11 @@ fn define_memory( recipe.opcodes(&MOV_LOAD), is_load_complex_length_two.clone(), ); + e.enc_r32_r64_instp( + load_complex, + recipe.opcodes(&MOV_LOAD), + is_load_complex_length_two.clone(), + ); e.enc_x86_64_instp( uload32_complex, recipe.opcodes(&MOV_LOAD), @@ -855,6 +886,11 @@ fn define_memory( recipe.opcodes(&MOV_STORE), is_store_complex_length_three.clone(), ); + e.enc_r32_r64_instp( + store_complex, + recipe.opcodes(&MOV_STORE), + is_store_complex_length_three.clone(), + ); e.enc_x86_64_instp( istore32_complex, recipe.opcodes(&MOV_STORE), @@ -948,6 +984,10 @@ fn define_memory( e.enc64_rec(fill_nop.bind(ty), rec_ffillnull, 0); e.enc32_rec(fill_nop.bind(ty), rec_ffillnull, 0); } + for &ty in &[R64, R32] { + e.enc64_rec(fill_nop.bind(ty), rec_fillnull, 0); + e.enc32_rec(fill_nop.bind(ty), rec_fillnull, 0); + } // Load 32 bits from `b1`, `i8` and `i16` spill slots. See `spill.b1` above. @@ -1355,6 +1395,7 @@ fn define_alu( let rotr = shared.by_name("rotr"); let rotr_imm = shared.by_name("rotr_imm"); let selectif = shared.by_name("selectif"); + let selectif_spectre_guard = shared.by_name("selectif_spectre_guard"); let sshr = shared.by_name("sshr"); let sshr_imm = shared.by_name("sshr_imm"); let trueff = shared.by_name("trueff"); @@ -1568,6 +1609,11 @@ fn define_alu( // Conditional move (a.k.a integer select). e.enc_i32_i64(selectif, rec_cmov.opcodes(&CMOV_OVERFLOW)); + // A Spectre-guard integer select is exactly the same as a selectif, but + // is not associated with any other legalization rules and is not + // recognized by any optimizations, so it must arrive here unmodified + // and in its original place. + e.enc_i32_i64(selectif_spectre_guard, rec_cmov.opcodes(&CMOV_OVERFLOW)); } #[inline(never)] @@ -1596,10 +1642,9 @@ fn define_simd( let fdiv = shared.by_name("fdiv"); let fill = shared.by_name("fill"); let fill_nop = shared.by_name("fill_nop"); - let fmax = shared.by_name("fmax"); - let fmin = shared.by_name("fmin"); let fmul = shared.by_name("fmul"); let fsub = shared.by_name("fsub"); + let iabs = shared.by_name("iabs"); let iadd = shared.by_name("iadd"); let icmp = shared.by_name("icmp"); let imul = shared.by_name("imul"); @@ -1635,7 +1680,10 @@ fn define_simd( let usub_sat = shared.by_name("usub_sat"); let vconst = shared.by_name("vconst"); let vselect = shared.by_name("vselect"); + let x86_cvtt2si = x86.by_name("x86_cvtt2si"); let x86_insertps = x86.by_name("x86_insertps"); + let x86_fmax = x86.by_name("x86_fmax"); + let x86_fmin = x86.by_name("x86_fmin"); let x86_movlhps = x86.by_name("x86_movlhps"); let x86_movsd = x86.by_name("x86_movsd"); let x86_packss = x86.by_name("x86_packss"); @@ -1902,6 +1950,13 @@ fn define_simd( rec_evex_reg_rm_128.opcodes(&VCVTUDQ2PS), Some(use_avx512vl_simd), // TODO need an OR predicate to join with AVX512F ); + + e.enc_both_inferred( + x86_cvtt2si + .bind(vector(I32, sse_vector_size)) + .bind(vector(F32, sse_vector_size)), + rec_furm.opcodes(&CVTTPS2DQ), + ); } // SIMD vconst for special cases (all zeroes, all ones) @@ -2136,6 +2191,12 @@ fn define_simd( e.enc_both_inferred(avgr, rec_fa.opcodes(opcodes)); } + // SIMD integer absolute value. + for (ty, opcodes) in &[(I8, &PABSB[..]), (I16, &PABSW[..]), (I32, &PABSD)] { + let iabs = iabs.bind(vector(*ty, sse_vector_size)); + e.enc_both_inferred_maybe_isap(iabs, rec_furm.opcodes(opcodes), Some(use_ssse3_simd)); + } + // SIMD logical operations let band = shared.by_name("band"); let band_not = shared.by_name("band_not"); @@ -2268,10 +2329,10 @@ fn define_simd( (F64, fmul, &MULPD[..]), (F32, fdiv, &DIVPS[..]), (F64, fdiv, &DIVPD[..]), - (F32, fmin, &MINPS[..]), - (F64, fmin, &MINPD[..]), - (F32, fmax, &MAXPS[..]), - (F64, fmax, &MAXPD[..]), + (F32, x86_fmin, &MINPS[..]), + (F64, x86_fmin, &MINPD[..]), + (F32, x86_fmax, &MAXPS[..]), + (F64, x86_fmax, &MAXPD[..]), ] { let inst = inst.bind(vector(*ty, sse_vector_size)); e.enc_both_inferred(inst, rec_fa.opcodes(opcodes)); diff --git a/third_party/rust/cranelift-codegen-meta/src/isa/x86/legalize.rs b/third_party/rust/cranelift-codegen-meta/src/isa/x86/legalize.rs index 940ffe6d01ab..51453322e9ba 100644 --- a/third_party/rust/cranelift-codegen-meta/src/isa/x86/legalize.rs +++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/legalize.rs @@ -379,9 +379,12 @@ fn define_simd( let bnot = insts.by_name("bnot"); let bxor = insts.by_name("bxor"); let extractlane = insts.by_name("extractlane"); + let fabs = insts.by_name("fabs"); let fcmp = insts.by_name("fcmp"); let fcvt_from_uint = insts.by_name("fcvt_from_uint"); - let fabs = insts.by_name("fabs"); + let fcvt_to_sint_sat = insts.by_name("fcvt_to_sint_sat"); + let fmax = insts.by_name("fmax"); + let fmin = insts.by_name("fmin"); let fneg = insts.by_name("fneg"); let iadd_imm = insts.by_name("iadd_imm"); let icmp = insts.by_name("icmp"); @@ -788,6 +791,9 @@ fn define_simd( narrow.custom_legalize(ineg, "convert_ineg"); narrow.custom_legalize(ushr, "convert_ushr"); narrow.custom_legalize(ishl, "convert_ishl"); + narrow.custom_legalize(fcvt_to_sint_sat, "expand_fcvt_to_sint_sat_vector"); + narrow.custom_legalize(fmin, "expand_minmax_vector"); + narrow.custom_legalize(fmax, "expand_minmax_vector"); narrow_avx.custom_legalize(imul, "convert_i64x2_imul"); narrow_avx.custom_legalize(fcvt_from_uint, "expand_fcvt_from_uint_vector"); diff --git a/third_party/rust/cranelift-codegen-meta/src/isa/x86/opcodes.rs b/third_party/rust/cranelift-codegen-meta/src/isa/x86/opcodes.rs index d2391fe2eef8..f7f7480f9b7d 100644 --- a/third_party/rust/cranelift-codegen-meta/src/isa/x86/opcodes.rs +++ b/third_party/rust/cranelift-codegen-meta/src/isa/x86/opcodes.rs @@ -103,6 +103,10 @@ pub static CVTSI2SS: [u8; 3] = [0xf3, 0x0f, 0x2a]; /// float-point value. pub static CVTSS2SD: [u8; 3] = [0xf3, 0x0f, 0x5a]; +/// Convert four packed single-precision floating-point values from xmm2/mem to four packed signed +/// doubleword values in xmm1 using truncation (SSE2). +pub static CVTTPS2DQ: [u8; 3] = [0xf3, 0x0f, 0x5b]; + /// Convert with truncation scalar double-precision floating-point value to signed /// integer. pub static CVTTSD2SI: [u8; 3] = [0xf2, 0x0f, 0x2c]; @@ -299,6 +303,17 @@ pub static OR_IMM8_SIGN_EXTEND: [u8; 1] = [0x83]; /// Return the bitwise logical OR of packed single-precision values in xmm and x/m (SSE). pub static ORPS: [u8; 2] = [0x0f, 0x56]; +/// Compute the absolute value of bytes in xmm2/m128 and store the unsigned result in xmm1 (SSSE3). +pub static PABSB: [u8; 4] = [0x66, 0x0f, 0x38, 0x1c]; + +/// Compute the absolute value of 32-bit integers in xmm2/m128 and store the unsigned result in +/// xmm1 (SSSE3). +pub static PABSD: [u8; 4] = [0x66, 0x0f, 0x38, 0x1e]; + +/// Compute the absolute value of 16-bit integers in xmm2/m128 and store the unsigned result in +/// xmm1 (SSSE3). +pub static PABSW: [u8; 4] = [0x66, 0x0f, 0x38, 0x1d]; + /// Converts 8 packed signed word integers from xmm1 and from xxm2/m128 into 16 packed signed byte /// integers in xmm1 using signed saturation (SSE2). pub static PACKSSWB: [u8; 3] = [0x66, 0x0f, 0x63]; diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs b/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs index 7a30a755eed3..fb91ae0ae9ca 100644 --- a/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs +++ b/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs @@ -211,7 +211,7 @@ fn define_control_flow( let iAddr = &TypeVar::new( "iAddr", "An integer address type", - TypeSetBuilder::new().ints(32..64).build(), + TypeSetBuilder::new().ints(32..64).refs(32..64).build(), ); { @@ -549,9 +549,9 @@ fn define_simd_lane_access( r#" Vector swizzle. - Returns a new vector with byte-width lanes selected from the lanes of the first input - vector ``x`` specified in the second input vector ``s``. The indices ``i`` in range - ``[0, 15]`` select the ``i``-th element of ``x``. For indices outside of the range the + Returns a new vector with byte-width lanes selected from the lanes of the first input + vector ``x`` specified in the second input vector ``s``. The indices ``i`` in range + ``[0, 15]`` select the ``i``-th element of ``x``. For indices outside of the range the resulting lane is 0. Note that this operates on byte-width lanes. "#, &formats.binary, @@ -744,7 +744,7 @@ pub(crate) fn define( let iAddr = &TypeVar::new( "iAddr", "An integer address type", - TypeSetBuilder::new().ints(32..64).build(), + TypeSetBuilder::new().ints(32..64).refs(32..64).build(), ); let Ref = &TypeVar::new( @@ -1176,7 +1176,7 @@ pub(crate) fn define( Inst::new( "uload8x8", r#" - Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i16x8 + Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i16x8 vector. "#, &formats.load, @@ -1190,7 +1190,7 @@ pub(crate) fn define( Inst::new( "uload8x8_complex", r#" - Load an 8x8 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an + Load an 8x8 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an i16x8 vector. "#, &formats.load_complex, @@ -1204,7 +1204,7 @@ pub(crate) fn define( Inst::new( "sload8x8", r#" - Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i16x8 + Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i16x8 vector. "#, &formats.load, @@ -1218,7 +1218,7 @@ pub(crate) fn define( Inst::new( "sload8x8_complex", r#" - Load an 8x8 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an + Load an 8x8 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an i16x8 vector. "#, &formats.load_complex, @@ -1243,7 +1243,7 @@ pub(crate) fn define( Inst::new( "uload16x4", r#" - Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i32x4 + Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i32x4 vector. "#, &formats.load, @@ -1257,7 +1257,7 @@ pub(crate) fn define( Inst::new( "uload16x4_complex", r#" - Load a 16x4 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an + Load a 16x4 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an i32x4 vector. "#, &formats.load_complex, @@ -1271,7 +1271,7 @@ pub(crate) fn define( Inst::new( "sload16x4", r#" - Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i32x4 + Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i32x4 vector. "#, &formats.load, @@ -1285,7 +1285,7 @@ pub(crate) fn define( Inst::new( "sload16x4_complex", r#" - Load a 16x4 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an + Load a 16x4 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an i32x4 vector. "#, &formats.load_complex, @@ -1310,7 +1310,7 @@ pub(crate) fn define( Inst::new( "uload32x2", r#" - Load an 32x2 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i64x2 + Load an 32x2 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i64x2 vector. "#, &formats.load, @@ -1324,7 +1324,7 @@ pub(crate) fn define( Inst::new( "uload32x2_complex", r#" - Load a 32x2 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an + Load a 32x2 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an i64x2 vector. "#, &formats.load_complex, @@ -1338,7 +1338,7 @@ pub(crate) fn define( Inst::new( "sload32x2", r#" - Load a 32x2 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i64x2 + Load a 32x2 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i64x2 vector. "#, &formats.load, @@ -1352,7 +1352,7 @@ pub(crate) fn define( Inst::new( "sload32x2_complex", r#" - Load a 32x2 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an + Load a 32x2 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an i64x2 vector. "#, &formats.load_complex, @@ -1748,6 +1748,34 @@ pub(crate) fn define( .operands_out(vec![a]), ); + ig.push( + Inst::new( + "selectif_spectre_guard", + r#" + Conditional select intended for Spectre guards. + + This operation is semantically equivalent to a selectif instruction. + However, it is guaranteed to not be removed or otherwise altered by any + optimization pass, and is guaranteed to result in a conditional-move + instruction, not a branch-based lowering. As such, it is suitable + for use when producing Spectre guards. For example, a bounds-check + may guard against unsafe speculation past a bounds-check conditional + branch by passing the address or index to be accessed through a + conditional move, also gated on the same condition. Because no + Spectre-vulnerable processors are known to perform speculation on + conditional move instructions, this is guaranteed to pick the + correct input. If the selected input in case of overflow is a "safe" + value, for example a null pointer that causes an exception in the + speculative path, this ensures that no Spectre vulnerability will + exist. + "#, + &formats.int_select, + ) + .operands_in(vec![cc, flags, x, y]) + .operands_out(vec![a]) + .other_side_effects(true), + ); + let c = &Operand::new("c", Any).with_doc("Controlling value to test"); ig.push( Inst::new( @@ -2347,6 +2375,18 @@ pub(crate) fn define( .operands_out(vec![a]), ); + ig.push( + Inst::new( + "iabs", + r#" + Integer absolute value with wrapping: `a := |x|`. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + ig.push( Inst::new( "imul", diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/settings.rs b/third_party/rust/cranelift-codegen-meta/src/shared/settings.rs index 3d15387896aa..88db45d9edc7 100644 --- a/third_party/rust/cranelift-codegen-meta/src/shared/settings.rs +++ b/third_party/rust/cranelift-codegen-meta/src/shared/settings.rs @@ -264,5 +264,23 @@ pub(crate) fn define() -> SettingGroup { true, ); + // Spectre options. + + settings.add_bool( + "enable_heap_access_spectre_mitigation", + r#" + Enable Spectre mitigation on heap bounds checks. + + This is a no-op for any heap that needs no bounds checks; e.g., + if the limit is static and the guard region is large enough that + the index cannot reach past it. + + This option is enabled by default because it is highly + recommended for secure sandboxing. The embedder should consider + the security implications carefully before disabling this option. + "#, + true, + ); + settings.build() } diff --git a/third_party/rust/cranelift-codegen/.cargo-checksum.json b/third_party/rust/cranelift-codegen/.cargo-checksum.json index 386c5e1f180f..26891e63f059 100644 --- a/third_party/rust/cranelift-codegen/.cargo-checksum.json +++ b/third_party/rust/cranelift-codegen/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"c13a6985786ada5080bd39cfed568808f428644b9ad951175745051cbbea8fb0","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"e5127227a7db4a8aa92fa6613ed71801025790e696bb41b0323fb7f3c6f7495a","build.rs":"7a075ede3b4100abcf2ba9dfe9cd1851bf9829a1bf399de845c5c1ba1fd9f052","src/abi.rs":"cfc08ffeae7d2333750ba8b0f50a94e79b26bcd81bad997de37609493742d30e","src/binemit/memorysink.rs":"378cf70ad5114a8a056d642b0c2d42b4285fe6902855a43085e30575a67d8a4d","src/binemit/mod.rs":"38ec613f6dd8e4f4ffd05d5ce6c0a65d38f4d5c8ef3f44f69e5e68441e8c729a","src/binemit/relaxation.rs":"e40b254bc30d4aaf7d6390d15071ea31352fc82cd57eb67a6602d53ba4612b67","src/binemit/shrink.rs":"552d64dff3b044bca77f39891d0e39ee619f6ec0669bf9917132c97becea79b0","src/binemit/stackmap.rs":"f22bf4010af22f3eae08a0157d72287873b58d55ddcc90330a8afb88605cad13","src/bitset.rs":"ff667c19a63a6a9bcf06f99a46b6db839323f5d3d5cc5e7b00f1ab77d331fc77","src/cfg_printer.rs":"a4cd85ecb2c403b29dc3fc6c60ee7f97b6ed602e8ba88ec87accb988d009271e","src/constant_hash.rs":"8f53afb38891fbdab8553c7a89e61f0ce30947a22fb24ae84523c9a1ab025c3f","src/context.rs":"f3a82a9fb74706dd4e5f328f7f86e798c56071d1e0884c31ddfbdcef2a28f6dc","src/cursor.rs":"eaa0e4ea65bec30aa79e2fb62e89c713b1adec6eeddd5592237d7fcce47c5fa7","src/dbg.rs":"1898d94cff0975815eb348651702e95c8f2f63886501d3b7043ee75668480472","src/dce.rs":"b98545dbf5c8d0c4a33b2ec6cd905d6371fe843481ad608ff59b4a009fc9be19","src/divconst_magic_numbers.rs":"e7f8f46f3a25ed7155890e9d76c10f5dde9dc5bbcebb623c8166fa415abdd959","src/dominator_tree.rs":"b3a5c7837150390a78ade90807965dfcb8768a7e3ae9ee02f2e4a1ad8f3bafa9","src/flowgraph.rs":"71490e2f7a1ea905758a258b2bebe77f125302f446f9839dd60003fdafaef5fe","src/fx.rs":"8a5d07487906d8316a179e826fcd817a92a4860686256a6fd9d78ba47c63f330","src/inst_predicates.rs":"4d136ccb8699caa139d8f346714def38185138df2f34c156ecf9d136b9c96459","src/ir/builder.rs":"3425957b118a0c40847ef7f473543baef5a24f0c7d8af198955180c8f297ee5a","src/ir/constant.rs":"850f26824ccbb6b4755ddab3ca92dd1f1c9c4efd9793f5b51547014138bcb22d","src/ir/dfg.rs":"e803858c6cef60cec09a82d686a73715d0eb142ac96742023cd6112dcf87073a","src/ir/entities.rs":"f0b49b9c351b94703d6c878b564e53a30010c142c47effaf603ec4ade784686b","src/ir/extfunc.rs":"3a9f9857c11de8490a2bbc25065fb4b734b7e1bc25dca8904960297e0d78a3d0","src/ir/extname.rs":"977b2e380094e99cfdd080112bda709ca61a6d942e1e4c2015d39eaf3ea80f4c","src/ir/function.rs":"3034e03ac2e6bcf300b91c9df03e144550ff6c91ef76b105c48ccbc7a752401b","src/ir/globalvalue.rs":"ce1e65a61c6c47deb9a92899d897aa377fbd7b6c070cddf7fb311667ebbd18bc","src/ir/heap.rs":"a59d3e5901412b53c0b53a8cdf10765ff5921de9c410ae9acea226c89827df3c","src/ir/immediates.rs":"aeebb787b8cee4b30bb890cd45fd90d4618732e7df7f00f0508eb38d0e6e7228","src/ir/instructions.rs":"39570df439988ffb8dca909c50e9617fe8adb24ac2a52dde68e76bf284aeceb6","src/ir/jumptable.rs":"184fa640957e8b0e451dc594e4b348f9aff5fb5b54ab3d856ab19df25f4f3322","src/ir/layout.rs":"2956643a822e67a0c1196f8d3c81af11d0c0122b2d97427ce3ed0c57bb633cbf","src/ir/libcall.rs":"7c9255bdef9a16937d817a3bfd542be4c2734eea3c8b8a870ff411eac2efaad8","src/ir/memflags.rs":"dbcf3798ab66dc764b73fb7f139a621c54cc6bcc683f1f70a33ed7e8c3486bfd","src/ir/mod.rs":"5329068e89f79fe230f0530b14652c35b07ab8d8f4070742a7dde4a3d5dab531","src/ir/progpoint.rs":"a985a7fccde1e985ca24bf2344f16faa4cda7cffb30cb56bf27fabbb5727bc62","src/ir/sourceloc.rs":"67c66adace734645ddf311ecb22e975f20756e07c91e10f698a830cfc2cd08aa","src/ir/stackslot.rs":"5d489d21f17c9992d0862c38c661fab2a19e025a7f29a4417d0336d9bfc5f7f0","src/ir/table.rs":"dcc3b663a989b2b084402b08dc9a0e928dbd052e194a46a1886cc6f0cf1a5f2c","src/ir/trapcode.rs":"71c1f66c3202c966ad58a03bf0d58a236b1bdcc80e48e690925e7332e9299351","src/ir/types.rs":"02c84f7249bc9fa2c19d074213d8d0be487ce012dee33f214fafd428ada0497f","src/ir/valueloc.rs":"fc31e06b2b0f37333829c4658b7f6fa8a717e93e18a5c67a63947e5882bbe1a2","src/isa/aarch64/abi.rs":"f35dabf81b9e45ed15b858fbd4898bdd5649c982ca83830af12b8f39cc1fb532","src/isa/aarch64/inst/args.rs":"2670b9768904f000680e04231592d8a28ea23fa0f4a20a0651865fec0c177f30","src/isa/aarch64/inst/emit.rs":"3c56d881ebd77d78e60c7bd080202b31fc66e871e1b6d0f69388cfc862d9a906","src/isa/aarch64/inst/emit_tests.rs":"fcefbf84edec31b92f184991f3e922ea2c2cea0e44a234a92e46efb125306b2f","src/isa/aarch64/inst/imms.rs":"50121f96c96a63d73252e83cfe92cdb00bad1f9bbce62507f3558da923b9d0a8","src/isa/aarch64/inst/mod.rs":"4be7f5e976d7b72617ef09d5ce212510cbc7388f72c100991780ea97b7717860","src/isa/aarch64/inst/regs.rs":"f0b2d5449c5ed45f0cb6a32956a9fac2a192d7963e886045725102130eb4b648","src/isa/aarch64/lower.rs":"049e1efac3a83610f6d612998ed731c23ac5f7a56581bce46a6bef5c53f9e0c9","src/isa/aarch64/lower_inst.rs":"2376fc1cc361257c132856a4829b0b0cfa31526ae4cf8fccb204c6c85eb13bd4","src/isa/aarch64/mod.rs":"1943690939e630dd6fc79631cd4f43368df7fa70e4247198c5eb2a7a67fcf54b","src/isa/arm32/abi.rs":"59abc42d75445f7f3335b035655083772e98e413ee58a72fc4e224620327b5ea","src/isa/arm32/binemit.rs":"eecd7296c27d2520db8e86dd462833ecf06afd33548b03ae676c02454cdd13c2","src/isa/arm32/enc_tables.rs":"e94b12af802de59484cab1124d2ac8a88d08433f6e1a476724ed0403f4b5967f","src/isa/arm32/mod.rs":"63a47a6def4eecc4527a1bd1207b4f08d533a5e17e79eda245ebc62e3c862185","src/isa/arm32/registers.rs":"100033c78e2a891ad90a2765137bd678f1d11ae4ba61974d1083106fa1324037","src/isa/arm32/settings.rs":"2314460f885c24f9571d640f9737a8e0b7d20ca02bcda1127f878fd3891c0529","src/isa/call_conv.rs":"e4782f3a45d292971ad64c7ca6c8f2f3ffe06003bd4bdc47cb3094979b8383bb","src/isa/constraints.rs":"296f473a95146a743ecb73d8d5908675be02e37607efd287f55322549dc25763","src/isa/enc_tables.rs":"382e714f9500afc292c563cb66d4c963d6787e58f197b1db242db7a099c22b9a","src/isa/encoding.rs":"22e21fdb8e15859c69e4f836bb61e1a82cd6838d093ce5fe641a90f16fb65c9e","src/isa/mod.rs":"baa0eee642b9ee9b17856a234b78e8a5e6ac506131f3f36cbd72db2b86365891","src/isa/registers.rs":"61840d736b1943c3e54ac324db6f7de4f76170800f047dde267dcc9aa2d53e6a","src/isa/riscv/abi.rs":"aa60b701efcef417ee1262a95398343578dc1a30decc8e11044b74d41654ec51","src/isa/riscv/binemit.rs":"264d223da311d4482ebf2f55438b665c67b163058251bc78173c76ba983a31ef","src/isa/riscv/enc_tables.rs":"8491f2082b24c7dedeb7c36cfd913bf9aeaa0a4c8fc754166e9285f4ae002f40","src/isa/riscv/mod.rs":"03ee02848dbc3325a5ef38e66c05be571c380fbe4ca520b4f87c7572db228beb","src/isa/riscv/registers.rs":"6275ec3ef19195d16556c1856cb23095f25a80c31d6b429eaa749d549515a6d3","src/isa/riscv/settings.rs":"e3c063add822ca366e845f5a19d25b56635e828b157a37d29aca5355650e6a66","src/isa/stack.rs":"c391216fb8ee6566e3b14aa4bd83ba85aef2bd23422a9dca2c8c6f47016372e2","src/isa/test_utils.rs":"df889e5a4fe6d1069ca87a34ba6260e077de9b0839db5e5c5780f18983aaeeef","src/isa/unwind.rs":"dab61943d7ca89e099604df2a36dba3b8a661fcf7c10dc13e1257ff8f4890bb3","src/isa/unwind/systemv.rs":"f93c36a366470a54ce2af229eea5ef7c299d28b34093dd656f3c5ad87ea94386","src/isa/unwind/winx64.rs":"78e2356ef8e6c886fde1432e4fd16d3bb860a0990ee69aa25f0fabb108a581e2","src/isa/x64/abi.rs":"4ebf950f44c4db04650c94d89768a0d0dd771ccaefca71da9d400f22267a8542","src/isa/x64/inst/args.rs":"f08a5e0f4ecb0d46f23a39b4eff54d73d3159dc383841d4472f28ef723a1c76f","src/isa/x64/inst/emit.rs":"7b0777eb4e0fc8aea8fa6aa4abc82d207b924ae33c1171a31937d5172e912c32","src/isa/x64/inst/emit_tests.rs":"6ba64a6cb0f0770b0e068b9aa5a5af2802a04eb6d694d57fa8d65618d17c6b3d","src/isa/x64/inst/mod.rs":"9ee792e3c13016984eed92f49e1315a1a93d99f53ed4c12cfb41786c83cd165b","src/isa/x64/inst/regs.rs":"ea43cf121e299688b460118b9772f78c21611e864170d534fa33f2e99d96eba2","src/isa/x64/lower.rs":"43b56255a517e4564a0f341c214cf888b5e7abd0644e6f960c8bdeb3322ef0dd","src/isa/x64/mod.rs":"1b420d38abc0510e6fb2fdb21b7dee80a004cc64d977c47434da687dcc29382d","src/isa/x86/abi.rs":"a5aea2a01bb524279d59b96c81300158140e651def2cbcd3e51900a18f01ebfb","src/isa/x86/binemit.rs":"fb5051471cd9c860455a0c1d74aec7d919d4c7229ada10222631c3694f9a091f","src/isa/x86/enc_tables.rs":"759248ca55f02c0ac43dc631917b182f0074cc1bd5c9b5f5d3bbaae6ffe7d05d","src/isa/x86/mod.rs":"1ec40c5b920d41814799d2d0434682088fcb99526047bce47648f10135867df9","src/isa/x86/registers.rs":"1abbc1aa24c6fc4c7e610b6e840eab29046b821de08cf57fc05e2c2f665479c0","src/isa/x86/settings.rs":"d3e403db3507830f79bcc976c17340b57052cf1b50877fcf1a79549f2a054458","src/isa/x86/unwind.rs":"2fb77f98437c425b3c6438a9e964501568579ca47f74c1a7eba47ab59c7a5046","src/isa/x86/unwind/systemv.rs":"fe3a4e404d2cff5d831a07d82c17bbd3bb539b976a0f9825051208d4520e5403","src/isa/x86/unwind/winx64.rs":"f4df95fa54a995daa04ab588fb7751c4617f8a6f8f210df69fd8be8c6a90b5fe","src/iterators.rs":"d399a80bc65b97f7d650978e19086dcc1d280ac96d60628eac3d17eb363b0d71","src/legalizer/boundary.rs":"9daba50ac9138b076137a3416f48e20584901630456656221ec090a849f32962","src/legalizer/call.rs":"be6074c64c1a00e5e81159dd94c8401fef62205b22c15e07e0c56cf922554d00","src/legalizer/globalvalue.rs":"a5d09ee41a04ba991d7f5d2d8d8c30a209748d38501a005e0ea568df2663cbb5","src/legalizer/heap.rs":"4f6328907e0cb65a0f2e291ada603b9cf669190a4b1ce1fe4156b49a8b48f337","src/legalizer/libcall.rs":"859662cfc49a479a0b0ebad401f806eb067bddbc4be01994131302c2a89a8183","src/legalizer/mod.rs":"0ea23e04f4d26803518f3bb634719a9da2968180b10b09dadb4458bdde59d0e6","src/legalizer/split.rs":"697f08886dbf35fcc69eccc7b597986a58cc73ca7d3cf5d581fffc658a0dad33","src/legalizer/table.rs":"c36d03525312e3191aba8ee00c26a87c1ea200f9a9a0370f0cc84eeacff71786","src/lib.rs":"baac4bced5536908fd80a14a12b9b89bba9c3ea799d939949abdfaa0a8a46ea2","src/licm.rs":"75e94228c37a7d32cc9b0d51644439f9b1837768df67fd3de08ee43b8cdf8123","src/loop_analysis.rs":"4f23c08df9bc95195d68e9721a3a040c6192276ad33894c1225647034f01b23d","src/machinst/abi.rs":"4569b7565fe5057d56fab7b1ff52f30251b9f96e6fc189f4ea0956045325ab4c","src/machinst/adapter.rs":"caf5fcde25b0f41b49d108e3da72f6d9e5dfa97e24d5171698cf9eba1d283399","src/machinst/blockorder.rs":"04387238c1855051a44f8faffb76983514251a031af7d1837224551b8f574b60","src/machinst/buffer.rs":"8e2dd5046ae2f228d73d16a186830736a14f06bbe7ac15389885aa3a8383131a","src/machinst/compile.rs":"b0946fd4d081c5eabb31fdb75da014aa13347203233206a39006b13c25387512","src/machinst/lower.rs":"cccb279e897578d100157c831d9eaad87105e7e10fdb5efa7a90be1eae24cd5b","src/machinst/mod.rs":"ad87e3d377f3990a6d1ff251d6c8d0c9c47f31db33489cc7a4dc2609c7c9dbf4","src/machinst/pretty_print.rs":"6076d9ae0ec48ada9e674ad46c25f48e502adb6b8049c6e1edbcb3b68bd37f87","src/machinst/vcode.rs":"a4465b3c160e28dd005edc2e07dafb37c69d1539c844b183ec2d6f8fb8ee86ee","src/nan_canonicalization.rs":"dd853d0b1c1b5274513e9fb24b8beb8db877aca8fdfea5ceccbd1a80309c8842","src/partition_slice.rs":"861e778f887ea5d0841d730963565690fd8c298edddf08848d0d215ae5125eba","src/peepmatic.rs":"9b9c77486ff912718b501010db459b0a2ecaa3b5a192e32bb95af13c0f109491","src/postopt.rs":"e3374205f6982c5cf2d47ac1e602aba49c85126122ddf98b1823b59c5161dca8","src/predicates.rs":"d4fa993d8b3036ac9e19d1c1d8880ab5b33030fff0a38d65e2a24b9f9d3956c9","src/preopt.peepmatic":"e6344df695715dee3adf535dcfa4e92e0cde2206e91725f8fa481b0729194e9c","src/preopt.serialized":"4c95f34a4f935f310c39764c7dceda3e7a78b37202d7cb101d09783d7f431b5e","src/print_errors.rs":"075b74520f47070b839b43714f55a46a7cc2697b9b0f24a7162d481b7e46b7d2","src/redundant_reload_remover.rs":"17e64c2f253743e9d111bd48709f763a2746cc8cb6a95d2bdd9d7bca605b0621","src/regalloc/affinity.rs":"ec5d688c24043a8aa72efcfbfddc14497cd9bab288c9d339e5c0150cdade2b1d","src/regalloc/branch_splitting.rs":"32e34197f84e30cff758f4db611a9c70dd587dd8d094729c34aa00303538c0d0","src/regalloc/coalescing.rs":"154842e7f380f2626c698283dbc5e0d5e7f0cc5d22b938e90312d17b71a8bb72","src/regalloc/coloring.rs":"ded1d8e531c38412fb19fe746fed65a6b6598819a29cd76c9b4bd5c4d0d6011a","src/regalloc/context.rs":"d85f86a8a79e7c939c0e696d30632ae3856001de75411828fc57c0b5b93e63ef","src/regalloc/diversion.rs":"2e474940b0c38610ca231faba7c7c3cfadf401a2f24247b6f3730ac862fce21f","src/regalloc/live_value_tracker.rs":"845dc3f43cc6b795fea51bf293e7c6ab4961d59ab6ca2670fcab7a2a9bd996be","src/regalloc/liveness.rs":"0b027b8e4444a715af1b93d594a293d2fd430ad06940da05b06a4750535e9652","src/regalloc/liverange.rs":"2e98802e90868051b53ddc8555db0ea98aabc77df7789df2a92627650a43227e","src/regalloc/mod.rs":"50399d6569687a87bf1481304aca42506d946e34465e38ca8093e06485ab5fb6","src/regalloc/pressure.rs":"8408213afe07d4532da699f6604aff111c7061d71b585a84c5ec8db31582314c","src/regalloc/register_set.rs":"c740d10a061c4b8527ce319842b519d743e93e64db53851360f9ca2c099fd652","src/regalloc/reload.rs":"2132bd4cf45ce60b7799277d36bda35c05064ee1c60798388b8f55a0668fca47","src/regalloc/safepoint.rs":"e7538c92bb1c3fc3540ae08532f1782258e06dc5b28a100bda8a10b1e8f606ec","src/regalloc/solver.rs":"5ad745ce9075ae8ca742602411f260036df4598695a4f5f0905bd91efe2c69c9","src/regalloc/spilling.rs":"3b75be8be6568a091dd8a1fd174b099859c6e9969c03bd765b5fb50f52fcccb5","src/regalloc/virtregs.rs":"a01b5d3cb1753e344c6663dd73de00dd452d442990f89477610b22c86c9afdfd","src/remove_constant_phis.rs":"3ee19fa855371c7b113d919b48dbb377ad184646f2ccc2c760f359b4b8d53424","src/result.rs":"7bfaebb740cceb05e27bad841d3ae7af5902f773d15c87e5d4319a0dd97edf07","src/scoped_hash_map.rs":"c8d0071ce7e19438e9995b5fca1ea0fca81234811943b06b344331f9742c3718","src/settings.rs":"fb81ab40b699e3e31b4f80fd40f42a0cb68e5181805894e39f3294c654fafdba","src/simple_gvn.rs":"1de1d0c0e028833350eda7186243f255c9db97fe04f0e6fa688b8a710caa78be","src/simple_preopt.rs":"cac21be7637415f54be27af6135c1cc777352146b47bf25ac8e0b30cf5ab4d44","src/stack_layout.rs":"a2a7118ce32053501357a2f56c506ff4c19937c09856e459ee9d361d74a23d9e","src/timing.rs":"eb6d6688f886cfbe287347c59a5970a11244175a6377c7f2106ced308ab13de7","src/topo_order.rs":"c092ee7a44e5f14962501eafd4478dfb855ce66af15d9c94a9b244ea30d6e991","src/unreachable_code.rs":"baea08a55b1e7eb2379fa2c4bb5ed4f5a536a35eafcb377f8ab79dc41d14d3d4","src/value_label.rs":"e464557e5bab579773929fcfca843e553af201174da1a73460d199446abc7fc7","src/verifier/cssa.rs":"2590b0ecbc134bbedac50915ed9c9e054c89f81e455c7bc0f37d4ddf57a38d05","src/verifier/flags.rs":"233a4c6fb42e32d92bcbef4ec094a26aa79bdd25cb478847236b6ce5d88d3d54","src/verifier/liveness.rs":"b6ab6dfb1390cea8091b71a6f2fd629ee356987b6a0714e8773d7b0eb7fa889f","src/verifier/locations.rs":"2b4e62e1bb79551725414b5a77425c00e9ad56ad766d6293db1eb261b64f51f9","src/verifier/mod.rs":"233e8420b95dba0e261d6a85ebc8b6164b25a505883071ec82544f02c8ae0c07","src/write.rs":"d045e8269cd306c76d45925296517908a9cec3a5b3c736511c87f7bcbe6cde4f"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"c13a6985786ada5080bd39cfed568808f428644b9ad951175745051cbbea8fb0","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"e5127227a7db4a8aa92fa6613ed71801025790e696bb41b0323fb7f3c6f7495a","build.rs":"7a075ede3b4100abcf2ba9dfe9cd1851bf9829a1bf399de845c5c1ba1fd9f052","src/abi.rs":"cfc08ffeae7d2333750ba8b0f50a94e79b26bcd81bad997de37609493742d30e","src/binemit/memorysink.rs":"378cf70ad5114a8a056d642b0c2d42b4285fe6902855a43085e30575a67d8a4d","src/binemit/mod.rs":"38ec613f6dd8e4f4ffd05d5ce6c0a65d38f4d5c8ef3f44f69e5e68441e8c729a","src/binemit/relaxation.rs":"0547f1a33ea176b281d6f09d1bd7a253dcba895785008e61c114861f1a86307f","src/binemit/shrink.rs":"552d64dff3b044bca77f39891d0e39ee619f6ec0669bf9917132c97becea79b0","src/binemit/stackmap.rs":"106e022548c7dbd35c2f6bc2f8bc5131fc0550c7063daa9702f1151ef3f16108","src/bitset.rs":"ff667c19a63a6a9bcf06f99a46b6db839323f5d3d5cc5e7b00f1ab77d331fc77","src/cfg_printer.rs":"a4cd85ecb2c403b29dc3fc6c60ee7f97b6ed602e8ba88ec87accb988d009271e","src/constant_hash.rs":"8f53afb38891fbdab8553c7a89e61f0ce30947a22fb24ae84523c9a1ab025c3f","src/context.rs":"f3a82a9fb74706dd4e5f328f7f86e798c56071d1e0884c31ddfbdcef2a28f6dc","src/cursor.rs":"eaa0e4ea65bec30aa79e2fb62e89c713b1adec6eeddd5592237d7fcce47c5fa7","src/dbg.rs":"1898d94cff0975815eb348651702e95c8f2f63886501d3b7043ee75668480472","src/dce.rs":"b98545dbf5c8d0c4a33b2ec6cd905d6371fe843481ad608ff59b4a009fc9be19","src/divconst_magic_numbers.rs":"e7f8f46f3a25ed7155890e9d76c10f5dde9dc5bbcebb623c8166fa415abdd959","src/dominator_tree.rs":"b3a5c7837150390a78ade90807965dfcb8768a7e3ae9ee02f2e4a1ad8f3bafa9","src/flowgraph.rs":"71490e2f7a1ea905758a258b2bebe77f125302f446f9839dd60003fdafaef5fe","src/fx.rs":"8a5d07487906d8316a179e826fcd817a92a4860686256a6fd9d78ba47c63f330","src/inst_predicates.rs":"4d136ccb8699caa139d8f346714def38185138df2f34c156ecf9d136b9c96459","src/ir/builder.rs":"3425957b118a0c40847ef7f473543baef5a24f0c7d8af198955180c8f297ee5a","src/ir/constant.rs":"850f26824ccbb6b4755ddab3ca92dd1f1c9c4efd9793f5b51547014138bcb22d","src/ir/dfg.rs":"e803858c6cef60cec09a82d686a73715d0eb142ac96742023cd6112dcf87073a","src/ir/entities.rs":"f0b49b9c351b94703d6c878b564e53a30010c142c47effaf603ec4ade784686b","src/ir/extfunc.rs":"3bd956ace9267af4d99b5e947d54a5b2714111ded567853df2165c61276489a3","src/ir/extname.rs":"977b2e380094e99cfdd080112bda709ca61a6d942e1e4c2015d39eaf3ea80f4c","src/ir/function.rs":"3034e03ac2e6bcf300b91c9df03e144550ff6c91ef76b105c48ccbc7a752401b","src/ir/globalvalue.rs":"2f3a54cc2ade91439536b02c46ce790c3634a386a4cc4d27d7da7ad929b8bb90","src/ir/heap.rs":"a59d3e5901412b53c0b53a8cdf10765ff5921de9c410ae9acea226c89827df3c","src/ir/immediates.rs":"aeebb787b8cee4b30bb890cd45fd90d4618732e7df7f00f0508eb38d0e6e7228","src/ir/instructions.rs":"39570df439988ffb8dca909c50e9617fe8adb24ac2a52dde68e76bf284aeceb6","src/ir/jumptable.rs":"184fa640957e8b0e451dc594e4b348f9aff5fb5b54ab3d856ab19df25f4f3322","src/ir/layout.rs":"2956643a822e67a0c1196f8d3c81af11d0c0122b2d97427ce3ed0c57bb633cbf","src/ir/libcall.rs":"7c9255bdef9a16937d817a3bfd542be4c2734eea3c8b8a870ff411eac2efaad8","src/ir/memflags.rs":"dbcf3798ab66dc764b73fb7f139a621c54cc6bcc683f1f70a33ed7e8c3486bfd","src/ir/mod.rs":"5329068e89f79fe230f0530b14652c35b07ab8d8f4070742a7dde4a3d5dab531","src/ir/progpoint.rs":"a985a7fccde1e985ca24bf2344f16faa4cda7cffb30cb56bf27fabbb5727bc62","src/ir/sourceloc.rs":"67c66adace734645ddf311ecb22e975f20756e07c91e10f698a830cfc2cd08aa","src/ir/stackslot.rs":"5d489d21f17c9992d0862c38c661fab2a19e025a7f29a4417d0336d9bfc5f7f0","src/ir/table.rs":"dcc3b663a989b2b084402b08dc9a0e928dbd052e194a46a1886cc6f0cf1a5f2c","src/ir/trapcode.rs":"71c1f66c3202c966ad58a03bf0d58a236b1bdcc80e48e690925e7332e9299351","src/ir/types.rs":"02c84f7249bc9fa2c19d074213d8d0be487ce012dee33f214fafd428ada0497f","src/ir/valueloc.rs":"71356713e1dbb3744363b1585d8d2563f020936e35761c6ca01de2b70abebd0b","src/isa/aarch64/abi.rs":"ab5f835121f8fe88d8f8f7cf9e42fb852ff5a303370d9fc3daa81e166e966d6a","src/isa/aarch64/inst/args.rs":"2670b9768904f000680e04231592d8a28ea23fa0f4a20a0651865fec0c177f30","src/isa/aarch64/inst/emit.rs":"87cd10334ffdb9382fbb09e88e67ee6c707560bf3d06fed771a8a78e9a30ffd5","src/isa/aarch64/inst/emit_tests.rs":"d6497675e8f688b5a128d35d9d8cc8df8e6c93ae0fbe68c8cc4498ccfe58e560","src/isa/aarch64/inst/imms.rs":"bb3cdc63621136befe13e96016bc0ec0edb12137d255f379532892f6b25c5d95","src/isa/aarch64/inst/mod.rs":"c0ac893d0d8c5ad585e23578da0158b2e9537eff299ce375ec2a8b5cbdc46310","src/isa/aarch64/inst/regs.rs":"a495b7fa4a6249fbc7ea16a978490f10e281e00c0979432b2725b9cc6d859acb","src/isa/aarch64/lower.rs":"12c58ec01f8e79f8f2dfe3feb89b5bb546b03632ef17e797114b8dd3f0ede8ec","src/isa/aarch64/lower_inst.rs":"162d54eedc31c8ac94072d5495b06c8aabc695978bf5f82046fdfebbf461fc65","src/isa/aarch64/mod.rs":"1943690939e630dd6fc79631cd4f43368df7fa70e4247198c5eb2a7a67fcf54b","src/isa/arm32/abi.rs":"59abc42d75445f7f3335b035655083772e98e413ee58a72fc4e224620327b5ea","src/isa/arm32/binemit.rs":"eecd7296c27d2520db8e86dd462833ecf06afd33548b03ae676c02454cdd13c2","src/isa/arm32/enc_tables.rs":"e94b12af802de59484cab1124d2ac8a88d08433f6e1a476724ed0403f4b5967f","src/isa/arm32/mod.rs":"63a47a6def4eecc4527a1bd1207b4f08d533a5e17e79eda245ebc62e3c862185","src/isa/arm32/registers.rs":"100033c78e2a891ad90a2765137bd678f1d11ae4ba61974d1083106fa1324037","src/isa/arm32/settings.rs":"2314460f885c24f9571d640f9737a8e0b7d20ca02bcda1127f878fd3891c0529","src/isa/call_conv.rs":"e4782f3a45d292971ad64c7ca6c8f2f3ffe06003bd4bdc47cb3094979b8383bb","src/isa/constraints.rs":"296f473a95146a743ecb73d8d5908675be02e37607efd287f55322549dc25763","src/isa/enc_tables.rs":"382e714f9500afc292c563cb66d4c963d6787e58f197b1db242db7a099c22b9a","src/isa/encoding.rs":"22e21fdb8e15859c69e4f836bb61e1a82cd6838d093ce5fe641a90f16fb65c9e","src/isa/mod.rs":"751d1ba227081cb168347d4400cb004503e054518647617a76fdee663a974e11","src/isa/registers.rs":"61840d736b1943c3e54ac324db6f7de4f76170800f047dde267dcc9aa2d53e6a","src/isa/riscv/abi.rs":"aa60b701efcef417ee1262a95398343578dc1a30decc8e11044b74d41654ec51","src/isa/riscv/binemit.rs":"264d223da311d4482ebf2f55438b665c67b163058251bc78173c76ba983a31ef","src/isa/riscv/enc_tables.rs":"8491f2082b24c7dedeb7c36cfd913bf9aeaa0a4c8fc754166e9285f4ae002f40","src/isa/riscv/mod.rs":"03ee02848dbc3325a5ef38e66c05be571c380fbe4ca520b4f87c7572db228beb","src/isa/riscv/registers.rs":"6275ec3ef19195d16556c1856cb23095f25a80c31d6b429eaa749d549515a6d3","src/isa/riscv/settings.rs":"e3c063add822ca366e845f5a19d25b56635e828b157a37d29aca5355650e6a66","src/isa/stack.rs":"c391216fb8ee6566e3b14aa4bd83ba85aef2bd23422a9dca2c8c6f47016372e2","src/isa/test_utils.rs":"df889e5a4fe6d1069ca87a34ba6260e077de9b0839db5e5c5780f18983aaeeef","src/isa/unwind.rs":"dab61943d7ca89e099604df2a36dba3b8a661fcf7c10dc13e1257ff8f4890bb3","src/isa/unwind/systemv.rs":"f93c36a366470a54ce2af229eea5ef7c299d28b34093dd656f3c5ad87ea94386","src/isa/unwind/winx64.rs":"78e2356ef8e6c886fde1432e4fd16d3bb860a0990ee69aa25f0fabb108a581e2","src/isa/x64/abi.rs":"dc1fc46b13230af1cff207f83d5d8eb6fba794b7d56c6484df44c50b895f24d1","src/isa/x64/inst/args.rs":"410d31eb4e837177e8bc593f2ca6b5b1d322a467676caa6f27d4a5eab45403e3","src/isa/x64/inst/emit.rs":"e34c7a2884e193b7a425ffeb0d12aba7ac30c1b6611d53495f28b8b790ea2852","src/isa/x64/inst/emit_tests.rs":"5b7f446f2129445c5eee0380faa9bfa232779fa75fa31c6d007fdf842e15f462","src/isa/x64/inst/mod.rs":"b7d9db3c139915733b4bafd7c08cbe22813538dea83b032e9f2f6c14624b84c6","src/isa/x64/inst/regs.rs":"ea43cf121e299688b460118b9772f78c21611e864170d534fa33f2e99d96eba2","src/isa/x64/lower.rs":"b5f6c9193191dfa3866900c0ed70157de2b59bda4178125786ddad693afd06f9","src/isa/x64/mod.rs":"bae67e0d50bc7675f35295f5b36641c670fc7ccd34543ca830e3cfaeaf2ec7c1","src/isa/x86/abi.rs":"a5aea2a01bb524279d59b96c81300158140e651def2cbcd3e51900a18f01ebfb","src/isa/x86/binemit.rs":"fb5051471cd9c860455a0c1d74aec7d919d4c7229ada10222631c3694f9a091f","src/isa/x86/enc_tables.rs":"1c2d31a2c445c062e16768dca64a3ada272f8af767429f9b31215f475e7a040c","src/isa/x86/mod.rs":"1ec40c5b920d41814799d2d0434682088fcb99526047bce47648f10135867df9","src/isa/x86/registers.rs":"1abbc1aa24c6fc4c7e610b6e840eab29046b821de08cf57fc05e2c2f665479c0","src/isa/x86/settings.rs":"d3e403db3507830f79bcc976c17340b57052cf1b50877fcf1a79549f2a054458","src/isa/x86/unwind.rs":"2fb77f98437c425b3c6438a9e964501568579ca47f74c1a7eba47ab59c7a5046","src/isa/x86/unwind/systemv.rs":"fe3a4e404d2cff5d831a07d82c17bbd3bb539b976a0f9825051208d4520e5403","src/isa/x86/unwind/winx64.rs":"f4df95fa54a995daa04ab588fb7751c4617f8a6f8f210df69fd8be8c6a90b5fe","src/iterators.rs":"d399a80bc65b97f7d650978e19086dcc1d280ac96d60628eac3d17eb363b0d71","src/legalizer/boundary.rs":"9daba50ac9138b076137a3416f48e20584901630456656221ec090a849f32962","src/legalizer/call.rs":"be6074c64c1a00e5e81159dd94c8401fef62205b22c15e07e0c56cf922554d00","src/legalizer/globalvalue.rs":"a5d09ee41a04ba991d7f5d2d8d8c30a209748d38501a005e0ea568df2663cbb5","src/legalizer/heap.rs":"a6026d44c9ce31e0a21413c50581985dad1584700fde9dbab0b2cefafa5c9d14","src/legalizer/libcall.rs":"859662cfc49a479a0b0ebad401f806eb067bddbc4be01994131302c2a89a8183","src/legalizer/mod.rs":"0ea23e04f4d26803518f3bb634719a9da2968180b10b09dadb4458bdde59d0e6","src/legalizer/split.rs":"697f08886dbf35fcc69eccc7b597986a58cc73ca7d3cf5d581fffc658a0dad33","src/legalizer/table.rs":"c36d03525312e3191aba8ee00c26a87c1ea200f9a9a0370f0cc84eeacff71786","src/lib.rs":"baac4bced5536908fd80a14a12b9b89bba9c3ea799d939949abdfaa0a8a46ea2","src/licm.rs":"75e94228c37a7d32cc9b0d51644439f9b1837768df67fd3de08ee43b8cdf8123","src/loop_analysis.rs":"4f23c08df9bc95195d68e9721a3a040c6192276ad33894c1225647034f01b23d","src/machinst/abi.rs":"4569b7565fe5057d56fab7b1ff52f30251b9f96e6fc189f4ea0956045325ab4c","src/machinst/adapter.rs":"caf5fcde25b0f41b49d108e3da72f6d9e5dfa97e24d5171698cf9eba1d283399","src/machinst/blockorder.rs":"04387238c1855051a44f8faffb76983514251a031af7d1837224551b8f574b60","src/machinst/buffer.rs":"de9593aa7716107c7589ac24278d4b6628ba182aac2e2b4c551fc80bfb951c18","src/machinst/compile.rs":"b0946fd4d081c5eabb31fdb75da014aa13347203233206a39006b13c25387512","src/machinst/lower.rs":"8fc7d0307973cb618a59e7f973c4f24f832220e89b64e4924bf345c5dfcbcc57","src/machinst/mod.rs":"ad87e3d377f3990a6d1ff251d6c8d0c9c47f31db33489cc7a4dc2609c7c9dbf4","src/machinst/pretty_print.rs":"6076d9ae0ec48ada9e674ad46c25f48e502adb6b8049c6e1edbcb3b68bd37f87","src/machinst/vcode.rs":"1f876b369a21936488e3da43879c40b2d50c051f7cffb659e71df39872ae2a86","src/nan_canonicalization.rs":"dd853d0b1c1b5274513e9fb24b8beb8db877aca8fdfea5ceccbd1a80309c8842","src/partition_slice.rs":"861e778f887ea5d0841d730963565690fd8c298edddf08848d0d215ae5125eba","src/peepmatic.rs":"9b9c77486ff912718b501010db459b0a2ecaa3b5a192e32bb95af13c0f109491","src/postopt.rs":"ab74e2811909805d467d470da5e66879328c8f47db263422efedf3f1c449d8b2","src/predicates.rs":"d4fa993d8b3036ac9e19d1c1d8880ab5b33030fff0a38d65e2a24b9f9d3956c9","src/preopt.peepmatic":"e6344df695715dee3adf535dcfa4e92e0cde2206e91725f8fa481b0729194e9c","src/preopt.serialized":"3c758aa8aa03843bec0f32468fff00941ab56730842a5c928b1d182bb79a18ec","src/print_errors.rs":"075b74520f47070b839b43714f55a46a7cc2697b9b0f24a7162d481b7e46b7d2","src/redundant_reload_remover.rs":"2c72cc013f33e1257425592ef4ee2b9437ab3dc84d9759589c15fd217bde83a2","src/regalloc/affinity.rs":"ec5d688c24043a8aa72efcfbfddc14497cd9bab288c9d339e5c0150cdade2b1d","src/regalloc/branch_splitting.rs":"32e34197f84e30cff758f4db611a9c70dd587dd8d094729c34aa00303538c0d0","src/regalloc/coalescing.rs":"154842e7f380f2626c698283dbc5e0d5e7f0cc5d22b938e90312d17b71a8bb72","src/regalloc/coloring.rs":"ded1d8e531c38412fb19fe746fed65a6b6598819a29cd76c9b4bd5c4d0d6011a","src/regalloc/context.rs":"d85f86a8a79e7c939c0e696d30632ae3856001de75411828fc57c0b5b93e63ef","src/regalloc/diversion.rs":"2e474940b0c38610ca231faba7c7c3cfadf401a2f24247b6f3730ac862fce21f","src/regalloc/live_value_tracker.rs":"845dc3f43cc6b795fea51bf293e7c6ab4961d59ab6ca2670fcab7a2a9bd996be","src/regalloc/liveness.rs":"0b027b8e4444a715af1b93d594a293d2fd430ad06940da05b06a4750535e9652","src/regalloc/liverange.rs":"2e98802e90868051b53ddc8555db0ea98aabc77df7789df2a92627650a43227e","src/regalloc/mod.rs":"50399d6569687a87bf1481304aca42506d946e34465e38ca8093e06485ab5fb6","src/regalloc/pressure.rs":"8408213afe07d4532da699f6604aff111c7061d71b585a84c5ec8db31582314c","src/regalloc/register_set.rs":"c740d10a061c4b8527ce319842b519d743e93e64db53851360f9ca2c099fd652","src/regalloc/reload.rs":"2132bd4cf45ce60b7799277d36bda35c05064ee1c60798388b8f55a0668fca47","src/regalloc/safepoint.rs":"e7538c92bb1c3fc3540ae08532f1782258e06dc5b28a100bda8a10b1e8f606ec","src/regalloc/solver.rs":"5ad745ce9075ae8ca742602411f260036df4598695a4f5f0905bd91efe2c69c9","src/regalloc/spilling.rs":"3b75be8be6568a091dd8a1fd174b099859c6e9969c03bd765b5fb50f52fcccb5","src/regalloc/virtregs.rs":"a01b5d3cb1753e344c6663dd73de00dd452d442990f89477610b22c86c9afdfd","src/remove_constant_phis.rs":"3ee19fa855371c7b113d919b48dbb377ad184646f2ccc2c760f359b4b8d53424","src/result.rs":"7164a1b35f26aeb9a6eda79f773f64ecb97b80b50f5b01ea5d34b64361160cbd","src/scoped_hash_map.rs":"c8d0071ce7e19438e9995b5fca1ea0fca81234811943b06b344331f9742c3718","src/settings.rs":"ec3ae1db27cc2f307f27729bd0bf620053442c10a31ff70a917abf6b0ae76514","src/simple_gvn.rs":"1de1d0c0e028833350eda7186243f255c9db97fe04f0e6fa688b8a710caa78be","src/simple_preopt.rs":"cac21be7637415f54be27af6135c1cc777352146b47bf25ac8e0b30cf5ab4d44","src/stack_layout.rs":"a2a7118ce32053501357a2f56c506ff4c19937c09856e459ee9d361d74a23d9e","src/timing.rs":"eb6d6688f886cfbe287347c59a5970a11244175a6377c7f2106ced308ab13de7","src/topo_order.rs":"c092ee7a44e5f14962501eafd4478dfb855ce66af15d9c94a9b244ea30d6e991","src/unreachable_code.rs":"baea08a55b1e7eb2379fa2c4bb5ed4f5a536a35eafcb377f8ab79dc41d14d3d4","src/value_label.rs":"e464557e5bab579773929fcfca843e553af201174da1a73460d199446abc7fc7","src/verifier/cssa.rs":"2590b0ecbc134bbedac50915ed9c9e054c89f81e455c7bc0f37d4ddf57a38d05","src/verifier/flags.rs":"233a4c6fb42e32d92bcbef4ec094a26aa79bdd25cb478847236b6ce5d88d3d54","src/verifier/liveness.rs":"b6ab6dfb1390cea8091b71a6f2fd629ee356987b6a0714e8773d7b0eb7fa889f","src/verifier/locations.rs":"2b4e62e1bb79551725414b5a77425c00e9ad56ad766d6293db1eb261b64f51f9","src/verifier/mod.rs":"233e8420b95dba0e261d6a85ebc8b6164b25a505883071ec82544f02c8ae0c07","src/write.rs":"d045e8269cd306c76d45925296517908a9cec3a5b3c736511c87f7bcbe6cde4f"},"package":null} \ No newline at end of file diff --git a/third_party/rust/cranelift-codegen/src/binemit/relaxation.rs b/third_party/rust/cranelift-codegen/src/binemit/relaxation.rs index 554d9afa0040..142e400985dc 100644 --- a/third_party/rust/cranelift-codegen/src/binemit/relaxation.rs +++ b/third_party/rust/cranelift-codegen/src/binemit/relaxation.rs @@ -302,7 +302,11 @@ fn fallthroughs(func: &mut Function) { Opcode::Fallthrough => { // Somebody used a fall-through instruction before the branch relaxation pass. // Make sure it is correct, i.e. the destination is the layout successor. - debug_assert_eq!(destination, succ, "Illegal fall-through in {}", block) + debug_assert_eq!( + destination, succ, + "Illegal fallthrough from {} to {}, but {}'s successor is {}", + block, destination, block, succ + ) } Opcode::Jump => { // If this is a jump to the successor block, change it to a fall-through. diff --git a/third_party/rust/cranelift-codegen/src/binemit/stackmap.rs b/third_party/rust/cranelift-codegen/src/binemit/stackmap.rs index 4db9b929293c..45716dcae0e6 100644 --- a/third_party/rust/cranelift-codegen/src/binemit/stackmap.rs +++ b/third_party/rust/cranelift-codegen/src/binemit/stackmap.rs @@ -103,7 +103,7 @@ impl Stackmap { // Refer to the doc comment for `Stackmap` above to understand the // bitmap representation used here. - let map_size = (dbg!(info.frame_size) + dbg!(info.inbound_args_size)) as usize; + let map_size = (info.frame_size + info.inbound_args_size) as usize; let word_size = isa.pointer_bytes() as usize; let num_words = map_size / word_size; diff --git a/third_party/rust/cranelift-codegen/src/ir/extfunc.rs b/third_party/rust/cranelift-codegen/src/ir/extfunc.rs index e2f2bb984bf7..9bb089fd5606 100644 --- a/third_party/rust/cranelift-codegen/src/ir/extfunc.rs +++ b/third_party/rust/cranelift-codegen/src/ir/extfunc.rs @@ -383,7 +383,7 @@ pub struct ExtFuncData { /// flag is best used when the target is known to be in the same unit of code generation, such /// as a Wasm module. /// - /// See the documentation for [`RelocDistance`](machinst::RelocDistance) for more details. A + /// See the documentation for [`RelocDistance`](crate::machinst::RelocDistance) for more details. A /// `colocated` flag value of `true` implies `RelocDistance::Near`. pub colocated: bool, } diff --git a/third_party/rust/cranelift-codegen/src/ir/globalvalue.rs b/third_party/rust/cranelift-codegen/src/ir/globalvalue.rs index dae869445cf0..6c9b2d7bcf82 100644 --- a/third_party/rust/cranelift-codegen/src/ir/globalvalue.rs +++ b/third_party/rust/cranelift-codegen/src/ir/globalvalue.rs @@ -66,7 +66,7 @@ pub enum GlobalValueData { /// /// If `true`, some backends may use relocation forms that have limited range: for example, /// a +/- 2^27-byte range on AArch64. See the documentation for - /// [`RelocDistance`](machinst::RelocDistance) for more details. + /// [`RelocDistance`](crate::machinst::RelocDistance) for more details. colocated: bool, /// Does this symbol refer to a thread local storage value? diff --git a/third_party/rust/cranelift-codegen/src/ir/valueloc.rs b/third_party/rust/cranelift-codegen/src/ir/valueloc.rs index 9d81a5538174..6fbe7e0195a1 100644 --- a/third_party/rust/cranelift-codegen/src/ir/valueloc.rs +++ b/third_party/rust/cranelift-codegen/src/ir/valueloc.rs @@ -41,7 +41,7 @@ impl ValueLoc { pub fn unwrap_reg(self) -> RegUnit { match self { Self::Reg(ru) => ru, - _ => panic!("Expected register: {:?}", self), + _ => panic!("unwrap_reg expected register, found {:?}", self), } } @@ -49,7 +49,7 @@ impl ValueLoc { pub fn unwrap_stack(self) -> StackSlot { match self { Self::Stack(ss) => ss, - _ => panic!("Expected stack slot: {:?}", self), + _ => panic!("unwrap_stack expected stack slot, found {:?}", self), } } diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/abi.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/abi.rs index ed12ed036548..9a4f64896eee 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/abi.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/abi.rs @@ -86,9 +86,9 @@ //! formal arguments, would: //! - Accept a pointer P to the struct return area in x0 on entry. //! - Return v3 in x0. -//! - Return v2 in memory at [P]. -//! - Return v1 in memory at [P+8]. -//! - Return v0 in memory at [P+16]. +//! - Return v2 in memory at `[P]`. +//! - Return v1 in memory at `[P+8]`. +//! - Return v0 in memory at `[P+16]`. use crate::ir; use crate::ir::types; diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/emit.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/emit.rs index 263241835f63..abb9aa00454a 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/emit.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/emit.rs @@ -361,6 +361,20 @@ fn enc_vec_rr_misc(bits_12_16: u32, rd: Writable, rn: Reg) -> u32 { bits | bits_12_16 << 12 | machreg_to_vec(rn) << 5 | machreg_to_vec(rd.to_reg()) } +fn enc_vec_lanes(q: u32, u: u32, size: u32, opcode: u32, rd: Writable, rn: Reg) -> u32 { + debug_assert_eq!(q & 0b1, q); + debug_assert_eq!(u & 0b1, u); + debug_assert_eq!(size & 0b11, size); + debug_assert_eq!(opcode & 0b11111, opcode); + 0b0_0_0_01110_00_11000_0_0000_10_00000_00000 + | q << 30 + | u << 29 + | size << 22 + | opcode << 12 + | machreg_to_vec(rn) << 5 + | machreg_to_vec(rd.to_reg()) +} + /// State carried between emissions of a sequence of instructions. #[derive(Default, Clone, Debug)] pub struct EmitState { @@ -1061,6 +1075,18 @@ impl MachInstEmit for Inst { }; sink.put4(enc_vec_rr_misc(bits_12_16, rd, rn)); } + &Inst::VecLanes { op, rd, rn, ty } => { + let (q, size) = match ty { + I8X16 => (0b1, 0b00), + I16X8 => (0b1, 0b01), + I32X4 => (0b1, 0b10), + _ => unreachable!(), + }; + let (u, opcode) = match op { + VecLanesOp::Uminv => (0b1, 0b11010), + }; + sink.put4(enc_vec_lanes(q, u, size, opcode, rd, rn)); + } &Inst::FpuCmp32 { rn, rm } => { sink.put4(enc_fcmp(InstSize::Size32, rn, rm)); } @@ -1247,12 +1273,17 @@ impl MachInstEmit for Inst { alu_op, ty, } => { - let enc_size_for_cmp = match ty { + let enc_size = match ty { I8X16 => 0b00, I16X8 => 0b01, I32X4 => 0b10, _ => 0, }; + let enc_size_for_fcmp = match ty { + F32X4 => 0b0, + F64X2 => 0b1, + _ => 0, + }; let (top11, bit15_10) = match alu_op { VecALUOp::SQAddScalar => { @@ -1271,12 +1302,15 @@ impl MachInstEmit for Inst { debug_assert_eq!(I64, ty); (0b011_11110_11_1, 0b001011) } - VecALUOp::Cmeq => (0b011_01110_00_1 | enc_size_for_cmp << 1, 0b100011), - VecALUOp::Cmge => (0b010_01110_00_1 | enc_size_for_cmp << 1, 0b001111), - VecALUOp::Cmgt => (0b010_01110_00_1 | enc_size_for_cmp << 1, 0b001101), - VecALUOp::Cmhi => (0b011_01110_00_1 | enc_size_for_cmp << 1, 0b001101), - VecALUOp::Cmhs => (0b011_01110_00_1 | enc_size_for_cmp << 1, 0b001111), - // The following instructions operate on bytes, so are not encoded differently + VecALUOp::Cmeq => (0b011_01110_00_1 | enc_size << 1, 0b100011), + VecALUOp::Cmge => (0b010_01110_00_1 | enc_size << 1, 0b001111), + VecALUOp::Cmgt => (0b010_01110_00_1 | enc_size << 1, 0b001101), + VecALUOp::Cmhi => (0b011_01110_00_1 | enc_size << 1, 0b001101), + VecALUOp::Cmhs => (0b011_01110_00_1 | enc_size << 1, 0b001111), + VecALUOp::Fcmeq => (0b010_01110_00_1 | enc_size_for_fcmp << 1, 0b111001), + VecALUOp::Fcmgt => (0b011_01110_10_1 | enc_size_for_fcmp << 1, 0b111001), + VecALUOp::Fcmge => (0b011_01110_00_1 | enc_size_for_fcmp << 1, 0b111001), + // The following logical instructions operate on bytes, so are not encoded differently // for the different vector types. VecALUOp::And => { debug_assert_eq!(128, ty_bits(ty)); @@ -1298,6 +1332,7 @@ impl MachInstEmit for Inst { debug_assert_eq!(128, ty_bits(ty)); (0b011_01110_01_1, 0b000111) } + VecALUOp::Umaxp => (0b011_01110_00_1 | enc_size << 1, 0b101001), }; sink.put4(enc_vec_rrr(top11, rm, bit15_10, rn, rd)); } diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/emit_tests.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/emit_tests.rs index 7b2c0950356e..aaf4cfbae395 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/emit_tests.rs @@ -2209,6 +2209,42 @@ fn test_aarch64_binemit() { "cmhs v8.4s, v2.4s, v15.4s", )); + insns.push(( + Inst::VecRRR { + alu_op: VecALUOp::Fcmeq, + rd: writable_vreg(28), + rn: vreg(12), + rm: vreg(4), + ty: F32X4, + }, + "9CE5244E", + "fcmeq v28.4s, v12.4s, v4.4s", + )); + + insns.push(( + Inst::VecRRR { + alu_op: VecALUOp::Fcmgt, + rd: writable_vreg(3), + rn: vreg(16), + rm: vreg(31), + ty: F64X2, + }, + "03E6FF6E", + "fcmgt v3.2d, v16.2d, v31.2d", + )); + + insns.push(( + Inst::VecRRR { + alu_op: VecALUOp::Fcmge, + rd: writable_vreg(18), + rn: vreg(23), + rm: vreg(0), + ty: F64X2, + }, + "F2E6606E", + "fcmge v18.2d, v23.2d, v0.2d", + )); + insns.push(( Inst::VecRRR { alu_op: VecALUOp::And, @@ -2269,6 +2305,42 @@ fn test_aarch64_binemit() { "bsl v8.16b, v9.16b, v1.16b", )); + insns.push(( + Inst::VecRRR { + alu_op: VecALUOp::Umaxp, + rd: writable_vreg(8), + rn: vreg(12), + rm: vreg(1), + ty: I8X16, + }, + "88A5216E", + "umaxp v8.16b, v12.16b, v1.16b", + )); + + insns.push(( + Inst::VecRRR { + alu_op: VecALUOp::Umaxp, + rd: writable_vreg(1), + rn: vreg(6), + rm: vreg(1), + ty: I16X8, + }, + "C1A4616E", + "umaxp v1.8h, v6.8h, v1.8h", + )); + + insns.push(( + Inst::VecRRR { + alu_op: VecALUOp::Umaxp, + rd: writable_vreg(1), + rn: vreg(20), + rm: vreg(16), + ty: I32X4, + }, + "81A6B06E", + "umaxp v1.4s, v20.4s, v16.4s", + )); + insns.push(( Inst::VecMisc { op: VecMisc2::Not, @@ -2280,6 +2352,39 @@ fn test_aarch64_binemit() { "mvn v2.16b, v1.16b", )); + insns.push(( + Inst::VecLanes { + op: VecLanesOp::Uminv, + rd: writable_vreg(2), + rn: vreg(1), + ty: I8X16, + }, + "22A8316E", + "uminv b2, v1.16b", + )); + + insns.push(( + Inst::VecLanes { + op: VecLanesOp::Uminv, + rd: writable_vreg(3), + rn: vreg(11), + ty: I16X8, + }, + "63A9716E", + "uminv h3, v11.8h", + )); + + insns.push(( + Inst::VecLanes { + op: VecLanesOp::Uminv, + rd: writable_vreg(18), + rn: vreg(4), + ty: I32X4, + }, + "92A8B16E", + "uminv s18, v4.4s", + )); + insns.push(( Inst::Extend { rd: writable_xreg(1), diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/imms.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/imms.rs index 6fea5efb5c89..961559cc9f02 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/imms.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/imms.rs @@ -304,6 +304,14 @@ impl Imm12 { } } + /// Create a zero immediate of this format. + pub fn zero() -> Self { + Imm12 { + bits: 0, + shift12: false, + } + } + /// Bits for 2-bit "shift" field in e.g. AddI. pub fn shift_bits(&self) -> u32 { if self.shift12 { diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/mod.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/mod.rs index 6c5eb4d99505..1a5563d62a84 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/mod.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/mod.rs @@ -225,6 +225,12 @@ pub enum VecALUOp { Cmhs, /// Compare unsigned higher or same Cmhi, + /// Floating-point compare equal + Fcmeq, + /// Floating-point compare greater than + Fcmgt, + /// Floating-point compare greater than or equal + Fcmge, /// Bitwise and And, /// Bitwise bit clear @@ -235,6 +241,8 @@ pub enum VecALUOp { Eor, /// Bitwise select Bsl, + /// Unsigned maximum pairwise + Umaxp, } /// A Vector miscellaneous operation with two registers. @@ -244,6 +252,13 @@ pub enum VecMisc2 { Not, } +/// An operation across the lanes of vectors. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum VecLanesOp { + /// Unsigned minimum across a vector + Uminv, +} + /// An operation on the bits of a register. This can be paired with several instruction formats /// below (see `Inst`) in any combination. #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] @@ -743,6 +758,14 @@ pub enum Inst { ty: Type, }, + /// Vector instruction across lanes. + VecLanes { + op: VecLanesOp, + rd: Writable, + rn: Reg, + ty: Type, + }, + /// Move to the NZCV flags (actually a `MSR NZCV, Xn` insn). MovToNZCV { rn: Reg, @@ -876,7 +899,7 @@ pub enum Inst { }, /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This - /// controls MemArg::NominalSPOffset args are lowered. + /// controls how MemArg::NominalSPOffset args are lowered. VirtualSPOffsetAdj { offset: i64, }, @@ -1214,6 +1237,11 @@ fn aarch64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_def(rd); collector.add_use(rn); } + + &Inst::VecLanes { rd, rn, .. } => { + collector.add_def(rd); + collector.add_use(rn); + } &Inst::FpuCmp32 { rn, rm } | &Inst::FpuCmp64 { rn, rm } => { collector.add_use(rn); collector.add_use(rm); @@ -1708,6 +1736,14 @@ fn aarch64_map_regs(inst: &mut Inst, mapper: &RUM) { map_def(mapper, rd); map_use(mapper, rn); } + &mut Inst::VecLanes { + ref mut rd, + ref mut rn, + .. + } => { + map_def(mapper, rd); + map_use(mapper, rn); + } &mut Inst::FpuCmp32 { ref mut rn, ref mut rm, @@ -2055,7 +2091,9 @@ impl MachInst for Inst { I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 => Ok(RegClass::I64), F32 | F64 => Ok(RegClass::V128), IFLAGS | FFLAGS => Ok(RegClass::I64), - B8X16 | I8X16 | B16X8 | I16X8 | B32X4 | I32X4 | B64X2 | I64X2 => Ok(RegClass::V128), + B8X16 | I8X16 | B16X8 | I16X8 | B32X4 | I32X4 | B64X2 | I64X2 | F32X4 | F64X2 => { + Ok(RegClass::V128) + } _ => Err(CodegenError::Unsupported(format!( "Unexpected SSA-value type: {}", ty @@ -2482,7 +2520,7 @@ impl ShowWithRRU for Inst { let show_vreg_fn: fn(Reg, Option<&RealRegUniverse>) -> String = if vector { |reg, mb_rru| show_vreg_vector(reg, mb_rru, F32X2) } else { - show_vreg_scalar + |reg, mb_rru| show_vreg_scalar(reg, mb_rru, F64) }; let rd = show_vreg_fn(rd.to_reg(), mb_rru); let rn = show_vreg_fn(rn, mb_rru); @@ -2690,17 +2728,21 @@ impl ShowWithRRU for Inst { VecALUOp::Cmgt => ("cmgt", true, ty), VecALUOp::Cmhs => ("cmhs", true, ty), VecALUOp::Cmhi => ("cmhi", true, ty), + VecALUOp::Fcmeq => ("fcmeq", true, ty), + VecALUOp::Fcmgt => ("fcmgt", true, ty), + VecALUOp::Fcmge => ("fcmge", true, ty), VecALUOp::And => ("and", true, I8X16), VecALUOp::Bic => ("bic", true, I8X16), VecALUOp::Orr => ("orr", true, I8X16), VecALUOp::Eor => ("eor", true, I8X16), VecALUOp::Bsl => ("bsl", true, I8X16), + VecALUOp::Umaxp => ("umaxp", true, ty), }; let show_vreg_fn: fn(Reg, Option<&RealRegUniverse>, Type) -> String = if vector { |reg, mb_rru, ty| show_vreg_vector(reg, mb_rru, ty) } else { - |reg, mb_rru, _ty| show_vreg_scalar(reg, mb_rru) + |reg, mb_rru, _ty| show_vreg_scalar(reg, mb_rru, I64) }; let rd = show_vreg_fn(rd.to_reg(), mb_rru, ty); @@ -2722,6 +2764,15 @@ impl ShowWithRRU for Inst { let rn = show_vreg_vector(rn, mb_rru, ty); format!("{} {}, {}", op, rd, rn) } + &Inst::VecLanes { op, rd, rn, ty } => { + let op = match op { + VecLanesOp::Uminv => "uminv", + }; + + let rd = show_vreg_scalar(rd.to_reg(), mb_rru, ty); + let rn = show_vreg_vector(rn, mb_rru, ty); + format!("{} {}, {}", op, rd, rn) + } &Inst::MovToNZCV { rn } => { let rn = rn.show_rru(mb_rru); format!("msr nzcv, {}", rn) diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/regs.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/regs.rs index 9d746612567d..b92b0b70c9b5 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/regs.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/inst/regs.rs @@ -292,7 +292,7 @@ pub fn show_freg_sized(reg: Reg, mb_rru: Option<&RealRegUniverse>, size: InstSiz } /// Show a vector register used in a scalar context. -pub fn show_vreg_scalar(reg: Reg, mb_rru: Option<&RealRegUniverse>) -> String { +pub fn show_vreg_scalar(reg: Reg, mb_rru: Option<&RealRegUniverse>, ty: Type) -> String { let mut s = reg.show_rru(mb_rru); if reg.get_class() != RegClass::V128 { // We can't do any better. @@ -302,7 +302,14 @@ pub fn show_vreg_scalar(reg: Reg, mb_rru: Option<&RealRegUniverse>) -> String { if reg.is_real() { // Change (eg) "v0" into "d0". if reg.get_class() == RegClass::V128 && s.starts_with("v") { - s.replace_range(0..1, "d"); + let replacement = match ty { + I64 | F64 => "d", + I8X16 => "b", + I16X8 => "h", + I32X4 => "s", + _ => unimplemented!(), + }; + s.replace_range(0..1, replacement); } } else { // Add a "d" suffix to RegClass::V128 vregs. diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/lower.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/lower.rs index d1526c2ae966..1da3d4132895 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/lower.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/lower.rs @@ -14,7 +14,7 @@ use crate::ir::Inst as IRInst; use crate::ir::{InstructionData, Opcode, TrapCode, Type}; use crate::machinst::lower::*; use crate::machinst::*; -use crate::CodegenResult; +use crate::{CodegenError, CodegenResult}; use crate::isa::aarch64::inst::*; use crate::isa::aarch64::AArch64Backend; @@ -168,7 +168,7 @@ pub(crate) fn output_to_const_f128>( } /// How to handle narrow values loaded into registers; see note on `narrow_mode` -/// parameter to `input_to_*` below. +/// parameter to `put_input_in_*` below. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum NarrowValueMode { None, @@ -193,7 +193,7 @@ impl NarrowValueMode { } /// Allocate a register for an instruction output and return it. -pub(crate) fn output_to_reg>(ctx: &mut C, out: InsnOutput) -> Writable { +pub(crate) fn get_output_reg>(ctx: &mut C, out: InsnOutput) -> Writable { ctx.get_output(out.insn, out.output) } @@ -202,12 +202,12 @@ pub(crate) fn output_to_reg>(ctx: &mut C, out: InsnOutput) /// The given register will be extended appropriately, according to /// `narrow_mode` and the input's type. If extended, the value is /// always extended to 64 bits, for simplicity. -pub(crate) fn input_to_reg>( +pub(crate) fn put_input_in_reg>( ctx: &mut C, input: InsnInput, narrow_mode: NarrowValueMode, ) -> Reg { - debug!("input_to_reg: input {:?}", input); + debug!("put_input_in_reg: input {:?}", input); let ty = ctx.input_ty(input.insn, input.input); let from_bits = ty_bits(ty) as u8; let inputs = ctx.get_input(input.insn, input.input); @@ -302,7 +302,7 @@ pub(crate) fn input_to_reg>( /// divide or a right-shift or a compare-to-zero), `narrow_mode` should be /// set to `ZeroExtend` or `SignExtend` as appropriate, and the resulting /// register will be provided the extended value. -fn input_to_rs>( +fn put_input_in_rs>( ctx: &mut C, input: InsnInput, narrow_mode: NarrowValueMode, @@ -317,21 +317,21 @@ fn input_to_rs>( // Can we get the shift amount as an immediate? if let Some(shiftimm) = input_to_shiftimm(ctx, shift_amt) { - let reg = input_to_reg(ctx, shiftee, narrow_mode); + let reg = put_input_in_reg(ctx, shiftee, narrow_mode); return ResultRS::RegShift(reg, ShiftOpAndAmt::new(ShiftOp::LSL, shiftimm)); } } } - ResultRS::Reg(input_to_reg(ctx, input, narrow_mode)) + ResultRS::Reg(put_input_in_reg(ctx, input, narrow_mode)) } /// Lower an instruction input to a reg or reg/shift, or reg/extend operand. /// This does not actually codegen the source instruction; it just uses the /// vreg into which the source instruction will generate its value. /// -/// See note on `input_to_rs` for a description of `narrow_mode`. -fn input_to_rse>( +/// See note on `put_input_in_rs` for a description of `narrow_mode`. +fn put_input_in_rse>( ctx: &mut C, input: InsnInput, narrow_mode: NarrowValueMode, @@ -349,7 +349,7 @@ fn input_to_rse>( && ((narrow_mode.is_32bit() && out_bits < 32) || (!narrow_mode.is_32bit() && out_bits < 64)) { - let reg = input_to_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None); + let reg = put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None); let extendop = match (narrow_mode, out_bits) { (NarrowValueMode::SignExtend32, 1) | (NarrowValueMode::SignExtend64, 1) => { ExtendOp::SXTB @@ -394,15 +394,15 @@ fn input_to_rse>( (false, 32) => ExtendOp::UXTW, _ => unreachable!(), }; - let reg = input_to_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None); + let reg = put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None); return ResultRSE::RegExtend(reg, extendop); } } - ResultRSE::from_rs(input_to_rs(ctx, input, narrow_mode)) + ResultRSE::from_rs(put_input_in_rs(ctx, input, narrow_mode)) } -pub(crate) fn input_to_rse_imm12>( +pub(crate) fn put_input_in_rse_imm12>( ctx: &mut C, input: InsnInput, narrow_mode: NarrowValueMode, @@ -413,10 +413,10 @@ pub(crate) fn input_to_rse_imm12>( } } - ResultRSEImm12::from_rse(input_to_rse(ctx, input, narrow_mode)) + ResultRSEImm12::from_rse(put_input_in_rse(ctx, input, narrow_mode)) } -pub(crate) fn input_to_rs_immlogic>( +pub(crate) fn put_input_in_rs_immlogic>( ctx: &mut C, input: InsnInput, narrow_mode: NarrowValueMode, @@ -429,20 +429,22 @@ pub(crate) fn input_to_rs_immlogic>( } } - ResultRSImmLogic::from_rs(input_to_rs(ctx, input, narrow_mode)) + ResultRSImmLogic::from_rs(put_input_in_rs(ctx, input, narrow_mode)) } -pub(crate) fn input_to_reg_immshift>( +pub(crate) fn put_input_in_reg_immshift>( ctx: &mut C, input: InsnInput, + shift_width_bits: usize, ) -> ResultRegImmShift { if let Some(imm_value) = input_to_const(ctx, input) { + let imm_value = imm_value & ((shift_width_bits - 1) as u64); if let Some(immshift) = ImmShift::maybe_from_u64(imm_value) { return ResultRegImmShift::ImmShift(immshift); } } - ResultRegImmShift::Reg(input_to_reg(ctx, input, NarrowValueMode::None)) + ResultRegImmShift::Reg(put_input_in_reg(ctx, input, NarrowValueMode::None)) } //============================================================================ @@ -546,7 +548,7 @@ pub(crate) fn lower_address>( // Handle one reg and offset. if addends.len() == 1 { - let reg = input_to_reg(ctx, addends[0], NarrowValueMode::ZeroExtend64); + let reg = put_input_in_reg(ctx, addends[0], NarrowValueMode::ZeroExtend64); return MemArg::RegOffset(reg, offset as i64, elem_ty); } @@ -560,9 +562,9 @@ pub(crate) fn lower_address>( maybe_input_insn_multi(ctx, addends[i], &[Opcode::Uextend, Opcode::Sextend]) { // Non-extended addend. - let r1 = input_to_reg(ctx, addends[1 - i], NarrowValueMode::ZeroExtend64); + let r1 = put_input_in_reg(ctx, addends[1 - i], NarrowValueMode::ZeroExtend64); // Extended addend. - let r2 = input_to_reg( + let r2 = put_input_in_reg( ctx, InsnInput { insn: ext_insn, @@ -596,8 +598,8 @@ pub(crate) fn lower_address>( // Handle two regs and a zero offset in the general case, if possible. if addends.len() == 2 && offset == 0 { - let ra = input_to_reg(ctx, addends[0], NarrowValueMode::ZeroExtend64); - let rb = input_to_reg(ctx, addends[1], NarrowValueMode::ZeroExtend64); + let ra = put_input_in_reg(ctx, addends[0], NarrowValueMode::ZeroExtend64); + let rb = put_input_in_reg(ctx, addends[1], NarrowValueMode::ZeroExtend64); return MemArg::reg_plus_reg(ra, rb); } @@ -609,7 +611,7 @@ pub(crate) fn lower_address>( // Add each addend to the address. for addend in addends { - let reg = input_to_reg(ctx, *addend, NarrowValueMode::ZeroExtend64); + let reg = put_input_in_reg(ctx, *addend, NarrowValueMode::ZeroExtend64); // In an addition, the stack register is the zero register, so divert it to another // register just before doing the actual add. @@ -726,6 +728,77 @@ pub(crate) fn lower_fp_condcode(cc: FloatCC) -> Cond { } } +pub(crate) fn lower_vector_compare>( + ctx: &mut C, + rd: Writable, + mut rn: Reg, + mut rm: Reg, + ty: Type, + cond: Cond, +) -> CodegenResult<()> { + match ty { + F32X4 | F64X2 | I8X16 | I16X8 | I32X4 => {} + _ => { + return Err(CodegenError::Unsupported(format!( + "unsupported SIMD type: {:?}", + ty + ))); + } + }; + + let is_float = match ty { + F32X4 | F64X2 => true, + _ => false, + }; + // 'Less than' operations are implemented by swapping + // the order of operands and using the 'greater than' + // instructions. + // 'Not equal' is implemented with 'equal' and inverting + // the result. + let (alu_op, swap) = match (is_float, cond) { + (false, Cond::Eq) => (VecALUOp::Cmeq, false), + (false, Cond::Ne) => (VecALUOp::Cmeq, false), + (false, Cond::Ge) => (VecALUOp::Cmge, false), + (false, Cond::Gt) => (VecALUOp::Cmgt, false), + (false, Cond::Le) => (VecALUOp::Cmge, true), + (false, Cond::Lt) => (VecALUOp::Cmgt, true), + (false, Cond::Hs) => (VecALUOp::Cmhs, false), + (false, Cond::Hi) => (VecALUOp::Cmhi, false), + (false, Cond::Ls) => (VecALUOp::Cmhs, true), + (false, Cond::Lo) => (VecALUOp::Cmhi, true), + (true, Cond::Eq) => (VecALUOp::Fcmeq, false), + (true, Cond::Ne) => (VecALUOp::Fcmeq, false), + (true, Cond::Mi) => (VecALUOp::Fcmgt, true), + (true, Cond::Ls) => (VecALUOp::Fcmge, true), + (true, Cond::Ge) => (VecALUOp::Fcmge, false), + (true, Cond::Gt) => (VecALUOp::Fcmgt, false), + _ => unreachable!(), + }; + + if swap { + std::mem::swap(&mut rn, &mut rm); + } + + ctx.emit(Inst::VecRRR { + alu_op, + rd, + rn, + rm, + ty, + }); + + if cond == Cond::Ne { + ctx.emit(Inst::VecMisc { + op: VecMisc2::Not, + rd, + rn: rd.to_reg(), + ty: I8X16, + }); + } + + Ok(()) +} + /// Determines whether this condcode interprets inputs as signed or /// unsigned. See the documentation for the `icmp` instruction in /// cranelift-codegen/meta/src/shared/instructions.rs for further insights @@ -762,6 +835,7 @@ pub fn ty_bits(ty: Type) -> usize { IFLAGS | FFLAGS => 32, B8X8 | I8X8 | B16X4 | I16X4 | B32X2 | I32X2 => 64, B8X16 | I8X16 | B16X8 | I16X8 | B32X4 | I32X4 | B64X2 | I64X2 => 128, + F32X4 | F64X2 => 128, _ => panic!("ty_bits() on unknown type: {:?}", ty), } } @@ -925,8 +999,8 @@ pub(crate) fn lower_icmp_or_ifcmp_to_flags>( }, ]; let ty = ctx.input_ty(insn, 0); - let rn = input_to_reg(ctx, inputs[0], narrow_mode); - let rm = input_to_rse_imm12(ctx, inputs[1], narrow_mode); + let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); + let rm = put_input_in_rse_imm12(ctx, inputs[1], narrow_mode); debug!("lower_icmp_or_ifcmp_to_flags: rn = {:?} rm = {:?}", rn, rm); let alu_op = choose_32_64(ty, ALUOp::SubS32, ALUOp::SubS64); let rd = writable_zero_reg(); @@ -946,8 +1020,8 @@ pub(crate) fn lower_fcmp_or_ffcmp_to_flags>(ctx: &mut C, i input: 1, }, ]; - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); match bits { 32 => { ctx.emit(Inst::FpuCmp32 { rn, rm }); diff --git a/third_party/rust/cranelift-codegen/src/isa/aarch64/lower_inst.rs b/third_party/rust/cranelift-codegen/src/isa/aarch64/lower_inst.rs index a97eab76e72c..82eb35f13fb8 100644 --- a/third_party/rust/cranelift-codegen/src/isa/aarch64/lower_inst.rs +++ b/third_party/rust/cranelift-codegen/src/isa/aarch64/lower_inst.rs @@ -7,7 +7,7 @@ use crate::ir::Inst as IRInst; use crate::ir::{InstructionData, Opcode, TrapCode}; use crate::machinst::lower::*; use crate::machinst::*; -use crate::{CodegenError, CodegenResult}; +use crate::CodegenResult; use crate::isa::aarch64::abi::*; use crate::isa::aarch64::inst::*; @@ -42,31 +42,31 @@ pub(crate) fn lower_insn_to_regs>( match op { Opcode::Iconst | Opcode::Bconst | Opcode::Null => { let value = ctx.get_constant(insn).unwrap(); - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); lower_constant_u64(ctx, rd, value); } Opcode::F32const => { let value = f32::from_bits(ctx.get_constant(insn).unwrap() as u32); - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); lower_constant_f32(ctx, rd, value); } Opcode::F64const => { let value = f64::from_bits(ctx.get_constant(insn).unwrap()); - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); lower_constant_f64(ctx, rd, value); } Opcode::Iadd => { - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_rse_imm12(ctx, inputs[1], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_rse_imm12(ctx, inputs[1], NarrowValueMode::None); let ty = ty.unwrap(); let alu_op = choose_32_64(ty, ALUOp::Add32, ALUOp::Add64); ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm)); } Opcode::Isub => { - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_rse_imm12(ctx, inputs[1], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_rse_imm12(ctx, inputs[1], NarrowValueMode::None); let ty = ty.unwrap(); let alu_op = choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64); ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm)); @@ -87,9 +87,9 @@ pub(crate) fn lower_insn_to_regs>( }; let va = ctx.alloc_tmp(RegClass::V128, I128); let vb = ctx.alloc_tmp(RegClass::V128, I128); - let ra = input_to_reg(ctx, inputs[0], narrow_mode); - let rb = input_to_reg(ctx, inputs[1], narrow_mode); - let rd = output_to_reg(ctx, outputs[0]); + let ra = put_input_in_reg(ctx, inputs[0], narrow_mode); + let rb = put_input_in_reg(ctx, inputs[1], narrow_mode); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::MovToVec64 { rd: va, rn: ra }); ctx.emit(Inst::MovToVec64 { rd: vb, rn: rb }); ctx.emit(Inst::VecRRR { @@ -121,9 +121,9 @@ pub(crate) fn lower_insn_to_regs>( }; let va = ctx.alloc_tmp(RegClass::V128, I128); let vb = ctx.alloc_tmp(RegClass::V128, I128); - let ra = input_to_reg(ctx, inputs[0], narrow_mode); - let rb = input_to_reg(ctx, inputs[1], narrow_mode); - let rd = output_to_reg(ctx, outputs[0]); + let ra = put_input_in_reg(ctx, inputs[0], narrow_mode); + let rb = put_input_in_reg(ctx, inputs[1], narrow_mode); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::MovToVec64 { rd: va, rn: ra }); ctx.emit(Inst::MovToVec64 { rd: vb, rn: rb }); ctx.emit(Inst::VecRRR { @@ -142,18 +142,18 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Ineg => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let rn = zero_reg(); - let rm = input_to_rse_imm12(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_rse_imm12(ctx, inputs[0], NarrowValueMode::None); let ty = ty.unwrap(); let alu_op = choose_32_64(ty, ALUOp::Sub32, ALUOp::Sub64); ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm)); } Opcode::Imul => { - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); let ty = ty.unwrap(); let alu_op = choose_32_64(ty, ALUOp::MAdd32, ALUOp::MAdd64); ctx.emit(Inst::AluRRRR { @@ -166,7 +166,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Umulhi | Opcode::Smulhi => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let is_signed = op == Opcode::Smulhi; let input_ty = ctx.input_ty(insn, 0); assert!(ctx.input_ty(insn, 1) == input_ty); @@ -174,8 +174,8 @@ pub(crate) fn lower_insn_to_regs>( match input_ty { I64 => { - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); let ra = zero_reg(); let alu_op = if is_signed { ALUOp::SMulH @@ -196,8 +196,8 @@ pub(crate) fn lower_insn_to_regs>( } else { NarrowValueMode::ZeroExtend64 }; - let rn = input_to_reg(ctx, inputs[0], narrow_mode); - let rm = input_to_reg(ctx, inputs[1], narrow_mode); + let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); + let rm = put_input_in_reg(ctx, inputs[1], narrow_mode); let ra = zero_reg(); ctx.emit(Inst::AluRRRR { alu_op: ALUOp::MAdd64, @@ -254,9 +254,9 @@ pub(crate) fn lower_insn_to_regs>( ALUOp::UDiv64 }; - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[0], narrow_mode); - let rm = input_to_reg(ctx, inputs[1], narrow_mode); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); + let rm = put_input_in_reg(ctx, inputs[1], narrow_mode); // The div instruction does not trap on divide by zero or signed overflow // so checks are inserted below. // @@ -372,8 +372,8 @@ pub(crate) fn lower_insn_to_regs>( // If we reach this point, we weren't able to incorporate the extend as // a register-mode on another instruction, so we have a 'None' // narrow-value/extend mode here, and we emit the explicit instruction. - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::Extend { rd, rn, @@ -385,15 +385,15 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Bnot => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let ty = ty.unwrap(); if ty_bits(ty) < 128 { - let rm = input_to_rs_immlogic(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_rs_immlogic(ctx, inputs[0], NarrowValueMode::None); let alu_op = choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64); // NOT rd, rm ==> ORR_NOT rd, zero, rm ctx.emit(alu_inst_immlogic(alu_op, rd, zero_reg(), rm)); } else { - let rm = input_to_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); ctx.emit(Inst::VecMisc { op: VecMisc2::Not, rd, @@ -409,11 +409,11 @@ pub(crate) fn lower_insn_to_regs>( | Opcode::BandNot | Opcode::BorNot | Opcode::BxorNot => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let ty = ty.unwrap(); if ty_bits(ty) < 128 { - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_rs_immlogic(ctx, inputs[1], NarrowValueMode::None); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_rs_immlogic(ctx, inputs[1], NarrowValueMode::None); let alu_op = match op { Opcode::Band => choose_32_64(ty, ALUOp::And32, ALUOp::And64), Opcode::Bor => choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64), @@ -433,9 +433,9 @@ pub(crate) fn lower_insn_to_regs>( _ => unreachable!(), }; - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::VecRRR { alu_op, @@ -458,9 +458,9 @@ pub(crate) fn lower_insn_to_regs>( (Opcode::Sshr, InstSize::Size32) => NarrowValueMode::SignExtend32, _ => unreachable!(), }; - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[0], narrow_mode); - let rm = input_to_reg_immshift(ctx, inputs[1]); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); + let rm = put_input_in_reg_immshift(ctx, inputs[1], ty_bits(ty)); let alu_op = match op { Opcode::Ishl => choose_32_64(ty, ALUOp::Lsl32, ALUOp::Lsl64), Opcode::Ushr => choose_32_64(ty, ALUOp::Lsr32, ALUOp::Lsr64), @@ -503,8 +503,8 @@ pub(crate) fn lower_insn_to_regs>( let ty = ty.unwrap(); let ty_bits_size = ty_bits(ty) as u8; - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg( + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg( ctx, inputs[0], if ty_bits_size <= 32 { @@ -513,7 +513,7 @@ pub(crate) fn lower_insn_to_regs>( NarrowValueMode::ZeroExtend64 }, ); - let rm = input_to_reg_immshift(ctx, inputs[1]); + let rm = put_input_in_reg_immshift(ctx, inputs[1], ty_bits(ty)); if ty_bits_size == 32 || ty_bits_size == 64 { let alu_op = choose_32_64(ty, ALUOp::RotR32, ALUOp::RotR64); @@ -652,7 +652,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Bitrev | Opcode::Clz | Opcode::Cls | Opcode::Ctz => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let needs_zext = match op { Opcode::Bitrev | Opcode::Ctz => false, Opcode::Clz | Opcode::Cls => true, @@ -666,7 +666,7 @@ pub(crate) fn lower_insn_to_regs>( } else { NarrowValueMode::None }; - let rn = input_to_reg(ctx, inputs[0], narrow_mode); + let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); let op_ty = match ty { I8 | I16 | I32 => I32, I64 => I64, @@ -722,11 +722,11 @@ pub(crate) fn lower_insn_to_regs>( // x += x << 32 // x >> 56 let ty = ty.unwrap(); - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); // FIXME(#1537): zero-extend 8/16/32-bit operands only to 32 bits, // and fix the sequence below to work properly for this. let narrow_mode = NarrowValueMode::ZeroExtend64; - let rn = input_to_reg(ctx, inputs[0], narrow_mode); + let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); let tmp = ctx.alloc_tmp(RegClass::I64, I64); // If this is a 32-bit Popcnt, use Lsr32 to clear the top 32 bits of the register, then @@ -903,7 +903,7 @@ pub(crate) fn lower_insn_to_regs>( let is_float = ty_is_float(elem_ty); let mem = lower_address(ctx, elem_ty, &inputs[..], off); - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let memflags = ctx.memflags(insn).expect("memory flags"); let srcloc = if !memflags.notrap() { @@ -967,7 +967,7 @@ pub(crate) fn lower_insn_to_regs>( let is_float = ty_is_float(elem_ty); let mem = lower_address(ctx, elem_ty, &inputs[1..], off); - let rd = input_to_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); let memflags = ctx.memflags(insn).expect("memory flags"); let srcloc = if !memflags.notrap() { @@ -997,7 +997,7 @@ pub(crate) fn lower_insn_to_regs>( } => (stack_slot, offset), _ => unreachable!(), }; - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let offset: i32 = offset.into(); let inst = ctx .abi() @@ -1023,7 +1023,7 @@ pub(crate) fn lower_insn_to_regs>( // Nothing. } - Opcode::Select | Opcode::Selectif => { + Opcode::Select | Opcode::Selectif | Opcode::SelectifSpectreGuard => { let cond = if op == Opcode::Select { let (cmp_op, narrow_mode) = if ty_bits(ctx.input_ty(insn, 0)) > 32 { (ALUOp::SubS64, NarrowValueMode::ZeroExtend64) @@ -1031,7 +1031,7 @@ pub(crate) fn lower_insn_to_regs>( (ALUOp::SubS32, NarrowValueMode::ZeroExtend32) }; - let rcond = input_to_reg(ctx, inputs[0], narrow_mode); + let rcond = put_input_in_reg(ctx, inputs[0], narrow_mode); // cmp rcond, #0 ctx.emit(Inst::AluRRR { alu_op: cmp_op, @@ -1052,9 +1052,9 @@ pub(crate) fn lower_insn_to_regs>( }; // csel.COND rd, rn, rm - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[1], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[2], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None); let ty = ctx.output_ty(insn, 0); let bits = ty_bits(ty); if ty_is_float(ty) && bits == 32 { @@ -1070,10 +1070,10 @@ pub(crate) fn lower_insn_to_regs>( let ty = ty.unwrap(); if ty_bits(ty) < 128 { let tmp = ctx.alloc_tmp(RegClass::I64, I64); - let rd = output_to_reg(ctx, outputs[0]); - let rcond = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rn = input_to_reg(ctx, inputs[1], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[2], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + let rcond = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None); // AND rTmp, rn, rcond ctx.emit(Inst::AluRRR { alu_op: ALUOp::And64, @@ -1096,10 +1096,10 @@ pub(crate) fn lower_insn_to_regs>( rm: tmp.to_reg(), }); } else { - let rcond = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rn = input_to_reg(ctx, inputs[1], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[2], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rcond = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rn = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::gen_move(rd, rcond, ty)); ctx.emit(Inst::VecRRR { @@ -1120,7 +1120,7 @@ pub(crate) fn lower_insn_to_regs>( // single-def ifcmp. let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap(); lower_icmp_or_ifcmp_to_flags(ctx, ifcmp_insn, is_signed); - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::CSet { rd, cond }); } @@ -1129,7 +1129,7 @@ pub(crate) fn lower_insn_to_regs>( let cond = lower_fp_condcode(condcode); let ffcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ffcmp).unwrap(); lower_fcmp_or_ffcmp_to_flags(ctx, ffcmp_insn); - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::CSet { rd, cond }); } @@ -1138,8 +1138,8 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Copy => { - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); let ty = ctx.input_ty(insn, 0); ctx.emit(Inst::gen_move(rd, rn, ty)); } @@ -1157,16 +1157,16 @@ pub(crate) fn lower_insn_to_regs>( // - Ireduce: changing width of an integer. Smaller ints are stored // with undefined high-order bits, so we can simply do a copy. - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); + let rd = get_output_reg(ctx, outputs[0]); let ty = ctx.input_ty(insn, 0); ctx.emit(Inst::gen_move(rd, rn, ty)); } Opcode::Bmask => { // Bool is {0, 1}, so we can subtract from 0 to get all-1s. - let rd = output_to_reg(ctx, outputs[0]); - let rm = input_to_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); + let rd = get_output_reg(ctx, outputs[0]); + let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); ctx.emit(Inst::AluRRR { alu_op: ALUOp::Sub64, rd, @@ -1176,7 +1176,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Bitcast => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let ity = ctx.input_ty(insn, 0); let oty = ctx.output_ty(insn, 0); match (ty_is_float(ity), ty_is_float(oty)) { @@ -1186,19 +1186,19 @@ pub(crate) fn lower_insn_to_regs>( } else { NarrowValueMode::ZeroExtend64 }; - let rm = input_to_reg(ctx, inputs[0], narrow_mode); + let rm = put_input_in_reg(ctx, inputs[0], narrow_mode); ctx.emit(Inst::gen_move(rd, rm, oty)); } (false, false) => { - let rm = input_to_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); ctx.emit(Inst::gen_move(rd, rm, oty)); } (false, true) => { - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); ctx.emit(Inst::MovToVec64 { rd, rn }); } (true, false) => { - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); ctx.emit(Inst::MovFromVec { rd, rn, @@ -1214,7 +1214,7 @@ pub(crate) fn lower_insn_to_regs>( // N.B.: according to the AArch64 ABI, the top bits of a register // (above the bits for the value's type) are undefined, so we // need not extend the return values. - let reg = input_to_reg(ctx, *input, NarrowValueMode::None); + let reg = put_input_in_reg(ctx, *input, NarrowValueMode::None); let retval_reg = ctx.retval(i); let ty = ctx.input_ty(insn, i); ctx.emit(Inst::gen_move(retval_reg, reg, ty)); @@ -1234,6 +1234,7 @@ pub(crate) fn lower_insn_to_regs>( let condcode = inst_condcode(ctx.data(insn)).unwrap(); let cond = lower_condcode(condcode); let is_signed = condcode_is_signed(condcode); + let rd = get_output_reg(ctx, outputs[0]); let ty = ctx.input_ty(insn, 0); let bits = ty_bits(ty); let narrow_mode = match (bits <= 32, is_signed) { @@ -1242,68 +1243,16 @@ pub(crate) fn lower_insn_to_regs>( (false, true) => NarrowValueMode::SignExtend64, (false, false) => NarrowValueMode::ZeroExtend64, }; + let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); if ty_bits(ty) < 128 { let alu_op = choose_32_64(ty, ALUOp::SubS32, ALUOp::SubS64); - let rn = input_to_reg(ctx, inputs[0], narrow_mode); - let rm = input_to_rse_imm12(ctx, inputs[1], narrow_mode); - let rd = output_to_reg(ctx, outputs[0]); + let rm = put_input_in_rse_imm12(ctx, inputs[1], narrow_mode); ctx.emit(alu_inst_imm12(alu_op, writable_zero_reg(), rn, rm)); ctx.emit(Inst::CondSet { cond, rd }); } else { - match ty { - I8X16 | I16X8 | I32X4 => {} - _ => { - return Err(CodegenError::Unsupported(format!( - "unsupported simd type: {:?}", - ty - ))); - } - }; - - let mut rn = input_to_reg(ctx, inputs[0], narrow_mode); - let mut rm = input_to_reg(ctx, inputs[1], narrow_mode); - let rd = output_to_reg(ctx, outputs[0]); - - // 'Less than' operations are implemented by swapping - // the order of operands and using the 'greater than' - // instructions. - // 'Not equal' is implemented with 'equal' and inverting - // the result. - let (alu_op, swap) = match cond { - Cond::Eq => (VecALUOp::Cmeq, false), - Cond::Ne => (VecALUOp::Cmeq, false), - Cond::Ge => (VecALUOp::Cmge, false), - Cond::Gt => (VecALUOp::Cmgt, false), - Cond::Le => (VecALUOp::Cmge, true), - Cond::Lt => (VecALUOp::Cmgt, true), - Cond::Hs => (VecALUOp::Cmhs, false), - Cond::Hi => (VecALUOp::Cmhi, false), - Cond::Ls => (VecALUOp::Cmhs, true), - Cond::Lo => (VecALUOp::Cmhi, true), - _ => unreachable!(), - }; - - if swap { - std::mem::swap(&mut rn, &mut rm); - } - - ctx.emit(Inst::VecRRR { - alu_op, - rd, - rn, - rm, - ty, - }); - - if cond == Cond::Ne { - ctx.emit(Inst::VecMisc { - op: VecMisc2::Not, - rd, - rn: rd.to_reg(), - ty: I8X16, - }); - } + let rm = put_input_in_reg(ctx, inputs[1], narrow_mode); + lower_vector_compare(ctx, rd, rn, rm, ty, cond)?; } } @@ -1311,19 +1260,24 @@ pub(crate) fn lower_insn_to_regs>( let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); let cond = lower_fp_condcode(condcode); let ty = ctx.input_ty(insn, 0); - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); - match ty_bits(ty) { - 32 => { - ctx.emit(Inst::FpuCmp32 { rn, rm }); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + + if ty_bits(ty) < 128 { + match ty_bits(ty) { + 32 => { + ctx.emit(Inst::FpuCmp32 { rn, rm }); + } + 64 => { + ctx.emit(Inst::FpuCmp64 { rn, rm }); + } + _ => panic!("Bad float size"), } - 64 => { - ctx.emit(Inst::FpuCmp64 { rn, rm }); - } - _ => panic!("Bad float size"), + ctx.emit(Inst::CondSet { cond, rd }); + } else { + lower_vector_compare(ctx, rd, rn, rm, ty, cond)?; } - ctx.emit(Inst::CondSet { cond, rd }); } Opcode::JumpTableEntry | Opcode::JumpTableBase => { @@ -1390,7 +1344,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::FuncAddr => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let (extname, _) = ctx.call_target(insn).unwrap(); let extname = extname.clone(); let loc = ctx.srcloc(insn); @@ -1407,7 +1361,7 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::SymbolValue => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); let (extname, _, offset) = ctx.symbol_value(insn).unwrap(); let extname = extname.clone(); let loc = ctx.srcloc(insn); @@ -1434,7 +1388,7 @@ pub(crate) fn lower_insn_to_regs>( ) } Opcode::CallIndirect => { - let ptr = input_to_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); + let ptr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); let sig = ctx.call_sig(insn).unwrap(); assert!(inputs.len() - 1 == sig.params.len()); assert!(outputs.len() == sig.returns.len()); @@ -1446,24 +1400,24 @@ pub(crate) fn lower_insn_to_regs>( abi.emit_stack_pre_adjust(ctx); assert!(inputs.len() == abi.num_args()); for (i, input) in inputs.iter().enumerate() { - let arg_reg = input_to_reg(ctx, *input, NarrowValueMode::None); + let arg_reg = put_input_in_reg(ctx, *input, NarrowValueMode::None); abi.emit_copy_reg_to_arg(ctx, i, arg_reg); } abi.emit_call(ctx); for (i, output) in outputs.iter().enumerate() { - let retval_reg = output_to_reg(ctx, *output); + let retval_reg = get_output_reg(ctx, *output); abi.emit_copy_retval_to_reg(ctx, i, retval_reg); } abi.emit_stack_post_adjust(ctx); } Opcode::GetPinnedReg => { - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::mov(rd, xreg(PINNED_REG))); } Opcode::SetPinnedReg => { - let rm = input_to_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); ctx.emit(Inst::mov(writable_xreg(PINNED_REG), rm)); } @@ -1497,13 +1451,13 @@ pub(crate) fn lower_insn_to_regs>( Opcode::Vconst => { let value = output_to_const_f128(ctx, outputs[0]).unwrap(); - let rd = output_to_reg(ctx, outputs[0]); + let rd = get_output_reg(ctx, outputs[0]); lower_constant_f128(ctx, rd, value); } Opcode::RawBitcast => { - let rm = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); let ty = ctx.input_ty(insn, 0); ctx.emit(Inst::gen_move(rd, rm, ty)); } @@ -1511,8 +1465,8 @@ pub(crate) fn lower_insn_to_regs>( Opcode::Extractlane => { if let InstructionData::BinaryImm8 { imm, .. } = ctx.data(insn) { let idx = *imm; - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); let ty = ty.unwrap(); if ty_is_int(ty) { @@ -1529,8 +1483,8 @@ pub(crate) fn lower_insn_to_regs>( } Opcode::Splat => { - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); let ty = ctx.input_ty(insn, 0); let inst = if ty_is_int(ty) { Inst::VecDup { rd, rn, ty } @@ -1540,12 +1494,58 @@ pub(crate) fn lower_insn_to_regs>( ctx.emit(inst); } + Opcode::VanyTrue | Opcode::VallTrue => { + let rd = get_output_reg(ctx, outputs[0]); + let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let tmp = ctx.alloc_tmp(RegClass::V128, ty.unwrap()); + + // This operation is implemented by using umaxp or uminv to + // create a scalar value, which is then compared against zero. + // + // umaxp vn.16b, vm.16, vm.16 / uminv bn, vm.16b + // mov xm, vn.d[0] + // cmp xm, #0 + // cset xm, ne + + let input_ty = ctx.input_ty(insn, 0); + if op == Opcode::VanyTrue { + ctx.emit(Inst::VecRRR { + alu_op: VecALUOp::Umaxp, + rd: tmp, + rn: rm, + rm: rm, + ty: input_ty, + }); + } else { + ctx.emit(Inst::VecLanes { + op: VecLanesOp::Uminv, + rd: tmp, + rn: rm, + ty: input_ty, + }); + }; + + ctx.emit(Inst::MovFromVec { + rd, + rn: tmp.to_reg(), + idx: 0, + ty: I64, + }); + + ctx.emit(Inst::AluRRImm12 { + alu_op: ALUOp::SubS64, + rd: writable_zero_reg(), + rn: rd.to_reg(), + imm12: Imm12::zero(), + }); + + ctx.emit(Inst::CSet { rd, cond: Cond::Ne }); + } + Opcode::Shuffle | Opcode::Vsplit | Opcode::Vconcat | Opcode::Vselect - | Opcode::VanyTrue - | Opcode::VallTrue | Opcode::Insertlane | Opcode::ScalarToVector | Opcode::Swizzle @@ -1581,9 +1581,9 @@ pub(crate) fn lower_insn_to_regs>( (Opcode::Fmax, 64) => FPUOp2::Max64, _ => panic!("Unknown op/bits combination"), }; - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::FpuRRR { fpu_op, rd, rn, rm }); } @@ -1602,8 +1602,8 @@ pub(crate) fn lower_insn_to_regs>( (Opcode::Fdemote, 64) => panic!("Cannot demote to 64 bits"), _ => panic!("Unknown op/bits combination"), }; - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::FpuRR { fpu_op, rd, rn }); } @@ -1620,8 +1620,8 @@ pub(crate) fn lower_insn_to_regs>( (Opcode::Nearest, 64) => FpuRoundMode::Nearest64, _ => panic!("Unknown op/bits combination"), }; - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::FpuRound { op, rd, rn }); } @@ -1632,10 +1632,10 @@ pub(crate) fn lower_insn_to_regs>( 64 => FPUOp3::MAdd64, _ => panic!("Unknown op size"), }; - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None); - let ra = input_to_reg(ctx, inputs[2], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let ra = put_input_in_reg(ctx, inputs[2], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::FpuRRRR { fpu_op, rn, @@ -1658,9 +1658,9 @@ pub(crate) fn lower_insn_to_regs>( let ty = ctx.output_ty(insn, 0); let bits = ty_bits(ty) as u8; assert!(bits == 32 || bits == 64); - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_reg(ctx, inputs[1], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); let tmp = ctx.alloc_tmp(RegClass::V128, F64); // Copy LHS to rd. @@ -1699,8 +1699,8 @@ pub(crate) fn lower_insn_to_regs>( _ => panic!("Unknown input/output-bits combination"), }; - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); // First, check the output: it's important to carry the NaN conversion before the // in-bounds conversion, per wasm semantics. @@ -1842,8 +1842,8 @@ pub(crate) fn lower_insn_to_regs>( (true, 64) => NarrowValueMode::SignExtend64, _ => panic!("Unknown input size"), }; - let rn = input_to_reg(ctx, inputs[0], narrow_mode); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], narrow_mode); + let rd = get_output_reg(ctx, outputs[0]); ctx.emit(Inst::IntToFpu { op, rd, rn }); } @@ -1853,8 +1853,8 @@ pub(crate) fn lower_insn_to_regs>( let out_ty = ctx.output_ty(insn, 0); let out_bits = ty_bits(out_ty); let out_signed = op == Opcode::FcvtToSintSat; - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rd = output_to_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); // FIMM Vtmp1, u32::MAX or u64::MAX or i32::MAX or i64::MAX // FMIN Vtmp2, Vin, Vtmp1 @@ -1991,9 +1991,9 @@ pub(crate) fn lower_insn_to_regs>( // Now handle the iadd as above, except use an AddS opcode that sets // flags. - let rd = output_to_reg(ctx, outputs[0]); - let rn = input_to_reg(ctx, inputs[0], NarrowValueMode::None); - let rm = input_to_rse_imm12(ctx, inputs[1], NarrowValueMode::None); + let rd = get_output_reg(ctx, outputs[0]); + let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); + let rm = put_input_in_rse_imm12(ctx, inputs[1], NarrowValueMode::None); let ty = ty.unwrap(); let alu_op = choose_32_64(ty, ALUOp::AddS32, ALUOp::AddS64); ctx.emit(alu_inst_imm12(alu_op, rd, rn, rm)); @@ -2069,6 +2069,7 @@ pub(crate) fn lower_insn_to_regs>( panic!("x86-specific opcode in supposedly arch-neutral IR!"); } + Opcode::Iabs => unimplemented!(), Opcode::AvgRound => unimplemented!(), Opcode::TlsValue => unimplemented!(), } @@ -2139,7 +2140,7 @@ pub(crate) fn lower_branch>( kind: CondBrKind::Cond(cond), }); } else { - let rt = input_to_reg( + let rt = put_input_in_reg( ctx, InsnInput { insn: branches[0], @@ -2173,7 +2174,7 @@ pub(crate) fn lower_branch>( (false, true) => NarrowValueMode::SignExtend64, (false, false) => NarrowValueMode::ZeroExtend64, }; - let rn = input_to_reg( + let rn = put_input_in_reg( ctx, InsnInput { insn: branches[0], @@ -2181,7 +2182,7 @@ pub(crate) fn lower_branch>( }, narrow_mode, ); - let rm = input_to_rse_imm12( + let rm = put_input_in_rse_imm12( ctx, InsnInput { insn: branches[0], @@ -2220,7 +2221,7 @@ pub(crate) fn lower_branch>( } else { // If the ifcmp result is actually placed in a // register, we need to move it back into the flags. - let rn = input_to_reg(ctx, flag_input, NarrowValueMode::None); + let rn = put_input_in_reg(ctx, flag_input, NarrowValueMode::None); ctx.emit(Inst::MovToNZCV { rn }); ctx.emit(Inst::CondBr { taken, @@ -2248,7 +2249,7 @@ pub(crate) fn lower_branch>( } else { // If the ffcmp result is actually placed in a // register, we need to move it back into the flags. - let rn = input_to_reg(ctx, flag_input, NarrowValueMode::None); + let rn = put_input_in_reg(ctx, flag_input, NarrowValueMode::None); ctx.emit(Inst::MovToNZCV { rn }); ctx.emit(Inst::CondBr { taken, @@ -2294,7 +2295,7 @@ pub(crate) fn lower_branch>( needed_space: 4 * (6 + jt_size) as CodeOffset, }); - let ridx = input_to_reg( + let ridx = put_input_in_reg( ctx, InsnInput { insn: branches[0], diff --git a/third_party/rust/cranelift-codegen/src/isa/mod.rs b/third_party/rust/cranelift-codegen/src/isa/mod.rs index ad0d292c371c..3bd84fbc6e8a 100644 --- a/third_party/rust/cranelift-codegen/src/isa/mod.rs +++ b/third_party/rust/cranelift-codegen/src/isa/mod.rs @@ -150,6 +150,7 @@ pub enum LookupError { /// Builder for a `TargetIsa`. /// Modify the ISA-specific settings before creating the `TargetIsa` trait object with `finish`. +#[derive(Clone)] pub struct Builder { triple: Triple, setup: settings::Builder, diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/abi.rs b/third_party/rust/cranelift-codegen/src/isa/x64/abi.rs index 2505286e0885..7e40dd54e781 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/abi.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/abi.rs @@ -1,37 +1,59 @@ //! Implementation of the standard x64 ABI. use alloc::vec::Vec; +use log::trace; use regalloc::{RealReg, Reg, RegClass, Set, SpillSlot, Writable}; +use std::mem; use crate::ir::{self, types, types::*, ArgumentExtension, StackSlot, Type}; use crate::isa::{self, x64::inst::*}; use crate::machinst::*; use crate::settings; +use crate::{CodegenError, CodegenResult}; use args::*; -#[derive(Clone, Debug)] -enum ABIArg { - Reg(RealReg), - _Stack, -} +/// This is the limit for the size of argument and return-value areas on the +/// stack. We place a reasonable limit here to avoid integer overflow issues +/// with 32-bit arithmetic: for now, 128 MB. +static STACK_ARG_RET_SIZE_LIMIT: u64 = 128 * 1024 * 1024; #[derive(Clone, Debug)] -enum ABIRet { - Reg(RealReg), - _Stack, +enum ABIArg { + Reg(RealReg, ir::Type), + Stack(i64, ir::Type), +} + +/// X64 ABI information shared between body (callee) and caller. +struct ABISig { + /// Argument locations (regs or stack slots). Stack offsets are relative to + /// SP on entry to function. + args: Vec, + /// Return-value locations. Stack offsets are relative to the return-area + /// pointer. + rets: Vec, + /// Space on stack used to store arguments. + stack_arg_space: i64, + /// Space on stack used to store return values. + stack_ret_space: i64, + /// Index in `args` of the stack-return-value-area argument. + stack_ret_arg: Option, + /// Calling convention used. + call_conv: isa::CallConv, } pub(crate) struct X64ABIBody { - args: Vec, - rets: Vec, + sig: ABISig, /// Offsets to each stack slot. - _stack_slots: Vec, + stack_slots: Vec, /// Total stack size of all the stack slots. stack_slots_size: usize, + /// The register holding the return-area pointer, if needed. + ret_area_ptr: Option>, + /// Clobbered registers, as indicated by regalloc. clobbered: Set>, @@ -48,7 +70,7 @@ pub(crate) struct X64ABIBody { flags: settings::Flags, } -fn use_int_reg(ty: types::Type) -> bool { +fn in_int_reg(ty: types::Type) -> bool { match ty { types::I8 | types::I16 @@ -63,7 +85,7 @@ fn use_int_reg(ty: types::Type) -> bool { } } -fn use_flt_reg(ty: types::Type) -> bool { +fn in_vec_reg(ty: types::Type) -> bool { match ty { types::F32 | types::F64 => true, _ => false, @@ -132,97 +154,50 @@ fn get_callee_saves(regs: Vec>) -> Vec> { impl X64ABIBody { /// Create a new body ABI instance. - pub(crate) fn new(f: &ir::Function, flags: settings::Flags) -> Self { - // Compute args and retvals from signature. - let mut args = vec![]; - let mut next_int_arg = 0; - let mut next_flt_arg = 0; - for param in &f.signature.params { - match param.purpose { - ir::ArgumentPurpose::VMContext if f.signature.call_conv.extends_baldrdash() => { - // `VMContext` is `r14` in Baldrdash. - args.push(ABIArg::Reg(regs::r14().to_real_reg())); - } + pub(crate) fn new(f: &ir::Function, flags: settings::Flags) -> CodegenResult { + let sig = ABISig::from_func_sig(&f.signature)?; - ir::ArgumentPurpose::Normal | ir::ArgumentPurpose::VMContext => { - if use_int_reg(param.value_type) { - if let Some(reg) = get_intreg_for_arg_systemv(next_int_arg) { - args.push(ABIArg::Reg(reg.to_real_reg())); - } else { - unimplemented!("passing arg on the stack"); - } - next_int_arg += 1; - } else if use_flt_reg(param.value_type) { - if let Some(reg) = get_fltreg_for_arg_systemv(next_flt_arg) { - args.push(ABIArg::Reg(reg.to_real_reg())); - } else { - unimplemented!("passing arg on the stack"); - } - next_flt_arg += 1; - } else { - unimplemented!("non int normal register {:?}", param.value_type) - } - } - - _ => unimplemented!("other parameter purposes"), - } - } - - let mut rets = vec![]; - let mut next_int_retval = 0; - let mut next_flt_retval = 0; - for ret in &f.signature.returns { - match ret.purpose { - ir::ArgumentPurpose::Normal => { - if use_int_reg(ret.value_type) { - if let Some(reg) = get_intreg_for_retval_systemv(next_int_retval) { - rets.push(ABIRet::Reg(reg.to_real_reg())); - } else { - unimplemented!("passing return on the stack"); - } - next_int_retval += 1; - } else if use_flt_reg(ret.value_type) { - if let Some(reg) = get_fltreg_for_retval_systemv(next_flt_retval) { - rets.push(ABIRet::Reg(reg.to_real_reg())); - } else { - unimplemented!("passing return on the stack"); - } - next_flt_retval += 1; - } else { - unimplemented!("returning non integer normal value"); - } - } - - _ => { - unimplemented!("non normal argument purpose"); - } - } - } + let call_conv = f.signature.call_conv; + debug_assert!( + call_conv == isa::CallConv::SystemV || call_conv.extends_baldrdash(), + "unsupported or unimplemented calling convention {}", + call_conv + ); // Compute stackslot locations and total stackslot size. let mut stack_offset: usize = 0; - let mut _stack_slots = vec![]; + let mut stack_slots = vec![]; for (stackslot, data) in f.stack_slots.iter() { let off = stack_offset; stack_offset += data.size as usize; - - // 8-bit align. - stack_offset = (stack_offset + 7) & !7usize; - - debug_assert_eq!(stackslot.as_u32() as usize, _stack_slots.len()); - _stack_slots.push(off); + stack_offset = (stack_offset + 7) & !7; + debug_assert_eq!(stackslot.as_u32() as usize, stack_slots.len()); + stack_slots.push(off); } - Self { - args, - rets, - _stack_slots, + Ok(Self { + sig, + stack_slots, stack_slots_size: stack_offset, + ret_area_ptr: None, clobbered: Set::empty(), num_spill_slots: None, frame_size_bytes: None, call_conv: f.signature.call_conv.clone(), flags, + }) + } + + /// Returns the offset from FP to the argument area, i.e., jumping over the saved FP, return + /// address, and maybe other standard elements depending on ABI (e.g. Wasm TLS reg). + fn fp_to_arg_offset(&self) -> i64 { + if self.call_conv.extends_baldrdash() { + let num_words = self.flags.baldrdash_prologue_words() as i64; + debug_assert!(num_words > 0, "baldrdash must set baldrdash_prologue_words"); + debug_assert_eq!(num_words % 2, 0, "stack must be 16-aligned"); + num_words * 8 + } else { + 16 // frame pointer + return address. } } } @@ -231,31 +206,34 @@ impl ABIBody for X64ABIBody { type I = Inst; fn temp_needed(&self) -> bool { - false + self.sig.stack_ret_arg.is_some() } - fn init(&mut self, _: Option>) {} + fn init(&mut self, maybe_tmp: Option>) { + if self.sig.stack_ret_arg.is_some() { + assert!(maybe_tmp.is_some()); + self.ret_area_ptr = maybe_tmp; + } + } fn flags(&self) -> &settings::Flags { &self.flags } fn num_args(&self) -> usize { - unimplemented!() + self.sig.args.len() } - fn num_retvals(&self) -> usize { - unimplemented!() + self.sig.rets.len() } - fn num_stackslots(&self) -> usize { - unimplemented!() + self.stack_slots.len() } fn liveins(&self) -> Set { let mut set: Set = Set::empty(); - for arg in &self.args { - if let &ABIArg::Reg(r) = arg { + for arg in &self.sig.args { + if let &ABIArg::Reg(r, _) = arg { set.insert(r); } } @@ -264,8 +242,8 @@ impl ABIBody for X64ABIBody { fn liveouts(&self) -> Set { let mut set: Set = Set::empty(); - for ret in &self.rets { - if let &ABIRet::Reg(r) = ret { + for ret in &self.sig.rets { + if let &ABIArg::Reg(r, _) = ret { set.insert(r); } } @@ -273,18 +251,19 @@ impl ABIBody for X64ABIBody { } fn gen_copy_arg_to_reg(&self, idx: usize, to_reg: Writable) -> Inst { - match &self.args[idx] { - ABIArg::Reg(from_reg) => { - if from_reg.get_class() == RegClass::I32 || from_reg.get_class() == RegClass::I64 { - // TODO do we need a sign extension if it's I32? - return Inst::mov_r_r(/*is64=*/ true, from_reg.to_reg(), to_reg); - } else if from_reg.get_class() == RegClass::V128 { - // TODO: How to support Movss. Should is64 always be true? - return Inst::xmm_r_r(SSE_Op::SSE2_Movsd, from_reg.to_reg(), to_reg); - } - unimplemented!("moving from non-int arg to vreg {:?}", from_reg.get_class()); + match &self.sig.args[idx] { + ABIArg::Reg(from_reg, ty) => Inst::gen_move(to_reg, from_reg.to_reg(), *ty), + &ABIArg::Stack(off, ty) => { + assert!( + self.fp_to_arg_offset() + off <= u32::max_value() as i64, + "large offset nyi" + ); + load_stack( + Amode::imm_reg((self.fp_to_arg_offset() + off) as u32, regs::rbp()), + to_reg, + ty, + ) } - ABIArg::_Stack => unimplemented!("moving from stack arg to vreg"), } } @@ -298,38 +277,74 @@ impl ABIBody for X64ABIBody { from_reg: Writable, ext: ArgumentExtension, ) -> Vec { - match ext { - ArgumentExtension::None => {} - _ => unimplemented!( - "unimplemented argument extension {:?} is required for baldrdash", - ext - ), - }; - let mut ret = Vec::new(); - match &self.rets[idx] { - ABIRet::Reg(to_reg) => { - if to_reg.get_class() == RegClass::I32 || to_reg.get_class() == RegClass::I64 { - ret.push(Inst::mov_r_r( - /*is64=*/ true, - from_reg.to_reg(), - Writable::::from_reg(to_reg.to_reg()), - )) - } else if to_reg.get_class() == RegClass::V128 - || to_reg.get_class() == RegClass::V128 - { - ret.push(Inst::xmm_r_r( - SSE_Op::SSE2_Movsd, - from_reg.to_reg(), - Writable::::from_reg(to_reg.to_reg()), - )) - } else { - unimplemented!("moving from vreg to unsupported return value"); - } + match &self.sig.rets[idx] { + &ABIArg::Reg(r, ty) => { + let from_bits = ty.bits() as u8; + let ext_mode = match from_bits { + 1 | 8 => Some(ExtMode::BQ), + 16 => Some(ExtMode::WQ), + 32 => Some(ExtMode::LQ), + 64 => None, + _ => unreachable!(), + }; + + let dest_reg = Writable::from_reg(r.to_reg()); + match (ext, ext_mode) { + (ArgumentExtension::Uext, Some(ext_mode)) => { + ret.push(Inst::movzx_rm_r( + ext_mode, + RegMem::reg(r.to_reg()), + dest_reg, + )); + } + (ArgumentExtension::Sext, Some(ext_mode)) => { + ret.push(Inst::movsx_rm_r( + ext_mode, + RegMem::reg(r.to_reg()), + dest_reg, + )); + } + _ => ret.push(Inst::gen_move(dest_reg, from_reg.to_reg(), ty)), + }; } - ABIRet::_Stack => { - unimplemented!("moving from vreg to stack return value"); + &ABIArg::Stack(off, ty) => { + let from_bits = ty.bits() as u8; + let ext_mode = match from_bits { + 1 | 8 => Some(ExtMode::BQ), + 16 => Some(ExtMode::WQ), + 32 => Some(ExtMode::LQ), + 64 => None, + _ => unreachable!(), + }; + + // Trash the from_reg; it should be its last use. + match (ext, ext_mode) { + (ArgumentExtension::Uext, Some(ext_mode)) => { + ret.push(Inst::movzx_rm_r( + ext_mode, + RegMem::reg(from_reg.to_reg()), + from_reg, + )); + } + (ArgumentExtension::Sext, Some(ext_mode)) => { + ret.push(Inst::movsx_rm_r( + ext_mode, + RegMem::reg(from_reg.to_reg()), + from_reg, + )); + } + _ => {} + }; + + assert!( + off < u32::max_value() as i64, + "large stack return offset nyi" + ); + + let mem = Amode::imm_reg(off as u32, self.ret_area_ptr.unwrap().to_reg()); + ret.push(store_stack(mem, from_reg.to_reg(), ty)) } } @@ -352,8 +367,10 @@ impl ABIBody for X64ABIBody { self.clobbered = clobbered; } - fn stackslot_addr(&self, _slot: StackSlot, _offset: u32, _into_reg: Writable) -> Inst { - unimplemented!() + fn stackslot_addr(&self, slot: StackSlot, offset: u32, dst: Writable) -> Inst { + let stack_off = self.stack_slots[slot.as_u32() as usize] as i64; + let sp_off: i64 = stack_off + (offset as i64); + Inst::lea(SyntheticAmode::nominal_sp_offset(sp_off as u32), dst) } fn load_stackslot( @@ -386,29 +403,23 @@ impl ABIBody for X64ABIBody { // Baldrdash generates its own prologue sequence, so we don't have to. if !self.call_conv.extends_baldrdash() { let r_rbp = regs::rbp(); - let w_rbp = Writable::::from_reg(r_rbp); + let w_rbp = Writable::from_reg(r_rbp); // The "traditional" pre-preamble // RSP before the call will be 0 % 16. So here, it is 8 % 16. - insts.push(Inst::push64(RMI::reg(r_rbp))); + insts.push(Inst::push64(RegMemImm::reg(r_rbp))); // RSP is now 0 % 16 insts.push(Inst::mov_r_r(true, r_rsp, w_rbp)); } - // Save callee saved registers that we trash. Keep track of how much space we've used, so - // as to know what we have to do to get the base of the spill area 0 % 16. - let mut callee_saved_used = 0; let clobbered = get_callee_saves(self.clobbered.to_vec()); - for reg in clobbered { - let r_reg = reg.to_reg(); - match r_reg.get_class() { - RegClass::I64 => { - insts.push(Inst::push64(RMI::reg(r_reg.to_reg()))); - callee_saved_used += 8; - } - _ => unimplemented!(), - } - } + let callee_saved_used: usize = clobbered + .iter() + .map(|reg| match reg.to_reg().get_class() { + RegClass::I64 => 8, + _ => todo!(), + }) + .sum(); let mut total_stacksize = self.stack_slots_size + 8 * self.num_spill_slots.unwrap(); if self.call_conv.extends_baldrdash() { @@ -421,28 +432,47 @@ impl ABIBody for X64ABIBody { total_stacksize += self.flags.baldrdash_prologue_words() as usize * 8; } - debug_assert!(callee_saved_used % 16 == 0 || callee_saved_used % 16 == 8); - let frame_size = total_stacksize + callee_saved_used % 16; - // Now make sure the frame stack is aligned, so RSP == 0 % 16 in the function's body. - let frame_size = (frame_size + 15) & !15; - if frame_size > 0x7FFF_FFFF { - unimplemented!("gen_prologue(x86): total_stacksize >= 2G"); - } + let padding = (16 - ((total_stacksize + callee_saved_used) % 16)) & 15; + let frame_size = total_stacksize + padding; + debug_assert!( + frame_size <= u32::max_value() as usize, + "gen_prologue(x86): total_stacksize >= 2G" + ); + debug_assert_eq!((frame_size + callee_saved_used) % 16, 0, "misaligned stack"); if !self.call_conv.extends_baldrdash() { // Explicitly allocate the frame. - let w_rsp = Writable::::from_reg(r_rsp); + let w_rsp = Writable::from_reg(r_rsp); if frame_size > 0 { insts.push(Inst::alu_rmi_r( true, - RMI_R_Op::Sub, - RMI::imm(frame_size as u32), + AluRmiROpcode::Sub, + RegMemImm::imm(frame_size as u32), w_rsp, )); } } + // Save callee saved registers that we trash. Keep track of how much space we've used, so + // as to know what we have to do to get the base of the spill area 0 % 16. + let clobbered = get_callee_saves(self.clobbered.to_vec()); + for reg in clobbered { + let r_reg = reg.to_reg(); + match r_reg.get_class() { + RegClass::I64 => { + insts.push(Inst::push64(RegMemImm::reg(r_reg.to_reg()))); + } + _ => unimplemented!(), + } + } + + if callee_saved_used > 0 { + insts.push(Inst::VirtualSPOffsetAdj { + offset: callee_saved_used as i64, + }); + } + // Stash this value. We'll need it for the epilogue. debug_assert!(self.frame_size_bytes.is_none()); self.frame_size_bytes = Some(frame_size); @@ -455,44 +485,43 @@ impl ABIBody for X64ABIBody { // Undo what we did in the prologue. - // Clear the spill area and the 16-alignment padding below it. - if !self.call_conv.extends_baldrdash() { - let frame_size = self.frame_size_bytes.unwrap(); - if frame_size > 0 { - let r_rsp = regs::rsp(); - let w_rsp = Writable::::from_reg(r_rsp); - - insts.push(Inst::alu_rmi_r( - true, - RMI_R_Op::Add, - RMI::imm(frame_size as u32), - w_rsp, - )); - } - } - // Restore regs. let clobbered = get_callee_saves(self.clobbered.to_vec()); - for w_real_reg in clobbered.into_iter().rev() { - match w_real_reg.to_reg().get_class() { + for wreg in clobbered.into_iter().rev() { + let rreg = wreg.to_reg(); + match rreg.get_class() { RegClass::I64 => { // TODO: make these conversion sequences less cumbersome. - insts.push(Inst::pop64(Writable::::from_reg( - w_real_reg.to_reg().to_reg(), - ))) + insts.push(Inst::pop64(Writable::from_reg(rreg.to_reg()))); } _ => unimplemented!(), } } + // No need to adjust the virtual sp offset here: + // - this would create issues when there's a return in the middle of a function, + // - and nothing in this sequence may try to access stack slots from the nominal SP. + + // Clear the spill area and the 16-alignment padding below it. + if !self.call_conv.extends_baldrdash() { + let frame_size = self.frame_size_bytes.unwrap(); + if frame_size > 0 { + let r_rsp = regs::rsp(); + let w_rsp = Writable::from_reg(r_rsp); + insts.push(Inst::alu_rmi_r( + true, + AluRmiROpcode::Add, + RegMemImm::imm(frame_size as u32), + w_rsp, + )); + } + } + // Baldrdash generates its own preamble. if !self.call_conv.extends_baldrdash() { - let r_rbp = regs::rbp(); - let w_rbp = Writable::::from_reg(r_rbp); - // Undo the "traditional" pre-preamble // RSP before the call will be 0 % 16. So here, it is 8 % 16. - insts.push(Inst::pop64(w_rbp)); + insts.push(Inst::pop64(Writable::from_reg(regs::rbp()))); insts.push(Inst::ret()); } @@ -522,3 +551,471 @@ impl ABIBody for X64ABIBody { unimplemented!() } } + +fn get_caller_saves(call_conv: isa::CallConv) -> Vec> { + let mut caller_saved = Vec::new(); + + // Systemv calling convention: + // - GPR: all except RBX, RBP, R12 to R15 (which are callee-saved). + caller_saved.push(Writable::from_reg(regs::rsi())); + caller_saved.push(Writable::from_reg(regs::rdi())); + caller_saved.push(Writable::from_reg(regs::rax())); + caller_saved.push(Writable::from_reg(regs::rcx())); + caller_saved.push(Writable::from_reg(regs::rdx())); + caller_saved.push(Writable::from_reg(regs::r8())); + caller_saved.push(Writable::from_reg(regs::r9())); + caller_saved.push(Writable::from_reg(regs::r10())); + caller_saved.push(Writable::from_reg(regs::r11())); + + // - XMM: all the registers! + caller_saved.push(Writable::from_reg(regs::xmm0())); + caller_saved.push(Writable::from_reg(regs::xmm1())); + caller_saved.push(Writable::from_reg(regs::xmm2())); + caller_saved.push(Writable::from_reg(regs::xmm3())); + caller_saved.push(Writable::from_reg(regs::xmm4())); + caller_saved.push(Writable::from_reg(regs::xmm5())); + caller_saved.push(Writable::from_reg(regs::xmm6())); + caller_saved.push(Writable::from_reg(regs::xmm7())); + caller_saved.push(Writable::from_reg(regs::xmm8())); + caller_saved.push(Writable::from_reg(regs::xmm9())); + caller_saved.push(Writable::from_reg(regs::xmm10())); + caller_saved.push(Writable::from_reg(regs::xmm11())); + caller_saved.push(Writable::from_reg(regs::xmm12())); + caller_saved.push(Writable::from_reg(regs::xmm13())); + caller_saved.push(Writable::from_reg(regs::xmm14())); + caller_saved.push(Writable::from_reg(regs::xmm15())); + + if call_conv.extends_baldrdash() { + todo!("add the baldrdash caller saved") + } + + caller_saved +} + +fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec>) { + // Compute uses: all arg regs. + let mut uses = Vec::new(); + for arg in &sig.args { + match arg { + &ABIArg::Reg(reg, _) => uses.push(reg.to_reg()), + _ => {} + } + } + + // Compute defs: all retval regs, and all caller-save (clobbered) regs. + let mut defs = get_caller_saves(sig.call_conv); + for ret in &sig.rets { + match ret { + &ABIArg::Reg(reg, _) => defs.push(Writable::from_reg(reg.to_reg())), + _ => {} + } + } + + (uses, defs) +} + +/// Try to fill a Baldrdash register, returning it if it was found. +fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Option { + if call_conv.extends_baldrdash() { + match ¶m.purpose { + &ir::ArgumentPurpose::VMContext => { + // This is SpiderMonkey's `WasmTlsReg`. + Some(ABIArg::Reg(regs::r14().to_real_reg(), ir::types::I64)) + } + &ir::ArgumentPurpose::SignatureId => { + // This is SpiderMonkey's `WasmTableCallSigReg`. + Some(ABIArg::Reg(regs::r10().to_real_reg(), ir::types::I64)) + } + _ => None, + } + } else { + None + } +} + +/// Are we computing information about arguments or return values? Much of the +/// handling is factored out into common routines; this enum allows us to +/// distinguish which case we're handling. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ArgsOrRets { + Args, + Rets, +} + +/// Process a list of parameters or return values and allocate them to X-regs, +/// V-regs, and stack slots. +/// +/// Returns the list of argument locations, the stack-space used (rounded up +/// to a 16-byte-aligned boundary), and if `add_ret_area_ptr` was passed, the +/// index of the extra synthetic arg that was added. +fn compute_arg_locs( + call_conv: isa::CallConv, + params: &[ir::AbiParam], + args_or_rets: ArgsOrRets, + add_ret_area_ptr: bool, +) -> CodegenResult<(Vec, i64, Option)> { + let is_baldrdash = call_conv.extends_baldrdash(); + + // XXX assume SystemV at the moment. + debug_assert!(!is_baldrdash, "baldrdash nyi"); + + let mut next_gpr = 0; + let mut next_vreg = 0; + let mut next_stack: u64 = 0; + let mut ret = vec![]; + + for i in 0..params.len() { + // Process returns backward, according to the SpiderMonkey ABI (which we + // adopt internally if `is_baldrdash` is set). + let param = match (args_or_rets, is_baldrdash) { + (ArgsOrRets::Args, _) => ¶ms[i], + (ArgsOrRets::Rets, false) => ¶ms[i], + (ArgsOrRets::Rets, true) => ¶ms[params.len() - 1 - i], + }; + + // Validate "purpose". + match ¶m.purpose { + &ir::ArgumentPurpose::VMContext + | &ir::ArgumentPurpose::Normal + | &ir::ArgumentPurpose::StackLimit + | &ir::ArgumentPurpose::SignatureId => {} + _ => panic!( + "Unsupported argument purpose {:?} in signature: {:?}", + param.purpose, params + ), + } + + let intreg = in_int_reg(param.value_type); + let vecreg = in_vec_reg(param.value_type); + debug_assert!(intreg || vecreg); + debug_assert!(!(intreg && vecreg)); + + let (next_reg, candidate) = if intreg { + let candidate = match args_or_rets { + ArgsOrRets::Args => get_intreg_for_arg_systemv(next_gpr), + ArgsOrRets::Rets => get_intreg_for_retval_systemv(next_gpr), + }; + debug_assert!(candidate + .map(|r| r.get_class() == RegClass::I64) + .unwrap_or(true)); + (&mut next_gpr, candidate) + } else { + let candidate = match args_or_rets { + ArgsOrRets::Args => get_fltreg_for_arg_systemv(next_vreg), + ArgsOrRets::Rets => get_fltreg_for_retval_systemv(next_vreg), + }; + debug_assert!(candidate + .map(|r| r.get_class() == RegClass::V128) + .unwrap_or(true)); + (&mut next_vreg, candidate) + }; + + if let Some(param) = try_fill_baldrdash_reg(call_conv, param) { + assert!(intreg); + ret.push(param); + } else if let Some(reg) = candidate { + ret.push(ABIArg::Reg(reg.to_real_reg(), param.value_type)); + *next_reg += 1; + } else { + // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte + // stack alignment happens separately after all args.) + let size = (param.value_type.bits() / 8) as u64; + let size = std::cmp::max(size, 8); + // Align. + debug_assert!(size.is_power_of_two()); + next_stack = (next_stack + size - 1) & !(size - 1); + ret.push(ABIArg::Stack(next_stack as i64, param.value_type)); + next_stack += size; + } + } + + if args_or_rets == ArgsOrRets::Rets && is_baldrdash { + ret.reverse(); + } + + let extra_arg = if add_ret_area_ptr { + debug_assert!(args_or_rets == ArgsOrRets::Args); + if let Some(reg) = get_intreg_for_arg_systemv(next_gpr) { + ret.push(ABIArg::Reg(reg.to_real_reg(), ir::types::I64)); + } else { + ret.push(ABIArg::Stack(next_stack as i64, ir::types::I64)); + next_stack += 8; + } + Some(ret.len() - 1) + } else { + None + }; + + next_stack = (next_stack + 15) & !15; + + // To avoid overflow issues, limit the arg/return size to something reasonable. + if next_stack > STACK_ARG_RET_SIZE_LIMIT { + return Err(CodegenError::ImplLimitExceeded); + } + + Ok((ret, next_stack as i64, extra_arg)) +} + +impl ABISig { + fn from_func_sig(sig: &ir::Signature) -> CodegenResult { + // Compute args and retvals from signature. Handle retvals first, + // because we may need to add a return-area arg to the args. + let (rets, stack_ret_space, _) = compute_arg_locs( + sig.call_conv, + &sig.returns, + ArgsOrRets::Rets, + /* extra ret-area ptr = */ false, + )?; + let need_stack_return_area = stack_ret_space > 0; + let (args, stack_arg_space, stack_ret_arg) = compute_arg_locs( + sig.call_conv, + &sig.params, + ArgsOrRets::Args, + need_stack_return_area, + )?; + + trace!( + "ABISig: sig {:?} => args = {:?} rets = {:?} arg stack = {} ret stack = {} stack_ret_arg = {:?}", + sig, + args, + rets, + stack_arg_space, + stack_ret_space, + stack_ret_arg + ); + + Ok(ABISig { + args, + rets, + stack_arg_space, + stack_ret_space, + stack_ret_arg, + call_conv: sig.call_conv, + }) + } +} + +enum CallDest { + ExtName(ir::ExternalName, RelocDistance), + Reg(Reg), +} + +fn adjust_stack>(ctx: &mut C, amount: u64, is_sub: bool) { + if amount == 0 { + return; + } + + let (alu_op, sp_adjustment) = if is_sub { + (AluRmiROpcode::Sub, amount as i64) + } else { + (AluRmiROpcode::Add, -(amount as i64)) + }; + + ctx.emit(Inst::VirtualSPOffsetAdj { + offset: sp_adjustment, + }); + + if amount <= u32::max_value() as u64 { + ctx.emit(Inst::alu_rmi_r( + true, + alu_op, + RegMemImm::imm(amount as u32), + Writable::from_reg(regs::rsp()), + )); + } else { + // TODO will require a scratch register. + unimplemented!("adjust stack with large offset"); + } +} + +fn load_stack(mem: Amode, into_reg: Writable, ty: Type) -> Inst { + let ext_mode = match ty { + types::B1 | types::B8 | types::I8 => Some(ExtMode::BQ), + types::B16 | types::I16 => Some(ExtMode::WQ), + types::B32 | types::I32 => Some(ExtMode::LQ), + types::B64 | types::I64 => None, + types::F32 => todo!("f32 load_stack"), + types::F64 => todo!("f64 load_stack"), + _ => unimplemented!("load_stack({})", ty), + }; + + match ext_mode { + Some(ext_mode) => Inst::movsx_rm_r(ext_mode, RegMem::mem(mem), into_reg), + None => Inst::mov64_m_r(mem, into_reg), + } +} + +fn store_stack(mem: Amode, from_reg: Reg, ty: Type) -> Inst { + let (is_int, size) = match ty { + types::B1 | types::B8 | types::I8 => (true, 1), + types::B16 | types::I16 => (true, 2), + types::B32 | types::I32 => (true, 4), + types::B64 | types::I64 => (true, 8), + types::F32 => (false, 4), + types::F64 => (false, 8), + _ => unimplemented!("store_stack({})", ty), + }; + if is_int { + Inst::mov_r_m(size, from_reg, mem) + } else { + unimplemented!("f32/f64 store_stack"); + } +} + +/// X64 ABI object for a function call. +pub struct X64ABICall { + sig: ABISig, + uses: Vec, + defs: Vec>, + dest: CallDest, + loc: ir::SourceLoc, + opcode: ir::Opcode, +} + +impl X64ABICall { + /// Create a callsite ABI object for a call directly to the specified function. + pub fn from_func( + sig: &ir::Signature, + extname: &ir::ExternalName, + dist: RelocDistance, + loc: ir::SourceLoc, + ) -> CodegenResult { + let sig = ABISig::from_func_sig(sig)?; + let (uses, defs) = abisig_to_uses_and_defs(&sig); + Ok(Self { + sig, + uses, + defs, + dest: CallDest::ExtName(extname.clone(), dist), + loc, + opcode: ir::Opcode::Call, + }) + } + + /// Create a callsite ABI object for a call to a function pointer with the + /// given signature. + pub fn from_ptr( + sig: &ir::Signature, + ptr: Reg, + loc: ir::SourceLoc, + opcode: ir::Opcode, + ) -> CodegenResult { + let sig = ABISig::from_func_sig(sig)?; + let (uses, defs) = abisig_to_uses_and_defs(&sig); + Ok(Self { + sig, + uses, + defs, + dest: CallDest::Reg(ptr), + loc, + opcode, + }) + } +} + +impl ABICall for X64ABICall { + type I = Inst; + + fn num_args(&self) -> usize { + if self.sig.stack_ret_arg.is_some() { + self.sig.args.len() - 1 + } else { + self.sig.args.len() + } + } + + fn emit_stack_pre_adjust>(&self, ctx: &mut C) { + let off = self.sig.stack_arg_space + self.sig.stack_ret_space; + adjust_stack(ctx, off as u64, /* is_sub = */ true) + } + + fn emit_stack_post_adjust>(&self, ctx: &mut C) { + let off = self.sig.stack_arg_space + self.sig.stack_ret_space; + adjust_stack(ctx, off as u64, /* is_sub = */ false) + } + + fn emit_copy_reg_to_arg>( + &self, + ctx: &mut C, + idx: usize, + from_reg: Reg, + ) { + match &self.sig.args[idx] { + &ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move( + Writable::from_reg(reg.to_reg()), + from_reg, + ty, + )), + &ABIArg::Stack(off, ty) => { + debug_assert!(off <= u32::max_value() as i64); + debug_assert!(off >= 0); + ctx.emit(store_stack( + Amode::imm_reg(off as u32, regs::rsp()), + from_reg, + ty, + )) + } + } + } + + fn emit_copy_retval_to_reg>( + &self, + ctx: &mut C, + idx: usize, + into_reg: Writable, + ) { + match &self.sig.rets[idx] { + &ABIArg::Reg(reg, ty) => ctx.emit(Inst::gen_move(into_reg, reg.to_reg(), ty)), + &ABIArg::Stack(off, ty) => { + let ret_area_base = self.sig.stack_arg_space; + let sp_offset = off + ret_area_base; + // TODO handle offsets bigger than u32::max + debug_assert!(sp_offset >= 0); + debug_assert!(sp_offset <= u32::max_value() as i64); + ctx.emit(load_stack( + Amode::imm_reg(sp_offset as u32, regs::rsp()), + into_reg, + ty, + )); + } + } + } + + fn emit_call>(&mut self, ctx: &mut C) { + let (uses, defs) = ( + mem::replace(&mut self.uses, Default::default()), + mem::replace(&mut self.defs, Default::default()), + ); + + if let Some(i) = self.sig.stack_ret_arg { + let dst = ctx.alloc_tmp(RegClass::I64, I64); + let ret_area_base = self.sig.stack_arg_space; + debug_assert!( + ret_area_base <= u32::max_value() as i64, + "large offset for ret area NYI" + ); + ctx.emit(Inst::lea( + Amode::imm_reg(ret_area_base as u32, regs::rsp()), + dst, + )); + self.emit_copy_reg_to_arg(ctx, i, dst.to_reg()); + } + + match &self.dest { + &CallDest::ExtName(ref name, ref _reloc_distance) => ctx.emit(Inst::call_known( + name.clone(), + uses, + defs, + self.loc, + self.opcode, + )), + &CallDest::Reg(reg) => ctx.emit(Inst::call_unknown( + RegMem::reg(reg), + uses, + defs, + self.loc, + self.opcode, + )), + } + } +} diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/inst/args.rs b/third_party/rust/cranelift-codegen/src/isa/x64/inst/args.rs index 6f4b52156fd8..a19874a92325 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/inst/args.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/inst/args.rs @@ -3,20 +3,25 @@ use std::fmt; use std::string::{String, ToString}; -use regalloc::{RealRegUniverse, Reg, RegClass, RegUsageCollector}; +use regalloc::{RealRegUniverse, Reg, RegClass, RegUsageCollector, RegUsageMapper}; +use crate::ir::condcodes::IntCC; use crate::machinst::*; -use super::regs::show_ireg_sized; +use super::{ + regs::{self, show_ireg_sized}, + EmitState, +}; -/// A Memory Address. These denote a 64-bit value only. +/// A possible addressing mode (amode) that can be used in instructions. +/// These denote a 64-bit value only. #[derive(Clone)] -pub(crate) enum Addr { +pub enum Amode { /// Immediate sign-extended and a Register. - IR { simm32: u32, base: Reg }, + ImmReg { simm32: u32, base: Reg }, /// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift) - IRRS { + ImmRegRegShift { simm32: u32, base: Reg, index: Reg, @@ -24,19 +29,17 @@ pub(crate) enum Addr { }, } -impl Addr { - // Constructors. - +impl Amode { pub(crate) fn imm_reg(simm32: u32, base: Reg) -> Self { debug_assert!(base.get_class() == RegClass::I64); - Self::IR { simm32, base } + Self::ImmReg { simm32, base } } pub(crate) fn imm_reg_reg_shift(simm32: u32, base: Reg, index: Reg, shift: u8) -> Self { debug_assert!(base.get_class() == RegClass::I64); debug_assert!(index.get_class() == RegClass::I64); debug_assert!(shift <= 3); - Addr::IRRS { + Self::ImmRegRegShift { simm32, base, index, @@ -47,15 +50,10 @@ impl Addr { /// Add the regs mentioned by `self` to `collector`. pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { match self { - Addr::IR { simm32: _, base } => { + Amode::ImmReg { base, .. } => { collector.add_use(*base); } - Addr::IRRS { - simm32: _, - base, - index, - shift: _, - } => { + Amode::ImmRegRegShift { base, index, .. } => { collector.add_use(*base); collector.add_use(*index); } @@ -63,11 +61,13 @@ impl Addr { } } -impl ShowWithRRU for Addr { +impl ShowWithRRU for Amode { fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { match self { - Addr::IR { simm32, base } => format!("{}({})", *simm32 as i32, base.show_rru(mb_rru)), - Addr::IRRS { + Amode::ImmReg { simm32, base } => { + format!("{}({})", *simm32 as i32, base.show_rru(mb_rru)) + } + Amode::ImmRegRegShift { simm32, base, index, @@ -83,51 +83,119 @@ impl ShowWithRRU for Addr { } } -/// An operand which is either an integer Register, a value in Memory or an Immediate. This can -/// denote an 8, 16, 32 or 64 bit value. For the Immediate form, in the 8- and 16-bit case, only -/// the lower 8 or 16 bits of `simm32` is relevant. In the 64-bit case, the value denoted by -/// `simm32` is its sign-extension out to 64 bits. +/// A Memory Address. These denote a 64-bit value only. +/// Used for usual addressing modes as well as addressing modes used during compilation, when the +/// moving SP offset is not known. #[derive(Clone)] -pub(crate) enum RMI { - R { reg: Reg }, - M { addr: Addr }, - I { simm32: u32 }, +pub enum SyntheticAmode { + /// A real amode. + Real(Amode), + + /// A (virtual) offset to the "nominal SP" value, which will be recomputed as we push and pop + /// within the function. + NominalSPOffset { simm32: u32 }, } -impl RMI { - // Constructors - - pub(crate) fn reg(reg: Reg) -> RMI { - debug_assert!(reg.get_class() == RegClass::I64); - RMI::R { reg } - } - pub(crate) fn mem(addr: Addr) -> RMI { - RMI::M { addr } - } - pub(crate) fn imm(simm32: u32) -> RMI { - RMI::I { simm32 } +impl SyntheticAmode { + pub(crate) fn nominal_sp_offset(simm32: u32) -> Self { + SyntheticAmode::NominalSPOffset { simm32 } } /// Add the regs mentioned by `self` to `collector`. pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { match self { - RMI::R { reg } => collector.add_use(*reg), - RMI::M { addr } => addr.get_regs_as_uses(collector), - RMI::I { simm32: _ } => {} + SyntheticAmode::Real(addr) => addr.get_regs_as_uses(collector), + SyntheticAmode::NominalSPOffset { .. } => { + // Nothing to do; the base is SP and isn't involved in regalloc. + } + } + } + + pub(crate) fn map_uses(&mut self, map: &RUM) { + match self { + SyntheticAmode::Real(addr) => addr.map_uses(map), + SyntheticAmode::NominalSPOffset { .. } => { + // Nothing to do. + } + } + } + + pub(crate) fn finalize(&self, state: &mut EmitState) -> Amode { + match self { + SyntheticAmode::Real(addr) => addr.clone(), + SyntheticAmode::NominalSPOffset { simm32 } => { + let off = *simm32 as i64 + state.virtual_sp_offset; + // TODO will require a sequence of add etc. + assert!( + off <= u32::max_value() as i64, + "amode finalize: add sequence NYI" + ); + Amode::imm_reg(off as u32, regs::rsp()) + } } } } -impl ShowWithRRU for RMI { +impl Into for Amode { + fn into(self) -> SyntheticAmode { + SyntheticAmode::Real(self) + } +} + +impl ShowWithRRU for SyntheticAmode { + fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { + match self { + SyntheticAmode::Real(addr) => addr.show_rru(mb_rru), + SyntheticAmode::NominalSPOffset { simm32 } => { + format!("rsp({} + virtual offset)", *simm32 as i32) + } + } + } +} + +/// An operand which is either an integer Register, a value in Memory or an Immediate. This can +/// denote an 8, 16, 32 or 64 bit value. For the Immediate form, in the 8- and 16-bit case, only +/// the lower 8 or 16 bits of `simm32` is relevant. In the 64-bit case, the value denoted by +/// `simm32` is its sign-extension out to 64 bits. +#[derive(Clone)] +pub enum RegMemImm { + Reg { reg: Reg }, + Mem { addr: SyntheticAmode }, + Imm { simm32: u32 }, +} + +impl RegMemImm { + pub(crate) fn reg(reg: Reg) -> Self { + debug_assert!(reg.get_class() == RegClass::I64); + Self::Reg { reg } + } + pub(crate) fn mem(addr: impl Into) -> Self { + Self::Mem { addr: addr.into() } + } + pub(crate) fn imm(simm32: u32) -> Self { + Self::Imm { simm32 } + } + + /// Add the regs mentioned by `self` to `collector`. + pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { + match self { + Self::Reg { reg } => collector.add_use(*reg), + Self::Mem { addr } => addr.get_regs_as_uses(collector), + Self::Imm { simm32: _ } => {} + } + } +} + +impl ShowWithRRU for RegMemImm { fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { self.show_rru_sized(mb_rru, 8) } fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { match self { - RMI::R { reg } => show_ireg_sized(*reg, mb_rru, size), - RMI::M { addr } => addr.show_rru(mb_rru), - RMI::I { simm32 } => format!("${}", *simm32 as i32), + Self::Reg { reg } => show_ireg_sized(*reg, mb_rru, size), + Self::Mem { addr } => addr.show_rru(mb_rru), + Self::Imm { simm32 } => format!("${}", *simm32 as i32), } } } @@ -135,48 +203,45 @@ impl ShowWithRRU for RMI { /// An operand which is either an integer Register or a value in Memory. This can denote an 8, 16, /// 32 or 64 bit value. #[derive(Clone)] -pub(crate) enum RM { - R { reg: Reg }, - M { addr: Addr }, +pub enum RegMem { + Reg { reg: Reg }, + Mem { addr: SyntheticAmode }, } -impl RM { - // Constructors. - +impl RegMem { pub(crate) fn reg(reg: Reg) -> Self { debug_assert!(reg.get_class() == RegClass::I64 || reg.get_class() == RegClass::V128); - RM::R { reg } + Self::Reg { reg } } - - pub(crate) fn mem(addr: Addr) -> Self { - RM::M { addr } + pub(crate) fn mem(addr: impl Into) -> Self { + Self::Mem { addr: addr.into() } } /// Add the regs mentioned by `self` to `collector`. pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { match self { - RM::R { reg } => collector.add_use(*reg), - RM::M { addr } => addr.get_regs_as_uses(collector), + RegMem::Reg { reg } => collector.add_use(*reg), + RegMem::Mem { addr } => addr.get_regs_as_uses(collector), } } } -impl ShowWithRRU for RM { +impl ShowWithRRU for RegMem { fn show_rru(&self, mb_rru: Option<&RealRegUniverse>) -> String { self.show_rru_sized(mb_rru, 8) } fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { match self { - RM::R { reg } => show_ireg_sized(*reg, mb_rru, size), - RM::M { addr } => addr.show_rru(mb_rru), + RegMem::Reg { reg } => show_ireg_sized(*reg, mb_rru, size), + RegMem::Mem { addr } => addr.show_rru(mb_rru), } } } /// Some basic ALU operations. TODO: maybe add Adc, Sbb. #[derive(Clone, PartialEq)] -pub enum RMI_R_Op { +pub enum AluRmiROpcode { Add, Sub, And, @@ -186,89 +251,186 @@ pub enum RMI_R_Op { Mul, } -impl RMI_R_Op { - pub(crate) fn to_string(&self) -> String { +impl fmt::Debug for AluRmiROpcode { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let name = match self { + AluRmiROpcode::Add => "add", + AluRmiROpcode::Sub => "sub", + AluRmiROpcode::And => "and", + AluRmiROpcode::Or => "or", + AluRmiROpcode::Xor => "xor", + AluRmiROpcode::Mul => "imul", + }; + write!(fmt, "{}", name) + } +} + +impl ToString for AluRmiROpcode { + fn to_string(&self) -> String { + format!("{:?}", self) + } +} + +pub(crate) enum InstructionSet { + SSE, + SSE2, + SSE41, +} + +/// Some scalar SSE operations requiring 2 operands r/m and r. +/// TODO: Below only includes scalar operations. To be seen if packed will be added here. +#[derive(Clone, PartialEq)] +pub enum SseOpcode { + Addss, + Addsd, + Andps, + Andnps, + Comiss, + Comisd, + Cmpss, + Cmpsd, + Cvtsd2ss, + Cvtsd2si, + Cvtsi2ss, + Cvtsi2sd, + Cvtss2si, + Cvtss2sd, + Cvttss2si, + Cvttsd2si, + Divss, + Divsd, + Insertps, + Maxss, + Maxsd, + Minss, + Minsd, + Movaps, + Movd, + Movss, + Movsd, + Mulss, + Mulsd, + Orps, + Rcpss, + Roundss, + Roundsd, + Rsqrtss, + Sqrtss, + Sqrtsd, + Subss, + Subsd, + Ucomiss, + Ucomisd, +} + +impl SseOpcode { + /// Which `InstructionSet` is the first supporting this opcode? + pub(crate) fn available_from(&self) -> InstructionSet { + use InstructionSet::*; match self { - RMI_R_Op::Add => "add".to_string(), - RMI_R_Op::Sub => "sub".to_string(), - RMI_R_Op::And => "and".to_string(), - RMI_R_Op::Or => "or".to_string(), - RMI_R_Op::Xor => "xor".to_string(), - RMI_R_Op::Mul => "imul".to_string(), + SseOpcode::Addss + | SseOpcode::Andps + | SseOpcode::Andnps + | SseOpcode::Cvtsi2ss + | SseOpcode::Cvtss2si + | SseOpcode::Cvttss2si + | SseOpcode::Divss + | SseOpcode::Maxss + | SseOpcode::Movaps + | SseOpcode::Minss + | SseOpcode::Movss + | SseOpcode::Mulss + | SseOpcode::Orps + | SseOpcode::Rcpss + | SseOpcode::Rsqrtss + | SseOpcode::Subss + | SseOpcode::Ucomiss + | SseOpcode::Sqrtss + | SseOpcode::Comiss + | SseOpcode::Cmpss => SSE, + + SseOpcode::Addsd + | SseOpcode::Cvtsd2ss + | SseOpcode::Cvtsd2si + | SseOpcode::Cvtsi2sd + | SseOpcode::Cvtss2sd + | SseOpcode::Cvttsd2si + | SseOpcode::Divsd + | SseOpcode::Maxsd + | SseOpcode::Minsd + | SseOpcode::Movd + | SseOpcode::Movsd + | SseOpcode::Mulsd + | SseOpcode::Sqrtsd + | SseOpcode::Subsd + | SseOpcode::Ucomisd + | SseOpcode::Comisd + | SseOpcode::Cmpsd => SSE2, + + SseOpcode::Insertps | SseOpcode::Roundss | SseOpcode::Roundsd => SSE41, + } + } + + /// Returns the src operand size for an instruction + pub(crate) fn src_size(&self) -> u8 { + match self { + SseOpcode::Movd => 4, + _ => 8, } } } -impl fmt::Debug for RMI_R_Op { +impl fmt::Debug for SseOpcode { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "{}", self.to_string()) + let name = match self { + SseOpcode::Addss => "addss", + SseOpcode::Addsd => "addsd", + SseOpcode::Andps => "andps", + SseOpcode::Andnps => "andnps", + SseOpcode::Comiss => "comiss", + SseOpcode::Comisd => "comisd", + SseOpcode::Cvtsd2ss => "cvtsd2ss", + SseOpcode::Cvtsd2si => "cvtsd2si", + SseOpcode::Cvtsi2ss => "cvtsi2ss", + SseOpcode::Cvtsi2sd => "cvtsi2sd", + SseOpcode::Cvtss2si => "cvtss2si", + SseOpcode::Cvtss2sd => "cvtss2sd", + SseOpcode::Cvttss2si => "cvttss2si", + SseOpcode::Cvttsd2si => "cvttsd2si", + SseOpcode::Divss => "divss", + SseOpcode::Divsd => "divsd", + SseOpcode::Maxss => "maxss", + SseOpcode::Maxsd => "maxsd", + SseOpcode::Minss => "minss", + SseOpcode::Minsd => "minsd", + SseOpcode::Movaps => "movaps", + SseOpcode::Movd => "movd", + SseOpcode::Movss => "movss", + SseOpcode::Movsd => "movsd", + SseOpcode::Mulss => "mulss", + SseOpcode::Mulsd => "mulsd", + SseOpcode::Orps => "orps", + SseOpcode::Rcpss => "rcpss", + SseOpcode::Roundss => "roundss", + SseOpcode::Roundsd => "roundsd", + SseOpcode::Rsqrtss => "rsqrtss", + SseOpcode::Sqrtss => "sqrtss", + SseOpcode::Sqrtsd => "sqrtsd", + SseOpcode::Subss => "subss", + SseOpcode::Subsd => "subsd", + SseOpcode::Ucomiss => "ucomiss", + SseOpcode::Ucomisd => "ucomisd", + SseOpcode::Cmpss => "cmpss", + SseOpcode::Cmpsd => "cmpsd", + SseOpcode::Insertps => "insertps", + }; + write!(fmt, "{}", name) } } -/// Some scalar SSE operations requiring 2 operands r/m and r -/// Each instruction is prefixed with the SSE version that introduced -/// the particular instructions. -/// TODO: Below only includes scalar operations. To be seen if packed will -/// be added here. -#[derive(Clone, PartialEq)] -pub enum SSE_Op { - SSE_Addss, - SSE2_Addsd, - SSE_Comiss, - SSE2_Comisd, - SSE2_Cvtsd2ss, - SSE2_Cvtsd2si, - SSE_Cvtsi2ss, - SSE2_Cvtsi2sd, - SSE_Cvtss2si, - SSE2_Cvtss2sd, - SSE_Cvttss2si, - SSE2_Cvttsd2si, - SSE_Divss, - SSE2_Divsd, - SSE_Maxss, - SSE2_Maxsd, - SSE_Minss, - SSE2_Minsd, - SSE_Movss, - SSE2_Movsd, - SSE_Mulss, - SSE2_Mulsd, - SSE_Rcpss, - SSE41_Roundss, - SSE41_Roundsd, - SSE_Rsqrtss, - SSE_Sqrtss, - SSE2_Sqrtsd, - SSE_Subss, - SSE2_Subsd, - SSE_Ucomiss, - SSE2_Ucomisd, -} - -/// Some SSE operations requiring 3 operands i, r/m, and r -#[derive(Clone, PartialEq)] -pub enum SSE_RMI_Op { - SSE_Cmpss, - SSE2_Cmpsd, - SSE41_Insertps, -} - -impl SSE_Op { - pub(crate) fn to_string(&self) -> String { - match self { - SSE_Op::SSE_Addss => "addss".to_string(), - SSE_Op::SSE_Subss => "subss".to_string(), - SSE_Op::SSE_Movss => "movss".to_string(), - SSE_Op::SSE2_Movsd => "movsd".to_string(), - _ => "unimplemented sse_op".to_string(), - } - } -} - -impl fmt::Debug for SSE_Op { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "{}", self.to_string()) +impl ToString for SseOpcode { + fn to_string(&self) -> String { + format!("{:?}", self) } } @@ -289,30 +451,37 @@ pub enum ExtMode { } impl ExtMode { - pub(crate) fn to_string(&self) -> String { + pub(crate) fn src_size(&self) -> u8 { match self { - ExtMode::BL => "bl".to_string(), - ExtMode::BQ => "bq".to_string(), - ExtMode::WL => "wl".to_string(), - ExtMode::WQ => "wq".to_string(), - ExtMode::LQ => "lq".to_string(), + ExtMode::BL | ExtMode::BQ => 1, + ExtMode::WL | ExtMode::WQ => 2, + ExtMode::LQ => 4, } } - pub(crate) fn dst_size(&self) -> u8 { match self { - ExtMode::BL => 4, - ExtMode::BQ => 8, - ExtMode::WL => 4, - ExtMode::WQ => 8, - ExtMode::LQ => 8, + ExtMode::BL | ExtMode::WL => 4, + ExtMode::BQ | ExtMode::WQ | ExtMode::LQ => 8, } } } impl fmt::Debug for ExtMode { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "{}", self.to_string()) + let name = match self { + ExtMode::BL => "bl", + ExtMode::BQ => "bq", + ExtMode::WL => "wl", + ExtMode::WQ => "wq", + ExtMode::LQ => "lq", + }; + write!(fmt, "{}", name) + } +} + +impl ToString for ExtMode { + fn to_string(&self) -> String { + format!("{:?}", self) } } @@ -324,19 +493,20 @@ pub enum ShiftKind { RightS, } -impl ShiftKind { - pub(crate) fn to_string(&self) -> String { - match self { - ShiftKind::Left => "shl".to_string(), - ShiftKind::RightZ => "shr".to_string(), - ShiftKind::RightS => "sar".to_string(), - } +impl fmt::Debug for ShiftKind { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + let name = match self { + ShiftKind::Left => "shl", + ShiftKind::RightZ => "shr", + ShiftKind::RightS => "sar", + }; + write!(fmt, "{}", name) } } -impl fmt::Debug for ShiftKind { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "{}", self.to_string()) +impl ToString for ShiftKind { + fn to_string(&self) -> String { + format!("{:?}", self) } } @@ -382,26 +552,24 @@ pub enum CC { } impl CC { - pub(crate) fn to_string(&self) -> String { - match self { - CC::O => "o".to_string(), - CC::NO => "no".to_string(), - CC::B => "b".to_string(), - CC::NB => "nb".to_string(), - CC::Z => "z".to_string(), - CC::NZ => "nz".to_string(), - CC::BE => "be".to_string(), - CC::NBE => "nbe".to_string(), - CC::S => "s".to_string(), - CC::NS => "ns".to_string(), - CC::L => "l".to_string(), - CC::NL => "nl".to_string(), - CC::LE => "le".to_string(), - CC::NLE => "nle".to_string(), + pub(crate) fn from_intcc(intcc: IntCC) -> Self { + match intcc { + IntCC::Equal => CC::Z, + IntCC::NotEqual => CC::NZ, + IntCC::SignedGreaterThanOrEqual => CC::NL, + IntCC::SignedGreaterThan => CC::NLE, + IntCC::SignedLessThanOrEqual => CC::LE, + IntCC::SignedLessThan => CC::L, + IntCC::UnsignedGreaterThanOrEqual => CC::NB, + IntCC::UnsignedGreaterThan => CC::NBE, + IntCC::UnsignedLessThanOrEqual => CC::BE, + IntCC::UnsignedLessThan => CC::B, + IntCC::Overflow => CC::O, + IntCC::NotOverflow => CC::NO, } } - pub(crate) fn invert(&self) -> CC { + pub(crate) fn invert(&self) -> Self { match self { CC::O => CC::NO, CC::NO => CC::O, @@ -433,7 +601,29 @@ impl CC { impl fmt::Debug for CC { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "{}", self.to_string()) + let name = match self { + CC::O => "o", + CC::NO => "no", + CC::B => "b", + CC::NB => "nb", + CC::Z => "z", + CC::NZ => "nz", + CC::BE => "be", + CC::NBE => "nbe", + CC::S => "s", + CC::NS => "ns", + CC::L => "l", + CC::NL => "nl", + CC::LE => "le", + CC::NLE => "nle", + }; + write!(fmt, "{}", name) + } +} + +impl ToString for CC { + fn to_string(&self) -> String { + format!("{:?}", self) } } diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit.rs b/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit.rs index 77ed7fcb4e31..d2666728c3b0 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit.rs @@ -1,12 +1,15 @@ -use crate::isa::x64::inst::*; +use log::debug; use regalloc::Reg; -fn low8willSXto64(x: u32) -> bool { +use crate::binemit::Reloc; +use crate::isa::x64::inst::*; + +fn low8_will_sign_extend_to_64(x: u32) -> bool { let xs = (x as i32) as i64; xs == ((xs << 56) >> 56) } -fn low8willSXto32(x: u32) -> bool { +fn low8_will_sign_extend_to_32(x: u32) -> bool { let xs = x as i32; xs == ((xs << 24) >> 24) } @@ -21,266 +24,310 @@ fn low8willSXto32(x: u32) -> bool { // "enc" in the following means "hardware register encoding number". #[inline(always)] -fn mkModRegRM(m0d: u8, encRegG: u8, rmE: u8) -> u8 { +fn encode_modrm(m0d: u8, enc_reg_g: u8, rm_e: u8) -> u8 { debug_assert!(m0d < 4); - debug_assert!(encRegG < 8); - debug_assert!(rmE < 8); - ((m0d & 3) << 6) | ((encRegG & 7) << 3) | (rmE & 7) + debug_assert!(enc_reg_g < 8); + debug_assert!(rm_e < 8); + ((m0d & 3) << 6) | ((enc_reg_g & 7) << 3) | (rm_e & 7) } #[inline(always)] -fn mkSIB(shift: u8, encIndex: u8, encBase: u8) -> u8 { +fn encode_sib(shift: u8, enc_index: u8, enc_base: u8) -> u8 { debug_assert!(shift < 4); - debug_assert!(encIndex < 8); - debug_assert!(encBase < 8); - ((shift & 3) << 6) | ((encIndex & 7) << 3) | (encBase & 7) + debug_assert!(enc_index < 8); + debug_assert!(enc_base < 8); + ((shift & 3) << 6) | ((enc_index & 7) << 3) | (enc_base & 7) } -/// Get the encoding number from something which we sincerely hope is a real -/// register of class I64. +/// Get the encoding number of a GPR. #[inline(always)] -fn iregEnc(reg: Reg) -> u8 { +fn int_reg_enc(reg: Reg) -> u8 { + debug_assert!(reg.is_real()); + debug_assert_eq!(reg.get_class(), RegClass::I64); + reg.get_hw_encoding() +} + +/// Get the encoding number of any register. +#[inline(always)] +fn reg_enc(reg: Reg) -> u8 { debug_assert!(reg.is_real()); reg.get_hw_encoding() } -// F_*: these flags describe special handling of the insn to be generated. Be -// careful with these. It is easy to create nonsensical combinations. -const F_NONE: u32 = 0; +/// A small bit field to record a REX prefix specification: +/// - bit 0 set to 1 indicates REX.W must be 0 (cleared). +/// - bit 1 set to 1 indicates the REX prefix must always be emitted. +#[repr(transparent)] +#[derive(Clone, Copy)] +struct RexFlags(u8); -/// Emit the REX prefix byte even if it appears to be redundant (== 0x40). -const F_RETAIN_REDUNDANT_REX: u32 = 1; +impl RexFlags { + /// By default, set the W field, and don't always emit. + #[inline(always)] + fn set_w() -> Self { + Self(0) + } + /// Creates a new RexPrefix for which the REX.W bit will be cleared. + #[inline(always)] + fn clear_w() -> Self { + Self(1) + } -/// Set the W bit in the REX prefix to zero. By default it will be set to 1, -/// indicating a 64-bit operation. -const F_CLEAR_REX_W: u32 = 2; + #[inline(always)] + fn always_emit(&mut self) -> &mut Self { + self.0 = self.0 | 2; + self + } -/// For specifying the legacy prefixes (or `PfxNone` if no prefix required) to -/// be used at the start an instruction. A select prefix may be required for + #[inline(always)] + fn must_clear_w(&self) -> bool { + (self.0 & 1) != 0 + } + #[inline(always)] + fn must_always_emit(&self) -> bool { + (self.0 & 2) != 0 + } + + #[inline(always)] + fn emit_two_op(&self, sink: &mut MachBuffer, enc_g: u8, enc_e: u8) { + let w = if self.must_clear_w() { 0 } else { 1 }; + let r = (enc_g >> 3) & 1; + let x = 0; + let b = (enc_e >> 3) & 1; + let rex = 0x40 | (w << 3) | (r << 2) | (x << 1) | b; + if rex != 0x40 || self.must_always_emit() { + sink.put1(rex); + } + } + + #[inline(always)] + fn emit_three_op(&self, sink: &mut MachBuffer, enc_g: u8, enc_index: u8, enc_base: u8) { + let w = if self.must_clear_w() { 0 } else { 1 }; + let r = (enc_g >> 3) & 1; + let x = (enc_index >> 3) & 1; + let b = (enc_base >> 3) & 1; + let rex = 0x40 | (w << 3) | (r << 2) | (x << 1) | b; + if rex != 0x40 || self.must_always_emit() { + sink.put1(rex); + } + } +} + +/// For specifying the legacy prefixes (or `None` if no prefix required) to +/// be used at the start an instruction. A given prefix may be required for /// various operations, including instructions that operate on GPR, SSE, and Vex /// registers. enum LegacyPrefix { - PfxNone, - Pfx66, - PfxF2, - PfxF3, + None, + _66, + _F2, + _F3, } + +impl LegacyPrefix { + #[inline(always)] + fn emit(&self, sink: &mut MachBuffer) { + match self { + LegacyPrefix::_66 => sink.put1(0x66), + LegacyPrefix::_F2 => sink.put1(0xF2), + LegacyPrefix::_F3 => sink.put1(0xF3), + LegacyPrefix::None => (), + } + } +} + /// This is the core 'emit' function for instructions that reference memory. /// -/// For an instruction that has as operands a register `encG` and a memory -/// address `memE`, create and emit, first the REX prefix, then caller-supplied -/// opcode byte(s) (`opcodes` and `numOpcodes`), then the MOD/RM byte, then -/// optionally, a SIB byte, and finally optionally an immediate that will be -/// derived from the `memE` operand. For most instructions up to and including -/// SSE4.2, that will be the whole instruction. +/// For an instruction that has as operands a reg encoding `enc_g` and a memory address `mem_e`, +/// create and emit: +/// - first the REX prefix, +/// - then caller-supplied opcode byte(s) (`opcodes` and `num_opcodes`), +/// - then the MOD/RM byte, +/// - then optionally, a SIB byte, +/// - and finally optionally an immediate that will be derived from the `mem_e` operand. /// -/// The opcodes are written bigendianly for the convenience of callers. For -/// example, if the opcode bytes to be emitted are, in this order, F3 0F 27, -/// then the caller should pass `opcodes` == 0xF3_0F_27 and `numOpcodes` == 3. +/// For most instructions up to and including SSE4.2, that will be the whole instruction: this is +/// what we call "standard" instructions, and abbreviate "std" in the name here. VEX instructions +/// will require their own emitter functions. /// -/// The register operand is represented here not as a `Reg` but as its hardware -/// encoding, `encG`. `flags` can specify special handling for the REX prefix. -/// By default, the REX prefix will indicate a 64-bit operation and will be -/// deleted if it is redundant (0x40). Note that for a 64-bit operation, the -/// REX prefix will normally never be redundant, since REX.W must be 1 to +/// This will also work for 32-bits x86 instructions, assuming no REX prefix is provided. +/// +/// The opcodes are written bigendianly for the convenience of callers. For example, if the opcode +/// bytes to be emitted are, in this order, F3 0F 27, then the caller should pass `opcodes` == +/// 0xF3_0F_27 and `num_opcodes` == 3. +/// +/// The register operand is represented here not as a `Reg` but as its hardware encoding, `enc_g`. +/// `rex` can specify special handling for the REX prefix. By default, the REX prefix will +/// indicate a 64-bit operation and will be deleted if it is redundant (0x40). Note that for a +/// 64-bit operation, the REX prefix will normally never be redundant, since REX.W must be 1 to /// indicate a 64-bit operation. -fn emit_REX_OPCODES_MODRM_SIB_IMM_encG_memE( +fn emit_std_enc_mem( sink: &mut MachBuffer, prefix: LegacyPrefix, opcodes: u32, - mut numOpcodes: usize, - encG: u8, - memE: &Addr, - flags: u32, + mut num_opcodes: usize, + enc_g: u8, + mem_e: &Amode, + rex: RexFlags, ) { - // General comment for this function: the registers in `memE` must be + // General comment for this function: the registers in `mem_e` must be // 64-bit integer registers, because they are part of an address - // expression. But `encG` can be derived from a register of any class. - let clearRexW = (flags & F_CLEAR_REX_W) != 0; - let retainRedundant = (flags & F_RETAIN_REDUNDANT_REX) != 0; + // expression. But `enc_g` can be derived from a register of any class. + + prefix.emit(sink); + + match mem_e { + Amode::ImmReg { simm32, base } => { + // First, the REX byte. + let enc_e = int_reg_enc(*base); + rex.emit_two_op(sink, enc_g, enc_e); - // Lower the prefix if applicable. - match prefix { - LegacyPrefix::Pfx66 => sink.put1(0x66), - LegacyPrefix::PfxF2 => sink.put1(0xF2), - LegacyPrefix::PfxF3 => sink.put1(0xF3), - LegacyPrefix::PfxNone => (), - } - match memE { - Addr::IR { simm32, base: regE } => { - // First, cook up the REX byte. This is easy. - let encE = iregEnc(*regE); - let w = if clearRexW { 0 } else { 1 }; - let r = (encG >> 3) & 1; - let x = 0; - let b = (encE >> 3) & 1; - let rex = 0x40 | (w << 3) | (r << 2) | (x << 1) | b; - if rex != 0x40 || retainRedundant { - sink.put1(rex); - } // Now the opcode(s). These include any other prefixes the caller // hands to us. - while numOpcodes > 0 { - numOpcodes -= 1; - sink.put1(((opcodes >> (numOpcodes << 3)) & 0xFF) as u8); + while num_opcodes > 0 { + num_opcodes -= 1; + sink.put1(((opcodes >> (num_opcodes << 3)) & 0xFF) as u8); } + // Now the mod/rm and associated immediates. This is // significantly complicated due to the multiple special cases. if *simm32 == 0 - && encE != regs::ENC_RSP - && encE != regs::ENC_RBP - && encE != regs::ENC_R12 - && encE != regs::ENC_R13 + && enc_e != regs::ENC_RSP + && enc_e != regs::ENC_RBP + && enc_e != regs::ENC_R12 + && enc_e != regs::ENC_R13 { // FIXME JRS 2020Feb11: those four tests can surely be // replaced by a single mask-and-compare check. We should do // that because this routine is likely to be hot. - sink.put1(mkModRegRM(0, encG & 7, encE & 7)); - } else if *simm32 == 0 && (encE == regs::ENC_RSP || encE == regs::ENC_R12) { - sink.put1(mkModRegRM(0, encG & 7, 4)); + sink.put1(encode_modrm(0, enc_g & 7, enc_e & 7)); + } else if *simm32 == 0 && (enc_e == regs::ENC_RSP || enc_e == regs::ENC_R12) { + sink.put1(encode_modrm(0, enc_g & 7, 4)); sink.put1(0x24); - } else if low8willSXto32(*simm32) && encE != regs::ENC_RSP && encE != regs::ENC_R12 { - sink.put1(mkModRegRM(1, encG & 7, encE & 7)); + } else if low8_will_sign_extend_to_32(*simm32) + && enc_e != regs::ENC_RSP + && enc_e != regs::ENC_R12 + { + sink.put1(encode_modrm(1, enc_g & 7, enc_e & 7)); sink.put1((simm32 & 0xFF) as u8); - } else if encE != regs::ENC_RSP && encE != regs::ENC_R12 { - sink.put1(mkModRegRM(2, encG & 7, encE & 7)); + } else if enc_e != regs::ENC_RSP && enc_e != regs::ENC_R12 { + sink.put1(encode_modrm(2, enc_g & 7, enc_e & 7)); sink.put4(*simm32); - } else if (encE == regs::ENC_RSP || encE == regs::ENC_R12) && low8willSXto32(*simm32) { + } else if (enc_e == regs::ENC_RSP || enc_e == regs::ENC_R12) + && low8_will_sign_extend_to_32(*simm32) + { // REX.B distinguishes RSP from R12 - sink.put1(mkModRegRM(1, encG & 7, 4)); + sink.put1(encode_modrm(1, enc_g & 7, 4)); sink.put1(0x24); sink.put1((simm32 & 0xFF) as u8); - } else if encE == regs::ENC_R12 || encE == regs::ENC_RSP { + } else if enc_e == regs::ENC_R12 || enc_e == regs::ENC_RSP { //.. wait for test case for RSP case // REX.B distinguishes RSP from R12 - sink.put1(mkModRegRM(2, encG & 7, 4)); + sink.put1(encode_modrm(2, enc_g & 7, 4)); sink.put1(0x24); sink.put4(*simm32); } else { - unreachable!("emit_REX_OPCODES_MODRM_SIB_IMM_encG_memE: IR"); + unreachable!("ImmReg"); } } - // Bizarrely, the IRRS case is much simpler. - Addr::IRRS { + + Amode::ImmRegRegShift { simm32, - base: regBase, - index: regIndex, + base: reg_base, + index: reg_index, shift, } => { - let encBase = iregEnc(*regBase); - let encIndex = iregEnc(*regIndex); - // The rex byte - let w = if clearRexW { 0 } else { 1 }; - let r = (encG >> 3) & 1; - let x = (encIndex >> 3) & 1; - let b = (encBase >> 3) & 1; - let rex = 0x40 | (w << 3) | (r << 2) | (x << 1) | b; - if rex != 0x40 || retainRedundant { - sink.put1(rex); + let enc_base = int_reg_enc(*reg_base); + let enc_index = int_reg_enc(*reg_index); + + // The rex byte. + rex.emit_three_op(sink, enc_g, enc_index, enc_base); + + // All other prefixes and opcodes. + while num_opcodes > 0 { + num_opcodes -= 1; + sink.put1(((opcodes >> (num_opcodes << 3)) & 0xFF) as u8); } - // All other prefixes and opcodes - while numOpcodes > 0 { - numOpcodes -= 1; - sink.put1(((opcodes >> (numOpcodes << 3)) & 0xFF) as u8); - } - // modrm, SIB, immediates - if low8willSXto32(*simm32) && encIndex != regs::ENC_RSP { - sink.put1(mkModRegRM(1, encG & 7, 4)); - sink.put1(mkSIB(*shift, encIndex & 7, encBase & 7)); + + // modrm, SIB, immediates. + if low8_will_sign_extend_to_32(*simm32) && enc_index != regs::ENC_RSP { + sink.put1(encode_modrm(1, enc_g & 7, 4)); + sink.put1(encode_sib(*shift, enc_index & 7, enc_base & 7)); sink.put1(*simm32 as u8); - } else if encIndex != regs::ENC_RSP { - sink.put1(mkModRegRM(2, encG & 7, 4)); - sink.put1(mkSIB(*shift, encIndex & 7, encBase & 7)); + } else if enc_index != regs::ENC_RSP { + sink.put1(encode_modrm(2, enc_g & 7, 4)); + sink.put1(encode_sib(*shift, enc_index & 7, enc_base & 7)); sink.put4(*simm32); } else { - panic!("emit_REX_OPCODES_MODRM_SIB_IMM_encG_memE: IRRS"); + panic!("ImmRegRegShift"); } } } } -/// This is the core 'emit' function for instructions that do not reference -/// memory. +/// This is the core 'emit' function for instructions that do not reference memory. /// -/// This is conceptually the same as -/// emit_REX_OPCODES_MODRM_SIB_IMM_encG_memE, except it is for the case -/// where the E operand is a register rather than memory. Hence it is much -/// simpler. -fn emit_REX_OPCODES_MODRM_encG_encE( +/// This is conceptually the same as emit_modrm_sib_enc_ge, except it is for the case where the E +/// operand is a register rather than memory. Hence it is much simpler. +fn emit_std_enc_enc( sink: &mut MachBuffer, prefix: LegacyPrefix, opcodes: u32, - mut numOpcodes: usize, - encG: u8, - encE: u8, - flags: u32, + mut num_opcodes: usize, + enc_g: u8, + enc_e: u8, + rex: RexFlags, ) { // EncG and EncE can be derived from registers of any class, and they // don't even have to be from the same class. For example, for an // integer-to-FP conversion insn, one might be RegClass::I64 and the other // RegClass::V128. - let clearRexW = (flags & F_CLEAR_REX_W) != 0; - let retainRedundant = (flags & F_RETAIN_REDUNDANT_REX) != 0; - // The operand-size override - match prefix { - LegacyPrefix::Pfx66 => sink.put1(0x66), - LegacyPrefix::PfxF2 => sink.put1(0xF2), - LegacyPrefix::PfxF3 => sink.put1(0xF3), - LegacyPrefix::PfxNone => (), + // The operand-size override. + prefix.emit(sink); + + // The rex byte. + rex.emit_two_op(sink, enc_g, enc_e); + + // All other prefixes and opcodes. + while num_opcodes > 0 { + num_opcodes -= 1; + sink.put1(((opcodes >> (num_opcodes << 3)) & 0xFF) as u8); } - // The rex byte - let w = if clearRexW { 0 } else { 1 }; - let r = (encG >> 3) & 1; - let x = 0; - let b = (encE >> 3) & 1; - let rex = 0x40 | (w << 3) | (r << 2) | (x << 1) | b; - - if rex != 0x40 || retainRedundant { - sink.put1(rex); - } - - // All other prefixes and opcodes - while numOpcodes > 0 { - numOpcodes -= 1; - sink.put1(((opcodes >> (numOpcodes << 3)) & 0xFF) as u8); - } // Now the mod/rm byte. The instruction we're generating doesn't access // memory, so there is no SIB byte or immediate -- we're done. - sink.put1(mkModRegRM(3, encG & 7, encE & 7)); + sink.put1(encode_modrm(3, enc_g & 7, enc_e & 7)); } // These are merely wrappers for the above two functions that facilitate passing // actual `Reg`s rather than their encodings. -fn emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( +fn emit_std_reg_mem( sink: &mut MachBuffer, prefix: LegacyPrefix, opcodes: u32, - numOpcodes: usize, - regG: Reg, - memE: &Addr, - flags: u32, + num_opcodes: usize, + reg_g: Reg, + mem_e: &Amode, + rex: RexFlags, ) { - // JRS FIXME 2020Feb07: this should really just be `regEnc` not `iregEnc` - let encG = iregEnc(regG); - emit_REX_OPCODES_MODRM_SIB_IMM_encG_memE(sink, prefix, opcodes, numOpcodes, encG, memE, flags); + let enc_g = reg_enc(reg_g); + emit_std_enc_mem(sink, prefix, opcodes, num_opcodes, enc_g, mem_e, rex); } -fn emit_REX_OPCODES_MODRM_regG_regE( +fn emit_std_reg_reg( sink: &mut MachBuffer, prefix: LegacyPrefix, opcodes: u32, - numOpcodes: usize, - regG: Reg, - regE: Reg, - flags: u32, + num_opcodes: usize, + reg_g: Reg, + reg_e: Reg, + rex: RexFlags, ) { - // JRS FIXME 2020Feb07: these should really just be `regEnc` not `iregEnc` - let encG = iregEnc(regG); - let encE = iregEnc(regE); - emit_REX_OPCODES_MODRM_encG_encE(sink, prefix, opcodes, numOpcodes, encG, encE, flags); + let enc_g = reg_enc(reg_g); + let enc_e = reg_enc(reg_e); + emit_std_enc_enc(sink, prefix, opcodes, num_opcodes, enc_g, enc_e, rex); } /// Write a suitable number of bits from an imm64 to the sink. @@ -289,7 +336,7 @@ fn emit_simm(sink: &mut MachBuffer, size: u8, simm32: u32) { 8 | 4 => sink.put4(simm32), 2 => sink.put2(simm32 as u16), 1 => sink.put1(simm32 as u8), - _ => panic!("x64::Inst::emit_simm: unreachable"), + _ => unreachable!(), } } @@ -345,168 +392,165 @@ fn emit_simm(sink: &mut MachBuffer, size: u8, simm32: u32) { /// /// * there's a shorter encoding for shl/shr/sar by a 1-bit immediate. (Do we /// care?) -pub(crate) fn emit(inst: &Inst, sink: &mut MachBuffer) { +pub(crate) fn emit( + inst: &Inst, + sink: &mut MachBuffer, + _flags: &settings::Flags, + state: &mut EmitState, +) { match inst { - Inst::Nop { len: 0 } => {} Inst::Alu_RMI_R { is_64, op, - src: srcE, - dst: regG, + src, + dst: reg_g, } => { - let flags = if *is_64 { F_NONE } else { F_CLEAR_REX_W }; - if *op == RMI_R_Op::Mul { + let rex = if *is_64 { + RexFlags::set_w() + } else { + RexFlags::clear_w() + }; + + if *op == AluRmiROpcode::Mul { // We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so // we have to special-case it. - match srcE { - RMI::R { reg: regE } => { - emit_REX_OPCODES_MODRM_regG_regE( + match src { + RegMemImm::Reg { reg: reg_e } => { + emit_std_reg_reg( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0x0FAF, 2, - regG.to_reg(), - *regE, - flags, + reg_g.to_reg(), + *reg_e, + rex, ); } - RMI::M { addr } => { - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( + + RegMemImm::Mem { addr } => { + emit_std_reg_mem( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0x0FAF, 2, - regG.to_reg(), - addr, - flags, + reg_g.to_reg(), + &addr.finalize(state), + rex, ); } - RMI::I { simm32 } => { - let useImm8 = low8willSXto32(*simm32); + + RegMemImm::Imm { simm32 } => { + let useImm8 = low8_will_sign_extend_to_32(*simm32); let opcode = if useImm8 { 0x6B } else { 0x69 }; - // Yes, really, regG twice. - emit_REX_OPCODES_MODRM_regG_regE( + // Yes, really, reg_g twice. + emit_std_reg_reg( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, opcode, 1, - regG.to_reg(), - regG.to_reg(), - flags, + reg_g.to_reg(), + reg_g.to_reg(), + rex, ); emit_simm(sink, if useImm8 { 1 } else { 4 }, *simm32); } } } else { - let (opcode_R, opcode_M, subopcode_I) = match op { - RMI_R_Op::Add => (0x01, 0x03, 0), - RMI_R_Op::Sub => (0x29, 0x2B, 5), - RMI_R_Op::And => (0x21, 0x23, 4), - RMI_R_Op::Or => (0x09, 0x0B, 1), - RMI_R_Op::Xor => (0x31, 0x33, 6), - RMI_R_Op::Mul => panic!("unreachable"), + let (opcode_r, opcode_m, subopcode_i) = match op { + AluRmiROpcode::Add => (0x01, 0x03, 0), + AluRmiROpcode::Sub => (0x29, 0x2B, 5), + AluRmiROpcode::And => (0x21, 0x23, 4), + AluRmiROpcode::Or => (0x09, 0x0B, 1), + AluRmiROpcode::Xor => (0x31, 0x33, 6), + AluRmiROpcode::Mul => panic!("unreachable"), }; - match srcE { - RMI::R { reg: regE } => { - // Note. The arguments .. regE .. regG .. sequence - // here is the opposite of what is expected. I'm not - // sure why this is. But I am fairly sure that the - // arg order could be switched back to the expected - // .. regG .. regE .. if opcode_rr is also switched - // over to the "other" basic integer opcode (viz, the - // R/RM vs RM/R duality). However, that would mean - // that the test results won't be in accordance with - // the GNU as reference output. In other words, the - // inversion exists as a result of using GNU as as a - // gold standard. - emit_REX_OPCODES_MODRM_regG_regE( + + match src { + RegMemImm::Reg { reg: reg_e } => { + // GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R + // duality). Do this too, so as to be able to compare generated machine + // code easily. + emit_std_reg_reg( sink, - LegacyPrefix::PfxNone, - opcode_R, + LegacyPrefix::None, + opcode_r, 1, - *regE, - regG.to_reg(), - flags, + *reg_e, + reg_g.to_reg(), + rex, ); - // NB: if this is ever extended to handle byte size - // ops, be sure to retain redundant REX prefixes. + // NB: if this is ever extended to handle byte size ops, be sure to retain + // redundant REX prefixes. } - RMI::M { addr } => { - // Whereas here we revert to the "normal" G-E ordering. - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( + + RegMemImm::Mem { addr } => { + // Here we revert to the "normal" G-E ordering. + emit_std_reg_mem( sink, - LegacyPrefix::PfxNone, - opcode_M, + LegacyPrefix::None, + opcode_m, 1, - regG.to_reg(), - addr, - flags, + reg_g.to_reg(), + &addr.finalize(state), + rex, ); } - RMI::I { simm32 } => { - let useImm8 = low8willSXto32(*simm32); - let opcode = if useImm8 { 0x83 } else { 0x81 }; + + RegMemImm::Imm { simm32 } => { + let use_imm8 = low8_will_sign_extend_to_32(*simm32); + let opcode = if use_imm8 { 0x83 } else { 0x81 }; // And also here we use the "normal" G-E ordering. - let encG = iregEnc(regG.to_reg()); - emit_REX_OPCODES_MODRM_encG_encE( + let enc_g = int_reg_enc(reg_g.to_reg()); + emit_std_enc_enc( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, opcode, 1, - subopcode_I, - encG, - flags, + subopcode_i, + enc_g, + rex, ); - emit_simm(sink, if useImm8 { 1 } else { 4 }, *simm32); + emit_simm(sink, if use_imm8 { 1 } else { 4 }, *simm32); } } } } + Inst::Imm_R { dst_is_64, simm64, dst, } => { - let encDst = iregEnc(dst.to_reg()); + let enc_dst = int_reg_enc(dst.to_reg()); if *dst_is_64 { // FIXME JRS 2020Feb10: also use the 32-bit case here when // possible - sink.put1(0x48 | ((encDst >> 3) & 1)); - sink.put1(0xB8 | (encDst & 7)); + sink.put1(0x48 | ((enc_dst >> 3) & 1)); + sink.put1(0xB8 | (enc_dst & 7)); sink.put8(*simm64); } else { - if ((encDst >> 3) & 1) == 1 { + if ((enc_dst >> 3) & 1) == 1 { sink.put1(0x41); } - sink.put1(0xB8 | (encDst & 7)); + sink.put1(0xB8 | (enc_dst & 7)); sink.put4(*simm64 as u32); } } + Inst::Mov_R_R { is_64, src, dst } => { - let flags = if *is_64 { F_NONE } else { F_CLEAR_REX_W }; - emit_REX_OPCODES_MODRM_regG_regE( - sink, - LegacyPrefix::PfxNone, - 0x89, - 1, - *src, - dst.to_reg(), - flags, - ); + let rex = if *is_64 { + RexFlags::set_w() + } else { + RexFlags::clear_w() + }; + emit_std_reg_reg(sink, LegacyPrefix::None, 0x89, 1, *src, dst.to_reg(), rex); } - Inst::MovZX_M_R { extMode, addr, dst } => { - match extMode { + + Inst::MovZX_RM_R { ext_mode, src, dst } => { + let (opcodes, num_opcodes, rex_flags) = match ext_mode { ExtMode::BL => { // MOVZBL is (REX.W==0) 0F B6 /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x0FB6, - 2, - dst.to_reg(), - addr, - F_CLEAR_REX_W, - ) + (0x0FB6, 2, RexFlags::clear_w()) } ExtMode::BQ => { // MOVZBQ is (REX.W==1) 0F B6 /r @@ -514,324 +558,323 @@ pub(crate) fn emit(inst: &Inst, sink: &mut MachBuffer) { // encodings for MOVZBQ than for MOVZBL. AIUI they should // achieve the same, since MOVZBL is just going to zero out // the upper half of the destination anyway. - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x0FB6, - 2, - dst.to_reg(), - addr, - F_NONE, - ) + (0x0FB6, 2, RexFlags::set_w()) } ExtMode::WL => { // MOVZWL is (REX.W==0) 0F B7 /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x0FB7, - 2, - dst.to_reg(), - addr, - F_CLEAR_REX_W, - ) + (0x0FB7, 2, RexFlags::clear_w()) } ExtMode::WQ => { // MOVZWQ is (REX.W==1) 0F B7 /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x0FB7, - 2, - dst.to_reg(), - addr, - F_NONE, - ) + (0x0FB7, 2, RexFlags::set_w()) } ExtMode::LQ => { // This is just a standard 32 bit load, and we rely on the // default zero-extension rule to perform the extension. + // Note that in reg/reg mode, gcc seems to use the swapped form R/RM, which we + // don't do here, since it's the same encoding size. // MOV r/m32, r32 is (REX.W==0) 8B /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x8B, - 1, - dst.to_reg(), - addr, - F_CLEAR_REX_W, - ) + (0x8B, 1, RexFlags::clear_w()) } + }; + + match src { + RegMem::Reg { reg: src } => emit_std_reg_reg( + sink, + LegacyPrefix::None, + opcodes, + num_opcodes, + dst.to_reg(), + *src, + rex_flags, + ), + RegMem::Mem { addr: src } => emit_std_reg_mem( + sink, + LegacyPrefix::None, + opcodes, + num_opcodes, + dst.to_reg(), + &src.finalize(state), + rex_flags, + ), } } - Inst::Mov64_M_R { addr, dst } => emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( + + Inst::Mov64_M_R { src, dst } => emit_std_reg_mem( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0x8B, 1, dst.to_reg(), - addr, - F_NONE, + &src.finalize(state), + RexFlags::set_w(), ), - Inst::MovSX_M_R { extMode, addr, dst } => { - match extMode { + + Inst::LoadEffectiveAddress { addr, dst } => emit_std_reg_mem( + sink, + LegacyPrefix::None, + 0x8D, + 1, + dst.to_reg(), + &addr.finalize(state), + RexFlags::set_w(), + ), + + Inst::MovSX_RM_R { ext_mode, src, dst } => { + let (opcodes, num_opcodes, rex_flags) = match ext_mode { ExtMode::BL => { // MOVSBL is (REX.W==0) 0F BE /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x0FBE, - 2, - dst.to_reg(), - addr, - F_CLEAR_REX_W, - ) + (0x0FBE, 2, RexFlags::clear_w()) } ExtMode::BQ => { // MOVSBQ is (REX.W==1) 0F BE /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x0FBE, - 2, - dst.to_reg(), - addr, - F_NONE, - ) + (0x0FBE, 2, RexFlags::set_w()) } ExtMode::WL => { // MOVSWL is (REX.W==0) 0F BF /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x0FBF, - 2, - dst.to_reg(), - addr, - F_CLEAR_REX_W, - ) + (0x0FBF, 2, RexFlags::clear_w()) } ExtMode::WQ => { // MOVSWQ is (REX.W==1) 0F BF /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x0FBF, - 2, - dst.to_reg(), - addr, - F_NONE, - ) + (0x0FBF, 2, RexFlags::set_w()) } ExtMode::LQ => { // MOVSLQ is (REX.W==1) 63 /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x63, - 1, - dst.to_reg(), - addr, - F_NONE, - ) + (0x63, 1, RexFlags::set_w()) } + }; + + match src { + RegMem::Reg { reg: src } => emit_std_reg_reg( + sink, + LegacyPrefix::None, + opcodes, + num_opcodes, + dst.to_reg(), + *src, + rex_flags, + ), + RegMem::Mem { addr: src } => emit_std_reg_mem( + sink, + LegacyPrefix::None, + opcodes, + num_opcodes, + dst.to_reg(), + &src.finalize(state), + rex_flags, + ), } } - Inst::Mov_R_M { size, src, addr } => { + + Inst::Mov_R_M { size, src, dst } => { + let dst = &dst.finalize(state); + match size { 1 => { // This is one of the few places where the presence of a // redundant REX prefix changes the meaning of the // instruction. - let encSrc = iregEnc(*src); - let retainRedundantRex = if encSrc >= 4 && encSrc <= 7 { - F_RETAIN_REDUNDANT_REX - } else { - 0 + let mut rex = RexFlags::clear_w(); + + let enc_src = int_reg_enc(*src); + if enc_src >= 4 && enc_src <= 7 { + rex.always_emit(); }; + // MOV r8, r/m8 is (REX.W==0) 88 /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxNone, - 0x88, - 1, - *src, - addr, - F_CLEAR_REX_W | retainRedundantRex, - ) + emit_std_reg_mem(sink, LegacyPrefix::None, 0x88, 1, *src, dst, rex) } + 2 => { // MOV r16, r/m16 is 66 (REX.W==0) 89 /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( + emit_std_reg_mem( sink, - LegacyPrefix::Pfx66, + LegacyPrefix::_66, 0x89, 1, *src, - addr, - F_CLEAR_REX_W, + dst, + RexFlags::clear_w(), ) } + 4 => { // MOV r32, r/m32 is (REX.W==0) 89 /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( + emit_std_reg_mem( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0x89, 1, *src, - addr, - F_CLEAR_REX_W, + dst, + RexFlags::clear_w(), ) } + 8 => { // MOV r64, r/m64 is (REX.W==1) 89 /r - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( + emit_std_reg_mem( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0x89, 1, *src, - addr, - F_NONE, + dst, + RexFlags::set_w(), ) } + _ => panic!("x64::Inst::Mov_R_M::emit: unreachable"), } } + Inst::Shift_R { is_64, kind, num_bits, dst, } => { - let encDst = iregEnc(dst.to_reg()); + let enc_dst = int_reg_enc(dst.to_reg()); let subopcode = match kind { ShiftKind::Left => 4, ShiftKind::RightZ => 5, ShiftKind::RightS => 7, }; + + let rex = if *is_64 { + RexFlags::set_w() + } else { + RexFlags::clear_w() + }; + match num_bits { None => { // SHL/SHR/SAR %cl, reg32 is (REX.W==0) D3 /subopcode // SHL/SHR/SAR %cl, reg64 is (REX.W==1) D3 /subopcode - emit_REX_OPCODES_MODRM_encG_encE( - sink, - LegacyPrefix::PfxNone, - 0xD3, - 1, - subopcode, - encDst, - if *is_64 { F_NONE } else { F_CLEAR_REX_W }, - ); + emit_std_enc_enc(sink, LegacyPrefix::None, 0xD3, 1, subopcode, enc_dst, rex); } + Some(num_bits) => { // SHL/SHR/SAR $ib, reg32 is (REX.W==0) C1 /subopcode ib // SHL/SHR/SAR $ib, reg64 is (REX.W==1) C1 /subopcode ib // When the shift amount is 1, there's an even shorter encoding, but we don't // bother with that nicety here. - emit_REX_OPCODES_MODRM_encG_encE( - sink, - LegacyPrefix::PfxNone, - 0xC1, - 1, - subopcode, - encDst, - if *is_64 { F_NONE } else { F_CLEAR_REX_W }, - ); + emit_std_enc_enc(sink, LegacyPrefix::None, 0xC1, 1, subopcode, enc_dst, rex); sink.put1(*num_bits); } } } + Inst::Cmp_RMI_R { size, - src: srcE, - dst: regG, + src: src_e, + dst: reg_g, } => { - let mut retainRedundantRex = 0; - let mut prefix = LegacyPrefix::PfxNone; - if *size == 1 { - // Here, a redundant REX prefix changes the meaning of the - // instruction. - let encG = iregEnc(*regG); - if encG >= 4 && encG <= 7 { - retainRedundantRex = F_RETAIN_REDUNDANT_REX; - } - } + let mut prefix = LegacyPrefix::None; if *size == 2 { - prefix = LegacyPrefix::Pfx66; + prefix = LegacyPrefix::_66; } - let mut flags = match size { - 8 => F_NONE, - 4 | 2 => F_CLEAR_REX_W, - 1 => F_CLEAR_REX_W | retainRedundantRex, + + let mut rex = match size { + 8 => RexFlags::set_w(), + 4 | 2 => RexFlags::clear_w(), + 1 => { + let mut rex = RexFlags::clear_w(); + // Here, a redundant REX prefix changes the meaning of the instruction. + let enc_g = int_reg_enc(*reg_g); + if enc_g >= 4 && enc_g <= 7 { + rex.always_emit(); + } + rex + } _ => panic!("x64::Inst::Cmp_RMI_R::emit: unreachable"), }; - match srcE { - RMI::R { reg: regE } => { - let opcode = if *size == 1 { 0x38 } else { 0x39 }; + + match src_e { + RegMemImm::Reg { reg: reg_e } => { if *size == 1 { - // We also need to check whether the E register forces - // the use of a redundant REX. - let encE = iregEnc(*regE); - if encE >= 4 && encE <= 7 { - flags |= F_RETAIN_REDUNDANT_REX; + // Check whether the E register forces the use of a redundant REX. + let enc_e = int_reg_enc(*reg_e); + if enc_e >= 4 && enc_e <= 7 { + rex.always_emit(); } } - // Same comment re swapped args as for Alu_RMI_R. - emit_REX_OPCODES_MODRM_regG_regE(sink, prefix, opcode, 1, *regE, *regG, flags); + + // Use the swapped operands encoding, to stay consistent with the output of + // gcc/llvm. + let opcode = if *size == 1 { 0x38 } else { 0x39 }; + emit_std_reg_reg(sink, prefix, opcode, 1, *reg_e, *reg_g, rex); } - RMI::M { addr } => { - let opcode = if *size == 1 { 0x3A } else { 0x3B }; + + RegMemImm::Mem { addr } => { + let addr = &addr.finalize(state); // Whereas here we revert to the "normal" G-E ordering. - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, prefix, opcode, 1, *regG, addr, flags, - ); + let opcode = if *size == 1 { 0x3A } else { 0x3B }; + emit_std_reg_mem(sink, prefix, opcode, 1, *reg_g, addr, rex); } - RMI::I { simm32 } => { + + RegMemImm::Imm { simm32 } => { // FIXME JRS 2020Feb11: there are shorter encodings for // cmp $imm, rax/eax/ax/al. - let useImm8 = low8willSXto32(*simm32); + let use_imm8 = low8_will_sign_extend_to_32(*simm32); + + // And also here we use the "normal" G-E ordering. let opcode = if *size == 1 { 0x80 - } else if useImm8 { + } else if use_imm8 { 0x83 } else { 0x81 }; - // And also here we use the "normal" G-E ordering. - let encG = iregEnc(*regG); - emit_REX_OPCODES_MODRM_encG_encE( - sink, prefix, opcode, 1, 7, /*subopcode*/ - encG, flags, - ); - emit_simm(sink, if useImm8 { 1 } else { *size }, *simm32); + + let enc_g = int_reg_enc(*reg_g); + emit_std_enc_enc(sink, prefix, opcode, 1, 7 /*subopcode*/, enc_g, rex); + emit_simm(sink, if use_imm8 { 1 } else { *size }, *simm32); } } } + + Inst::Setcc { cc, dst } => { + let opcode = 0x0f90 + cc.get_enc() as u32; + let mut rex_flags = RexFlags::clear_w(); + rex_flags.always_emit(); + emit_std_enc_enc( + sink, + LegacyPrefix::None, + opcode, + 2, + 0, + reg_enc(dst.to_reg()), + rex_flags, + ); + } + Inst::Push64 { src } => { match src { - RMI::R { reg } => { - let encReg = iregEnc(*reg); - let rex = 0x40 | ((encReg >> 3) & 1); + RegMemImm::Reg { reg } => { + let enc_reg = int_reg_enc(*reg); + let rex = 0x40 | ((enc_reg >> 3) & 1); if rex != 0x40 { sink.put1(rex); } - sink.put1(0x50 | (encReg & 7)); + sink.put1(0x50 | (enc_reg & 7)); } - RMI::M { addr } => { - emit_REX_OPCODES_MODRM_SIB_IMM_encG_memE( + + RegMemImm::Mem { addr } => { + let addr = &addr.finalize(state); + emit_std_enc_mem( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0xFF, 1, 6, /*subopcode*/ addr, - F_CLEAR_REX_W, + RexFlags::clear_w(), ); } - RMI::I { simm32 } => { - if low8willSXto64(*simm32) { + + RegMemImm::Imm { simm32 } => { + if low8_will_sign_extend_to_64(*simm32) { sink.put1(0x6A); sink.put1(*simm32 as u8); } else { @@ -841,8 +884,9 @@ pub(crate) fn emit(inst: &Inst, sink: &mut MachBuffer) { } } } + Inst::Pop64 { dst } => { - let encDst = iregEnc(dst.to_reg()); + let encDst = int_reg_enc(dst.to_reg()); if encDst >= 8 { // 0x41 == REX.{W=0, B=1}. It seems that REX.W is irrelevant // here. @@ -850,168 +894,226 @@ pub(crate) fn emit(inst: &Inst, sink: &mut MachBuffer) { } sink.put1(0x58 + (encDst & 7)); } - // - // ** Inst::CallKnown - // - Inst::CallUnknown { dest } => { + + Inst::CallKnown { + dest, loc, opcode, .. + } => { + sink.put1(0xE8); + // The addend adjusts for the difference between the end of the instruction and the + // beginning of the immediate field. + sink.add_reloc(*loc, Reloc::X86CallPCRel4, &dest, -4); + sink.put4(0); + if opcode.is_call() { + sink.add_call_site(*loc, *opcode); + } + } + + Inst::CallUnknown { + dest, opcode, loc, .. + } => { match dest { - RM::R { reg } => { - let regEnc = iregEnc(*reg); - emit_REX_OPCODES_MODRM_encG_encE( + RegMem::Reg { reg } => { + let reg_enc = int_reg_enc(*reg); + emit_std_enc_enc( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0xFF, 1, 2, /*subopcode*/ - regEnc, - F_CLEAR_REX_W, + reg_enc, + RexFlags::clear_w(), ); } - RM::M { addr } => { - emit_REX_OPCODES_MODRM_SIB_IMM_encG_memE( + + RegMem::Mem { addr } => { + let addr = &addr.finalize(state); + emit_std_enc_mem( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0xFF, 1, 2, /*subopcode*/ addr, - F_CLEAR_REX_W, + RexFlags::clear_w(), ); } } + if opcode.is_call() { + sink.add_call_site(*loc, *opcode); + } } + Inst::Ret {} => sink.put1(0xC3), - Inst::JmpKnown { dest } => { - let disp = dest.as_offset32_or_zero() - 5; - let disp = disp as u32; + Inst::JmpKnown { dst } => { let br_start = sink.cur_offset(); let br_disp_off = br_start + 1; let br_end = br_start + 5; - if let Some(l) = dest.as_label() { - sink.use_label_at_offset(br_disp_off, l, LabelUse::Rel32); + if let Some(l) = dst.as_label() { + sink.use_label_at_offset(br_disp_off, l, LabelUse::JmpRel32); sink.add_uncond_branch(br_start, br_end, l); } + + let disp = dst.as_offset32_or_zero(); + let disp = disp as u32; sink.put1(0xE9); sink.put4(disp); } - Inst::JmpCondSymm { + + Inst::JmpCond { cc, taken, not_taken, } => { - // Conditional part. - - // This insn is 6 bytes long. Currently `offset` is relative to - // the start of this insn, but the Intel encoding requires it to - // be relative to the start of the next instruction. Hence the - // adjustment. - let taken_disp = taken.as_offset32_or_zero() - 6; - let taken_disp = taken_disp as u32; + // If taken. let cond_start = sink.cur_offset(); let cond_disp_off = cond_start + 2; let cond_end = cond_start + 6; if let Some(l) = taken.as_label() { - sink.use_label_at_offset(cond_disp_off, l, LabelUse::Rel32); + sink.use_label_at_offset(cond_disp_off, l, LabelUse::JmpRel32); let inverted: [u8; 6] = - [0x0F, 0x80 + (cc.invert().get_enc()), 0xFA, 0xFF, 0xFF, 0xFF]; + [0x0F, 0x80 + (cc.invert().get_enc()), 0x00, 0x00, 0x00, 0x00]; sink.add_cond_branch(cond_start, cond_end, l, &inverted[..]); } + + let taken_disp = taken.as_offset32_or_zero(); + let taken_disp = taken_disp as u32; sink.put1(0x0F); sink.put1(0x80 + cc.get_enc()); sink.put4(taken_disp); - // Unconditional part. - - let nt_disp = not_taken.as_offset32_or_zero() - 5; - let nt_disp = nt_disp as u32; + // If not taken. let uncond_start = sink.cur_offset(); let uncond_disp_off = uncond_start + 1; let uncond_end = uncond_start + 5; if let Some(l) = not_taken.as_label() { - sink.use_label_at_offset(uncond_disp_off, l, LabelUse::Rel32); + sink.use_label_at_offset(uncond_disp_off, l, LabelUse::JmpRel32); sink.add_uncond_branch(uncond_start, uncond_end, l); } + + let nt_disp = not_taken.as_offset32_or_zero(); + let nt_disp = nt_disp as u32; sink.put1(0xE9); sink.put4(nt_disp); } + Inst::JmpUnknown { target } => { match target { - RM::R { reg } => { - let regEnc = iregEnc(*reg); - emit_REX_OPCODES_MODRM_encG_encE( + RegMem::Reg { reg } => { + let reg_enc = int_reg_enc(*reg); + emit_std_enc_enc( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0xFF, 1, 4, /*subopcode*/ - regEnc, - F_CLEAR_REX_W, + reg_enc, + RexFlags::clear_w(), ); } - RM::M { addr } => { - emit_REX_OPCODES_MODRM_SIB_IMM_encG_memE( + + RegMem::Mem { addr } => { + let addr = &addr.finalize(state); + emit_std_enc_mem( sink, - LegacyPrefix::PfxNone, + LegacyPrefix::None, 0xFF, 1, 4, /*subopcode*/ addr, - F_CLEAR_REX_W, + RexFlags::clear_w(), ); } } } - Inst::XMM_R_R { op, src, dst } => { - let flags = F_CLEAR_REX_W; - let opcode = match op { - SSE_Op::SSE_Movss => 0x0F10, - SSE_Op::SSE2_Movsd => 0x0F10, - _ => unimplemented!("XMM_R_R opcode"), + + Inst::XMM_Mov_RM_R { + op, + src: src_e, + dst: reg_g, + } => { + let rex = RexFlags::clear_w(); + let (prefix, opcode) = match op { + SseOpcode::Movaps => (LegacyPrefix::None, 0x0F28), + SseOpcode::Movd => (LegacyPrefix::_66, 0x0F6E), + SseOpcode::Movsd => (LegacyPrefix::_F2, 0x0F10), + SseOpcode::Movss => (LegacyPrefix::_F3, 0x0F10), + _ => unimplemented!("Opcode {:?} not implemented", op), }; - let prefix = match op { - SSE_Op::SSE_Movss => LegacyPrefix::PfxF3, - SSE_Op::SSE2_Movsd => LegacyPrefix::PfxF2, - _ => unimplemented!("XMM_R_R opcode"), - }; - emit_REX_OPCODES_MODRM_regG_regE(sink, prefix, opcode, 2, dst.to_reg(), *src, flags); + + match src_e { + RegMem::Reg { reg: reg_e } => { + emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); + } + + RegMem::Mem { addr } => { + let addr = &addr.finalize(state); + emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex); + } + } } + Inst::XMM_RM_R { op, - src: srcE, - dst: regG, + src: src_e, + dst: reg_g, } => { - let flags = F_CLEAR_REX_W; - let opcode = match op { - SSE_Op::SSE_Addss => 0x0F58, - SSE_Op::SSE_Subss => 0x0F5C, - _ => unimplemented!("XMM_RM_R opcode"), + let rex = RexFlags::clear_w(); + let (prefix, opcode) = match op { + SseOpcode::Addss => (LegacyPrefix::_F3, 0x0F58), + SseOpcode::Andps => (LegacyPrefix::None, 0x0F54), + SseOpcode::Andnps => (LegacyPrefix::None, 0x0F55), + SseOpcode::Divss => (LegacyPrefix::_F3, 0x0F5E), + SseOpcode::Mulss => (LegacyPrefix::_F3, 0x0F59), + SseOpcode::Orps => (LegacyPrefix::None, 0x0F56), + SseOpcode::Subss => (LegacyPrefix::_F3, 0x0F5C), + SseOpcode::Sqrtss => (LegacyPrefix::_F3, 0x0F51), + _ => unimplemented!("Opcode {:?} not implemented", op), }; - match srcE { - RM::R { reg: regE } => { - emit_REX_OPCODES_MODRM_regG_regE( - sink, - LegacyPrefix::PfxF3, - opcode, - 2, - regG.to_reg(), - *regE, - flags, - ); + + match src_e { + RegMem::Reg { reg: reg_e } => { + emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); } - RM::M { addr } => { - emit_REX_OPCODES_MODRM_SIB_IMM_regG_memE( - sink, - LegacyPrefix::PfxF3, - opcode, - 2, - regG.to_reg(), - addr, - flags, - ); + + RegMem::Mem { addr } => { + let addr = &addr.finalize(state); + emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex); } } } - _ => panic!("x64_emit: unhandled: {} ", inst.show_rru(None)), + Inst::XMM_Mov_R_M { op, src, dst } => { + let rex = RexFlags::clear_w(); + let (prefix, opcode) = match op { + SseOpcode::Movd => (LegacyPrefix::_66, 0x0F7E), + SseOpcode::Movss => (LegacyPrefix::_F3, 0x0F11), + _ => unimplemented!("Emit xmm mov r m"), + }; + + let dst = &dst.finalize(state); + emit_std_reg_mem(sink, prefix, opcode, 2, *src, dst, rex); + } + Inst::Hlt => { + sink.put1(0xcc); + } + + Inst::Ud2 { trap_info } => { + sink.add_trap(trap_info.0, trap_info.1); + sink.put1(0x0f); + sink.put1(0x0b); + } + + Inst::VirtualSPOffsetAdj { offset } => { + debug!( + "virtual sp offset adjusted by {} -> {}", + offset, + state.virtual_sp_offset + offset + ); + state.virtual_sp_offset += offset; + } + + Inst::Nop { .. } | Inst::EpiloguePlaceholder => { + // Generate no code. + } } } diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit_tests.rs b/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit_tests.rs index 6b54c7a1f274..96f350d43d01 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit_tests.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/inst/emit_tests.rs @@ -72,18 +72,18 @@ fn test_x64_emit() { let w_xmm1 = Writable::::from_reg(xmm1); let w_xmm2 = Writable::::from_reg(xmm2); let w_xmm3 = Writable::::from_reg(xmm3); - let _w_xmm4 = Writable::::from_reg(xmm4); + let w_xmm4 = Writable::::from_reg(xmm4); let _w_xmm5 = Writable::::from_reg(xmm5); let _w_xmm6 = Writable::::from_reg(xmm6); - let _w_xmm7 = Writable::::from_reg(xmm7); - let _w_xmm8 = Writable::::from_reg(xmm8); - let _w_xmm9 = Writable::::from_reg(xmm9); + let w_xmm7 = Writable::::from_reg(xmm7); + let w_xmm8 = Writable::::from_reg(xmm8); + let w_xmm9 = Writable::::from_reg(xmm9); let w_xmm10 = Writable::::from_reg(xmm10); - let _w_xmm11 = Writable::::from_reg(xmm11); - let _w_xmm12 = Writable::::from_reg(xmm12); + let w_xmm11 = Writable::::from_reg(xmm11); + let w_xmm12 = Writable::::from_reg(xmm12); let w_xmm13 = Writable::::from_reg(xmm13); - let _w_xmm14 = Writable::::from_reg(xmm14); - let _w_xmm15 = Writable::::from_reg(xmm15); + let w_xmm14 = Writable::::from_reg(xmm14); + let w_xmm15 = Writable::::from_reg(xmm15); let mut insns = Vec::<(Inst, &str, &str)>::new(); @@ -95,82 +95,82 @@ fn test_x64_emit() { // // Addr_IR, offset zero insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rax), w_rdi), "488B38", "movq 0(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rbx), w_rdi), "488B3B", "movq 0(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rcx), w_rdi), "488B39", "movq 0(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rdx), w_rdi), "488B3A", "movq 0(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rbp), w_rdi), "488B7D00", "movq 0(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rsp), w_rdi), "488B3C24", "movq 0(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rsi), w_rdi), "488B3E", "movq 0(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rdi), w_rdi), "488B3F", "movq 0(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r8), w_rdi), "498B38", "movq 0(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r9), w_rdi), "498B39", "movq 0(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r10), w_rdi), "498B3A", "movq 0(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r11), w_rdi), "498B3B", "movq 0(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r12), w_rdi), "498B3C24", "movq 0(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r13), w_rdi), "498B7D00", "movq 0(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r14), w_rdi), "498B3E", "movq 0(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r15), w_rdi), "498B3F", "movq 0(%r15), %rdi", )); @@ -178,82 +178,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset max simm8 insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rax), w_rdi), "488B787F", "movq 127(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rbx), w_rdi), "488B7B7F", "movq 127(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rcx), w_rdi), "488B797F", "movq 127(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rdx), w_rdi), "488B7A7F", "movq 127(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rbp), w_rdi), "488B7D7F", "movq 127(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rsp), w_rdi), "488B7C247F", "movq 127(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rsi), w_rdi), "488B7E7F", "movq 127(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rdi), w_rdi), "488B7F7F", "movq 127(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r8), w_rdi), "498B787F", "movq 127(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r9), w_rdi), "498B797F", "movq 127(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r10), w_rdi), "498B7A7F", "movq 127(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r11), w_rdi), "498B7B7F", "movq 127(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r12), w_rdi), "498B7C247F", "movq 127(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r13), w_rdi), "498B7D7F", "movq 127(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r14), w_rdi), "498B7E7F", "movq 127(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(127, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r15), w_rdi), "498B7F7F", "movq 127(%r15), %rdi", )); @@ -261,82 +261,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset min simm8 insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rax), w_rdi), "488B7880", "movq -128(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rbx), w_rdi), "488B7B80", "movq -128(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rcx), w_rdi), "488B7980", "movq -128(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rdx), w_rdi), "488B7A80", "movq -128(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rbp), w_rdi), "488B7D80", "movq -128(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rsp), w_rdi), "488B7C2480", "movq -128(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rsi), w_rdi), "488B7E80", "movq -128(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rdi), w_rdi), "488B7F80", "movq -128(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r8), w_rdi), "498B7880", "movq -128(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r9), w_rdi), "498B7980", "movq -128(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r10), w_rdi), "498B7A80", "movq -128(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r11), w_rdi), "498B7B80", "movq -128(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r12), w_rdi), "498B7C2480", "movq -128(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r13), w_rdi), "498B7D80", "movq -128(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r14), w_rdi), "498B7E80", "movq -128(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-128i32 as u32, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r15), w_rdi), "498B7F80", "movq -128(%r15), %rdi", )); @@ -344,82 +344,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset smallest positive simm32 insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rax), w_rdi), "488BB880000000", "movq 128(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rbx), w_rdi), "488BBB80000000", "movq 128(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rcx), w_rdi), "488BB980000000", "movq 128(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rdx), w_rdi), "488BBA80000000", "movq 128(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rbp), w_rdi), "488BBD80000000", "movq 128(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rsp), w_rdi), "488BBC2480000000", "movq 128(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rsi), w_rdi), "488BBE80000000", "movq 128(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rdi), w_rdi), "488BBF80000000", "movq 128(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r8), w_rdi), "498BB880000000", "movq 128(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r9), w_rdi), "498BB980000000", "movq 128(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r10), w_rdi), "498BBA80000000", "movq 128(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r11), w_rdi), "498BBB80000000", "movq 128(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r12), w_rdi), "498BBC2480000000", "movq 128(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r13), w_rdi), "498BBD80000000", "movq 128(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r14), w_rdi), "498BBE80000000", "movq 128(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(128, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r15), w_rdi), "498BBF80000000", "movq 128(%r15), %rdi", )); @@ -427,82 +427,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset smallest negative simm32 insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rax), w_rdi), "488BB87FFFFFFF", "movq -129(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rbx), w_rdi), "488BBB7FFFFFFF", "movq -129(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rcx), w_rdi), "488BB97FFFFFFF", "movq -129(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rdx), w_rdi), "488BBA7FFFFFFF", "movq -129(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rbp), w_rdi), "488BBD7FFFFFFF", "movq -129(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rsp), w_rdi), "488BBC247FFFFFFF", "movq -129(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rsi), w_rdi), "488BBE7FFFFFFF", "movq -129(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rdi), w_rdi), "488BBF7FFFFFFF", "movq -129(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r8), w_rdi), "498BB87FFFFFFF", "movq -129(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r9), w_rdi), "498BB97FFFFFFF", "movq -129(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r10), w_rdi), "498BBA7FFFFFFF", "movq -129(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r11), w_rdi), "498BBB7FFFFFFF", "movq -129(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r12), w_rdi), "498BBC247FFFFFFF", "movq -129(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r13), w_rdi), "498BBD7FFFFFFF", "movq -129(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r14), w_rdi), "498BBE7FFFFFFF", "movq -129(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-129i32 as u32, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r15), w_rdi), "498BBF7FFFFFFF", "movq -129(%r15), %rdi", )); @@ -510,82 +510,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset large positive simm32 insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rax), w_rdi), "488BB877207317", "movq 393420919(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rbx), w_rdi), "488BBB77207317", "movq 393420919(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rcx), w_rdi), "488BB977207317", "movq 393420919(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rdx), w_rdi), "488BBA77207317", "movq 393420919(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rbp), w_rdi), "488BBD77207317", "movq 393420919(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rsp), w_rdi), "488BBC2477207317", "movq 393420919(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rsi), w_rdi), "488BBE77207317", "movq 393420919(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rdi), w_rdi), "488BBF77207317", "movq 393420919(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r8), w_rdi), "498BB877207317", "movq 393420919(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r9), w_rdi), "498BB977207317", "movq 393420919(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r10), w_rdi), "498BBA77207317", "movq 393420919(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r11), w_rdi), "498BBB77207317", "movq 393420919(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r12), w_rdi), "498BBC2477207317", "movq 393420919(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r13), w_rdi), "498BBD77207317", "movq 393420919(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r14), w_rdi), "498BBE77207317", "movq 393420919(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(0x17732077, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r15), w_rdi), "498BBF77207317", "movq 393420919(%r15), %rdi", )); @@ -593,82 +593,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset large negative simm32 insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rax), w_rdi), "488BB8D9A6BECE", "movq -826366247(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rbx), w_rdi), "488BBBD9A6BECE", "movq -826366247(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rcx), w_rdi), "488BB9D9A6BECE", "movq -826366247(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rdx), w_rdi), "488BBAD9A6BECE", "movq -826366247(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rbp), w_rdi), "488BBDD9A6BECE", "movq -826366247(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rsp), w_rdi), "488BBC24D9A6BECE", "movq -826366247(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rsi), w_rdi), "488BBED9A6BECE", "movq -826366247(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rdi), w_rdi), "488BBFD9A6BECE", "movq -826366247(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r8), w_rdi), "498BB8D9A6BECE", "movq -826366247(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r9), w_rdi), "498BB9D9A6BECE", "movq -826366247(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r10), w_rdi), "498BBAD9A6BECE", "movq -826366247(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r11), w_rdi), "498BBBD9A6BECE", "movq -826366247(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r12), w_rdi), "498BBC24D9A6BECE", "movq -826366247(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r13), w_rdi), "498BBDD9A6BECE", "movq -826366247(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r14), w_rdi), "498BBED9A6BECE", "movq -826366247(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg(-0x31415927i32 as u32, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r15), w_rdi), "498BBFD9A6BECE", "movq -826366247(%r15), %rdi", )); @@ -680,42 +680,42 @@ fn test_x64_emit() { // // Addr_IRRS, offset max simm8 insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(127, rax, rax, 0), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rax, rax, 0), w_r11), "4C8B5C007F", "movq 127(%rax,%rax,1), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(127, rdi, rax, 1), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rdi, rax, 1), w_r11), "4C8B5C477F", "movq 127(%rdi,%rax,2), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(127, r8, rax, 2), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r8, rax, 2), w_r11), "4D8B5C807F", "movq 127(%r8,%rax,4), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(127, r15, rax, 3), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r15, rax, 3), w_r11), "4D8B5CC77F", "movq 127(%r15,%rax,8), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(127, rax, rdi, 3), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rax, rdi, 3), w_r11), "4C8B5CF87F", "movq 127(%rax,%rdi,8), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(127, rdi, rdi, 2), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rdi, rdi, 2), w_r11), "4C8B5CBF7F", "movq 127(%rdi,%rdi,4), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(127, r8, rdi, 1), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r8, rdi, 1), w_r11), "4D8B5C787F", "movq 127(%r8,%rdi,2), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(127, r15, rdi, 0), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r15, rdi, 0), w_r11), "4D8B5C3F7F", "movq 127(%r15,%rdi,1), %r11", )); @@ -723,42 +723,42 @@ fn test_x64_emit() { // ======================================================== // Addr_IRRS, offset min simm8 insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(-128i32 as u32, rax, r8, 2), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rax, r8, 2), w_r11), "4E8B5C8080", "movq -128(%rax,%r8,4), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(-128i32 as u32, rdi, r8, 3), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rdi, r8, 3), w_r11), "4E8B5CC780", "movq -128(%rdi,%r8,8), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(-128i32 as u32, r8, r8, 0), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r8, r8, 0), w_r11), "4F8B5C0080", "movq -128(%r8,%r8,1), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(-128i32 as u32, r15, r8, 1), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r15, r8, 1), w_r11), "4F8B5C4780", "movq -128(%r15,%r8,2), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(-128i32 as u32, rax, r15, 1), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rax, r15, 1), w_r11), "4E8B5C7880", "movq -128(%rax,%r15,2), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(-128i32 as u32, rdi, r15, 0), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rdi, r15, 0), w_r11), "4E8B5C3F80", "movq -128(%rdi,%r15,1), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(-128i32 as u32, r8, r15, 3), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r8, r15, 3), w_r11), "4F8B5CF880", "movq -128(%r8,%r15,8), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(-128i32 as u32, r15, r15, 2), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r15, r15, 2), w_r11), "4F8B5CBF80", "movq -128(%r15,%r15,4), %r11", )); @@ -766,42 +766,42 @@ fn test_x64_emit() { // ======================================================== // Addr_IRRS, offset large positive simm32 insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(0x4f6625be, rax, rax, 0), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rax, rax, 0), w_r11), "4C8B9C00BE25664F", "movq 1332094398(%rax,%rax,1), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(0x4f6625be, rdi, rax, 1), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rdi, rax, 1), w_r11), "4C8B9C47BE25664F", "movq 1332094398(%rdi,%rax,2), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(0x4f6625be, r8, rax, 2), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r8, rax, 2), w_r11), "4D8B9C80BE25664F", "movq 1332094398(%r8,%rax,4), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(0x4f6625be, r15, rax, 3), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r15, rax, 3), w_r11), "4D8B9CC7BE25664F", "movq 1332094398(%r15,%rax,8), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(0x4f6625be, rax, rdi, 3), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rax, rdi, 3), w_r11), "4C8B9CF8BE25664F", "movq 1332094398(%rax,%rdi,8), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(0x4f6625be, rdi, rdi, 2), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rdi, rdi, 2), w_r11), "4C8B9CBFBE25664F", "movq 1332094398(%rdi,%rdi,4), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(0x4f6625be, r8, rdi, 1), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r8, rdi, 1), w_r11), "4D8B9C78BE25664F", "movq 1332094398(%r8,%rdi,2), %r11", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(0x4f6625be, r15, rdi, 0), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r15, rdi, 0), w_r11), "4D8B9C3FBE25664F", "movq 1332094398(%r15,%rdi,1), %r11", )); @@ -810,7 +810,7 @@ fn test_x64_emit() { // Addr_IRRS, offset large negative simm32 insns.push(( Inst::mov64_m_r( - Addr::imm_reg_reg_shift(-0x264d1690i32 as u32, rax, r8, 2), + Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rax, r8, 2), w_r11, ), "4E8B9C8070E9B2D9", @@ -818,7 +818,7 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Addr::imm_reg_reg_shift(-0x264d1690i32 as u32, rdi, r8, 3), + Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rdi, r8, 3), w_r11, ), "4E8B9CC770E9B2D9", @@ -826,7 +826,7 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Addr::imm_reg_reg_shift(-0x264d1690i32 as u32, r8, r8, 0), + Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r8, r8, 0), w_r11, ), "4F8B9C0070E9B2D9", @@ -834,7 +834,7 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Addr::imm_reg_reg_shift(-0x264d1690i32 as u32, r15, r8, 1), + Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r15, r8, 1), w_r11, ), "4F8B9C4770E9B2D9", @@ -842,7 +842,7 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Addr::imm_reg_reg_shift(-0x264d1690i32 as u32, rax, r15, 1), + Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rax, r15, 1), w_r11, ), "4E8B9C7870E9B2D9", @@ -850,7 +850,7 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Addr::imm_reg_reg_shift(-0x264d1690i32 as u32, rdi, r15, 0), + Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rdi, r15, 0), w_r11, ), "4E8B9C3F70E9B2D9", @@ -858,7 +858,7 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Addr::imm_reg_reg_shift(-0x264d1690i32 as u32, r8, r15, 3), + Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r8, r15, 3), w_r11, ), "4F8B9CF870E9B2D9", @@ -866,7 +866,7 @@ fn test_x64_emit() { )); insns.push(( Inst::mov64_m_r( - Addr::imm_reg_reg_shift(-0x264d1690i32 as u32, r15, r15, 2), + Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r15, r15, 2), w_r11, ), "4F8B9CBF70E9B2D9", @@ -882,184 +882,274 @@ fn test_x64_emit() { // // Alu_RMI_R insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Add, RMI::reg(r15), w_rdx), + Inst::alu_rmi_r(true, AluRmiROpcode::Add, RegMemImm::reg(r15), w_rdx), "4C01FA", "addq %r15, %rdx", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::reg(rcx), w_r8), + Inst::alu_rmi_r(false, AluRmiROpcode::Add, RegMemImm::reg(rcx), w_r8), "4101C8", "addl %ecx, %r8d", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::reg(rcx), w_rsi), + Inst::alu_rmi_r(false, AluRmiROpcode::Add, RegMemImm::reg(rcx), w_rsi), "01CE", "addl %ecx, %esi", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Add, RMI::mem(Addr::imm_reg(99, rdi)), w_rdx), + Inst::alu_rmi_r( + true, + AluRmiROpcode::Add, + RegMemImm::mem(Amode::imm_reg(99, rdi)), + w_rdx, + ), "48035763", "addq 99(%rdi), %rdx", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::mem(Addr::imm_reg(99, rdi)), w_r8), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Add, + RegMemImm::mem(Amode::imm_reg(99, rdi)), + w_r8, + ), "44034763", "addl 99(%rdi), %r8d", )); insns.push(( Inst::alu_rmi_r( false, - RMI_R_Op::Add, - RMI::mem(Addr::imm_reg(99, rdi)), + AluRmiROpcode::Add, + RegMemImm::mem(Amode::imm_reg(99, rdi)), w_rsi, ), "037763", "addl 99(%rdi), %esi", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Add, RMI::imm(-127i32 as u32), w_rdx), + Inst::alu_rmi_r( + true, + AluRmiROpcode::Add, + RegMemImm::imm(-127i32 as u32), + w_rdx, + ), "4883C281", "addq $-127, %rdx", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Add, RMI::imm(-129i32 as u32), w_rdx), + Inst::alu_rmi_r( + true, + AluRmiROpcode::Add, + RegMemImm::imm(-129i32 as u32), + w_rdx, + ), "4881C27FFFFFFF", "addq $-129, %rdx", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Add, RMI::imm(76543210), w_rdx), + Inst::alu_rmi_r(true, AluRmiROpcode::Add, RegMemImm::imm(76543210), w_rdx), "4881C2EAF48F04", "addq $76543210, %rdx", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::imm(-127i32 as u32), w_r8), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Add, + RegMemImm::imm(-127i32 as u32), + w_r8, + ), "4183C081", "addl $-127, %r8d", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::imm(-129i32 as u32), w_r8), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Add, + RegMemImm::imm(-129i32 as u32), + w_r8, + ), "4181C07FFFFFFF", "addl $-129, %r8d", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::imm(-76543210i32 as u32), w_r8), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Add, + RegMemImm::imm(-76543210i32 as u32), + w_r8, + ), "4181C0160B70FB", "addl $-76543210, %r8d", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::imm(-127i32 as u32), w_rsi), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Add, + RegMemImm::imm(-127i32 as u32), + w_rsi, + ), "83C681", "addl $-127, %esi", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::imm(-129i32 as u32), w_rsi), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Add, + RegMemImm::imm(-129i32 as u32), + w_rsi, + ), "81C67FFFFFFF", "addl $-129, %esi", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Add, RMI::imm(76543210), w_rsi), + Inst::alu_rmi_r(false, AluRmiROpcode::Add, RegMemImm::imm(76543210), w_rsi), "81C6EAF48F04", "addl $76543210, %esi", )); // This is pretty feeble insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Sub, RMI::reg(r15), w_rdx), + Inst::alu_rmi_r(true, AluRmiROpcode::Sub, RegMemImm::reg(r15), w_rdx), "4C29FA", "subq %r15, %rdx", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::And, RMI::reg(r15), w_rdx), + Inst::alu_rmi_r(true, AluRmiROpcode::And, RegMemImm::reg(r15), w_rdx), "4C21FA", "andq %r15, %rdx", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Or, RMI::reg(r15), w_rdx), + Inst::alu_rmi_r(true, AluRmiROpcode::Or, RegMemImm::reg(r15), w_rdx), "4C09FA", "orq %r15, %rdx", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Xor, RMI::reg(r15), w_rdx), + Inst::alu_rmi_r(true, AluRmiROpcode::Xor, RegMemImm::reg(r15), w_rdx), "4C31FA", "xorq %r15, %rdx", )); // Test all mul cases, though insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Mul, RMI::reg(r15), w_rdx), + Inst::alu_rmi_r(true, AluRmiROpcode::Mul, RegMemImm::reg(r15), w_rdx), "490FAFD7", "imulq %r15, %rdx", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::reg(rcx), w_r8), + Inst::alu_rmi_r(false, AluRmiROpcode::Mul, RegMemImm::reg(rcx), w_r8), "440FAFC1", "imull %ecx, %r8d", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::reg(rcx), w_rsi), + Inst::alu_rmi_r(false, AluRmiROpcode::Mul, RegMemImm::reg(rcx), w_rsi), "0FAFF1", "imull %ecx, %esi", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Mul, RMI::mem(Addr::imm_reg(99, rdi)), w_rdx), + Inst::alu_rmi_r( + true, + AluRmiROpcode::Mul, + RegMemImm::mem(Amode::imm_reg(99, rdi)), + w_rdx, + ), "480FAF5763", "imulq 99(%rdi), %rdx", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::mem(Addr::imm_reg(99, rdi)), w_r8), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Mul, + RegMemImm::mem(Amode::imm_reg(99, rdi)), + w_r8, + ), "440FAF4763", "imull 99(%rdi), %r8d", )); insns.push(( Inst::alu_rmi_r( false, - RMI_R_Op::Mul, - RMI::mem(Addr::imm_reg(99, rdi)), + AluRmiROpcode::Mul, + RegMemImm::mem(Amode::imm_reg(99, rdi)), w_rsi, ), "0FAF7763", "imull 99(%rdi), %esi", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Mul, RMI::imm(-127i32 as u32), w_rdx), + Inst::alu_rmi_r( + true, + AluRmiROpcode::Mul, + RegMemImm::imm(-127i32 as u32), + w_rdx, + ), "486BD281", "imulq $-127, %rdx", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Mul, RMI::imm(-129i32 as u32), w_rdx), + Inst::alu_rmi_r( + true, + AluRmiROpcode::Mul, + RegMemImm::imm(-129i32 as u32), + w_rdx, + ), "4869D27FFFFFFF", "imulq $-129, %rdx", )); insns.push(( - Inst::alu_rmi_r(true, RMI_R_Op::Mul, RMI::imm(76543210), w_rdx), + Inst::alu_rmi_r(true, AluRmiROpcode::Mul, RegMemImm::imm(76543210), w_rdx), "4869D2EAF48F04", "imulq $76543210, %rdx", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::imm(-127i32 as u32), w_r8), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Mul, + RegMemImm::imm(-127i32 as u32), + w_r8, + ), "456BC081", "imull $-127, %r8d", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::imm(-129i32 as u32), w_r8), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Mul, + RegMemImm::imm(-129i32 as u32), + w_r8, + ), "4569C07FFFFFFF", "imull $-129, %r8d", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::imm(-76543210i32 as u32), w_r8), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Mul, + RegMemImm::imm(-76543210i32 as u32), + w_r8, + ), "4569C0160B70FB", "imull $-76543210, %r8d", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::imm(-127i32 as u32), w_rsi), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Mul, + RegMemImm::imm(-127i32 as u32), + w_rsi, + ), "6BF681", "imull $-127, %esi", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::imm(-129i32 as u32), w_rsi), + Inst::alu_rmi_r( + false, + AluRmiROpcode::Mul, + RegMemImm::imm(-129i32 as u32), + w_rsi, + ), "69F67FFFFFFF", "imull $-129, %esi", )); insns.push(( - Inst::alu_rmi_r(false, RMI_R_Op::Mul, RMI::imm(76543210), w_rsi), + Inst::alu_rmi_r(false, AluRmiROpcode::Mul, RegMemImm::imm(76543210), w_rsi), "69F6EAF48F04", "imull $76543210, %esi", )); @@ -1152,104 +1242,229 @@ fn test_x64_emit() { )); // ======================================================== - // MovZX_M_R + // MovZX_RM_R insns.push(( - Inst::movzx_m_r(ExtMode::BL, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movzx_rm_r(ExtMode::BL, RegMem::reg(rax), w_rsi), + "0FB6F0", + "movzbl %al, %esi", + )); + insns.push(( + Inst::movzx_rm_r(ExtMode::BL, RegMem::reg(r15), w_rsi), + "410FB6F7", + "movzbl %r15b, %esi", + )); + insns.push(( + Inst::movzx_rm_r( + ExtMode::BL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "0FB671F9", "movzbl -7(%rcx), %esi", )); insns.push(( - Inst::movzx_m_r(ExtMode::BL, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movzx_rm_r( + ExtMode::BL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "410FB658F9", "movzbl -7(%r8), %ebx", )); insns.push(( - Inst::movzx_m_r(ExtMode::BL, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movzx_rm_r( + ExtMode::BL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "450FB64AF9", "movzbl -7(%r10), %r9d", )); insns.push(( - Inst::movzx_m_r(ExtMode::BL, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movzx_rm_r( + ExtMode::BL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "410FB653F9", "movzbl -7(%r11), %edx", )); insns.push(( - Inst::movzx_m_r(ExtMode::BQ, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movzx_rm_r(ExtMode::BQ, RegMem::reg(rax), w_rsi), + "480FB6F0", + "movzbq %al, %rsi", + )); + insns.push(( + Inst::movzx_rm_r(ExtMode::BQ, RegMem::reg(r10), w_rsi), + "490FB6F2", + "movzbq %r10b, %rsi", + )); + insns.push(( + Inst::movzx_rm_r( + ExtMode::BQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "480FB671F9", "movzbq -7(%rcx), %rsi", )); insns.push(( - Inst::movzx_m_r(ExtMode::BQ, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movzx_rm_r( + ExtMode::BQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "490FB658F9", "movzbq -7(%r8), %rbx", )); insns.push(( - Inst::movzx_m_r(ExtMode::BQ, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movzx_rm_r( + ExtMode::BQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "4D0FB64AF9", "movzbq -7(%r10), %r9", )); insns.push(( - Inst::movzx_m_r(ExtMode::BQ, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movzx_rm_r( + ExtMode::BQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "490FB653F9", "movzbq -7(%r11), %rdx", )); insns.push(( - Inst::movzx_m_r(ExtMode::WL, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movzx_rm_r(ExtMode::WL, RegMem::reg(rcx), w_rsi), + "0FB7F1", + "movzwl %cx, %esi", + )); + insns.push(( + Inst::movzx_rm_r(ExtMode::WL, RegMem::reg(r10), w_rsi), + "410FB7F2", + "movzwl %r10w, %esi", + )); + insns.push(( + Inst::movzx_rm_r( + ExtMode::WL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "0FB771F9", "movzwl -7(%rcx), %esi", )); insns.push(( - Inst::movzx_m_r(ExtMode::WL, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movzx_rm_r( + ExtMode::WL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "410FB758F9", "movzwl -7(%r8), %ebx", )); insns.push(( - Inst::movzx_m_r(ExtMode::WL, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movzx_rm_r( + ExtMode::WL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "450FB74AF9", "movzwl -7(%r10), %r9d", )); insns.push(( - Inst::movzx_m_r(ExtMode::WL, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movzx_rm_r( + ExtMode::WL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "410FB753F9", "movzwl -7(%r11), %edx", )); insns.push(( - Inst::movzx_m_r(ExtMode::WQ, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movzx_rm_r(ExtMode::WQ, RegMem::reg(rcx), w_rsi), + "480FB7F1", + "movzwq %cx, %rsi", + )); + insns.push(( + Inst::movzx_rm_r(ExtMode::WQ, RegMem::reg(r11), w_rsi), + "490FB7F3", + "movzwq %r11w, %rsi", + )); + insns.push(( + Inst::movzx_rm_r( + ExtMode::WQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "480FB771F9", "movzwq -7(%rcx), %rsi", )); insns.push(( - Inst::movzx_m_r(ExtMode::WQ, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movzx_rm_r( + ExtMode::WQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "490FB758F9", "movzwq -7(%r8), %rbx", )); insns.push(( - Inst::movzx_m_r(ExtMode::WQ, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movzx_rm_r( + ExtMode::WQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "4D0FB74AF9", "movzwq -7(%r10), %r9", )); insns.push(( - Inst::movzx_m_r(ExtMode::WQ, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movzx_rm_r( + ExtMode::WQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "490FB753F9", "movzwq -7(%r11), %rdx", )); insns.push(( - Inst::movzx_m_r(ExtMode::LQ, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(rcx), w_rsi), + "8BF1", + "movl %ecx, %esi", + )); + insns.push(( + Inst::movzx_rm_r( + ExtMode::LQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "8B71F9", "movl -7(%rcx), %esi", )); insns.push(( - Inst::movzx_m_r(ExtMode::LQ, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movzx_rm_r( + ExtMode::LQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "418B58F9", "movl -7(%r8), %ebx", )); insns.push(( - Inst::movzx_m_r(ExtMode::LQ, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movzx_rm_r( + ExtMode::LQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "458B4AF9", "movl -7(%r10), %r9d", )); insns.push(( - Inst::movzx_m_r(ExtMode::LQ, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movzx_rm_r( + ExtMode::LQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "418B53F9", "movl -7(%r11), %edx", )); @@ -1257,145 +1472,293 @@ fn test_x64_emit() { // ======================================================== // Mov64_M_R insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(179, rax, rbx, 0), w_rcx), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, rbx, 0), w_rcx), "488B8C18B3000000", "movq 179(%rax,%rbx,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(179, rax, rbx, 0), w_r8), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, rbx, 0), w_r8), "4C8B8418B3000000", "movq 179(%rax,%rbx,1), %r8", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(179, rax, r9, 0), w_rcx), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, r9, 0), w_rcx), "4A8B8C08B3000000", "movq 179(%rax,%r9,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(179, rax, r9, 0), w_r8), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, r9, 0), w_r8), "4E8B8408B3000000", "movq 179(%rax,%r9,1), %r8", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(179, r10, rbx, 0), w_rcx), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, rbx, 0), w_rcx), "498B8C1AB3000000", "movq 179(%r10,%rbx,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(179, r10, rbx, 0), w_r8), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, rbx, 0), w_r8), "4D8B841AB3000000", "movq 179(%r10,%rbx,1), %r8", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(179, r10, r9, 0), w_rcx), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_rcx), "4B8B8C0AB3000000", "movq 179(%r10,%r9,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Addr::imm_reg_reg_shift(179, r10, r9, 0), w_r8), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_r8), "4F8B840AB3000000", "movq 179(%r10,%r9,1), %r8", )); // ======================================================== - // MovSX_M_R + // LoadEffectiveAddress insns.push(( - Inst::movsx_m_r(ExtMode::BL, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::lea(Amode::imm_reg(42, r10), w_r8), + "4D8D422A", + "lea 42(%r10), %r8", + )); + insns.push(( + Inst::lea(Amode::imm_reg(42, r10), w_r15), + "4D8D7A2A", + "lea 42(%r10), %r15", + )); + insns.push(( + Inst::lea(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_r8), + "4F8D840AB3000000", + "lea 179(%r10,%r9,1), %r8", + )); + + // ======================================================== + // MovSX_RM_R + insns.push(( + Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rcx), w_rsi), + "0FBEF1", + "movsbl %cl, %esi", + )); + insns.push(( + Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(r14), w_rsi), + "410FBEF6", + "movsbl %r14b, %esi", + )); + insns.push(( + Inst::movsx_rm_r( + ExtMode::BL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "0FBE71F9", "movsbl -7(%rcx), %esi", )); insns.push(( - Inst::movsx_m_r(ExtMode::BL, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movsx_rm_r( + ExtMode::BL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "410FBE58F9", "movsbl -7(%r8), %ebx", )); insns.push(( - Inst::movsx_m_r(ExtMode::BL, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movsx_rm_r( + ExtMode::BL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "450FBE4AF9", "movsbl -7(%r10), %r9d", )); insns.push(( - Inst::movsx_m_r(ExtMode::BL, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movsx_rm_r( + ExtMode::BL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "410FBE53F9", "movsbl -7(%r11), %edx", )); insns.push(( - Inst::movsx_m_r(ExtMode::BQ, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::BQ, RegMem::reg(rcx), w_rsi), + "480FBEF1", + "movsbq %cl, %rsi", + )); + insns.push(( + Inst::movsx_rm_r(ExtMode::BQ, RegMem::reg(r15), w_rsi), + "490FBEF7", + "movsbq %r15b, %rsi", + )); + insns.push(( + Inst::movsx_rm_r( + ExtMode::BQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "480FBE71F9", "movsbq -7(%rcx), %rsi", )); insns.push(( - Inst::movsx_m_r(ExtMode::BQ, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movsx_rm_r( + ExtMode::BQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "490FBE58F9", "movsbq -7(%r8), %rbx", )); insns.push(( - Inst::movsx_m_r(ExtMode::BQ, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movsx_rm_r( + ExtMode::BQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "4D0FBE4AF9", "movsbq -7(%r10), %r9", )); insns.push(( - Inst::movsx_m_r(ExtMode::BQ, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movsx_rm_r( + ExtMode::BQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "490FBE53F9", "movsbq -7(%r11), %rdx", )); insns.push(( - Inst::movsx_m_r(ExtMode::WL, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::WL, RegMem::reg(rcx), w_rsi), + "0FBFF1", + "movswl %cx, %esi", + )); + insns.push(( + Inst::movsx_rm_r(ExtMode::WL, RegMem::reg(r14), w_rsi), + "410FBFF6", + "movswl %r14w, %esi", + )); + insns.push(( + Inst::movsx_rm_r( + ExtMode::WL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "0FBF71F9", "movswl -7(%rcx), %esi", )); insns.push(( - Inst::movsx_m_r(ExtMode::WL, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movsx_rm_r( + ExtMode::WL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "410FBF58F9", "movswl -7(%r8), %ebx", )); insns.push(( - Inst::movsx_m_r(ExtMode::WL, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movsx_rm_r( + ExtMode::WL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "450FBF4AF9", "movswl -7(%r10), %r9d", )); insns.push(( - Inst::movsx_m_r(ExtMode::WL, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movsx_rm_r( + ExtMode::WL, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "410FBF53F9", "movswl -7(%r11), %edx", )); insns.push(( - Inst::movsx_m_r(ExtMode::WQ, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::WQ, RegMem::reg(rcx), w_rsi), + "480FBFF1", + "movswq %cx, %rsi", + )); + insns.push(( + Inst::movsx_rm_r(ExtMode::WQ, RegMem::reg(r13), w_rsi), + "490FBFF5", + "movswq %r13w, %rsi", + )); + insns.push(( + Inst::movsx_rm_r( + ExtMode::WQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "480FBF71F9", "movswq -7(%rcx), %rsi", )); insns.push(( - Inst::movsx_m_r(ExtMode::WQ, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movsx_rm_r( + ExtMode::WQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "490FBF58F9", "movswq -7(%r8), %rbx", )); insns.push(( - Inst::movsx_m_r(ExtMode::WQ, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movsx_rm_r( + ExtMode::WQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "4D0FBF4AF9", "movswq -7(%r10), %r9", )); insns.push(( - Inst::movsx_m_r(ExtMode::WQ, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movsx_rm_r( + ExtMode::WQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "490FBF53F9", "movswq -7(%r11), %rdx", )); insns.push(( - Inst::movsx_m_r(ExtMode::LQ, Addr::imm_reg(-7i32 as u32, rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::LQ, RegMem::reg(rcx), w_rsi), + "4863F1", + "movslq %ecx, %rsi", + )); + insns.push(( + Inst::movsx_rm_r(ExtMode::LQ, RegMem::reg(r15), w_rsi), + "4963F7", + "movslq %r15d, %rsi", + )); + insns.push(( + Inst::movsx_rm_r( + ExtMode::LQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), + w_rsi, + ), "486371F9", "movslq -7(%rcx), %rsi", )); insns.push(( - Inst::movsx_m_r(ExtMode::LQ, Addr::imm_reg(-7i32 as u32, r8), w_rbx), + Inst::movsx_rm_r( + ExtMode::LQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), + w_rbx, + ), "496358F9", "movslq -7(%r8), %rbx", )); insns.push(( - Inst::movsx_m_r(ExtMode::LQ, Addr::imm_reg(-7i32 as u32, r10), w_r9), + Inst::movsx_rm_r( + ExtMode::LQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), + w_r9, + ), "4D634AF9", "movslq -7(%r10), %r9", )); insns.push(( - Inst::movsx_m_r(ExtMode::LQ, Addr::imm_reg(-7i32 as u32, r11), w_rdx), + Inst::movsx_rm_r( + ExtMode::LQ, + RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), + w_rdx, + ), "496353F9", "movslq -7(%r11), %rdx", )); @@ -1403,325 +1766,325 @@ fn test_x64_emit() { // ======================================================== // Mov_R_M. Byte stores are tricky. Check everything carefully. insns.push(( - Inst::mov_r_m(8, rax, Addr::imm_reg(99, rdi)), + Inst::mov_r_m(8, rax, Amode::imm_reg(99, rdi)), "48894763", "movq %rax, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(8, rbx, Addr::imm_reg(99, r8)), + Inst::mov_r_m(8, rbx, Amode::imm_reg(99, r8)), "49895863", "movq %rbx, 99(%r8)", )); insns.push(( - Inst::mov_r_m(8, rcx, Addr::imm_reg(99, rsi)), + Inst::mov_r_m(8, rcx, Amode::imm_reg(99, rsi)), "48894E63", "movq %rcx, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(8, rdx, Addr::imm_reg(99, r9)), + Inst::mov_r_m(8, rdx, Amode::imm_reg(99, r9)), "49895163", "movq %rdx, 99(%r9)", )); insns.push(( - Inst::mov_r_m(8, rsi, Addr::imm_reg(99, rax)), + Inst::mov_r_m(8, rsi, Amode::imm_reg(99, rax)), "48897063", "movq %rsi, 99(%rax)", )); insns.push(( - Inst::mov_r_m(8, rdi, Addr::imm_reg(99, r15)), + Inst::mov_r_m(8, rdi, Amode::imm_reg(99, r15)), "49897F63", "movq %rdi, 99(%r15)", )); insns.push(( - Inst::mov_r_m(8, rsp, Addr::imm_reg(99, rcx)), + Inst::mov_r_m(8, rsp, Amode::imm_reg(99, rcx)), "48896163", "movq %rsp, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(8, rbp, Addr::imm_reg(99, r14)), + Inst::mov_r_m(8, rbp, Amode::imm_reg(99, r14)), "49896E63", "movq %rbp, 99(%r14)", )); insns.push(( - Inst::mov_r_m(8, r8, Addr::imm_reg(99, rdi)), + Inst::mov_r_m(8, r8, Amode::imm_reg(99, rdi)), "4C894763", "movq %r8, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(8, r9, Addr::imm_reg(99, r8)), + Inst::mov_r_m(8, r9, Amode::imm_reg(99, r8)), "4D894863", "movq %r9, 99(%r8)", )); insns.push(( - Inst::mov_r_m(8, r10, Addr::imm_reg(99, rsi)), + Inst::mov_r_m(8, r10, Amode::imm_reg(99, rsi)), "4C895663", "movq %r10, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(8, r11, Addr::imm_reg(99, r9)), + Inst::mov_r_m(8, r11, Amode::imm_reg(99, r9)), "4D895963", "movq %r11, 99(%r9)", )); insns.push(( - Inst::mov_r_m(8, r12, Addr::imm_reg(99, rax)), + Inst::mov_r_m(8, r12, Amode::imm_reg(99, rax)), "4C896063", "movq %r12, 99(%rax)", )); insns.push(( - Inst::mov_r_m(8, r13, Addr::imm_reg(99, r15)), + Inst::mov_r_m(8, r13, Amode::imm_reg(99, r15)), "4D896F63", "movq %r13, 99(%r15)", )); insns.push(( - Inst::mov_r_m(8, r14, Addr::imm_reg(99, rcx)), + Inst::mov_r_m(8, r14, Amode::imm_reg(99, rcx)), "4C897163", "movq %r14, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(8, r15, Addr::imm_reg(99, r14)), + Inst::mov_r_m(8, r15, Amode::imm_reg(99, r14)), "4D897E63", "movq %r15, 99(%r14)", )); // insns.push(( - Inst::mov_r_m(4, rax, Addr::imm_reg(99, rdi)), + Inst::mov_r_m(4, rax, Amode::imm_reg(99, rdi)), "894763", "movl %eax, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(4, rbx, Addr::imm_reg(99, r8)), + Inst::mov_r_m(4, rbx, Amode::imm_reg(99, r8)), "41895863", "movl %ebx, 99(%r8)", )); insns.push(( - Inst::mov_r_m(4, rcx, Addr::imm_reg(99, rsi)), + Inst::mov_r_m(4, rcx, Amode::imm_reg(99, rsi)), "894E63", "movl %ecx, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(4, rdx, Addr::imm_reg(99, r9)), + Inst::mov_r_m(4, rdx, Amode::imm_reg(99, r9)), "41895163", "movl %edx, 99(%r9)", )); insns.push(( - Inst::mov_r_m(4, rsi, Addr::imm_reg(99, rax)), + Inst::mov_r_m(4, rsi, Amode::imm_reg(99, rax)), "897063", "movl %esi, 99(%rax)", )); insns.push(( - Inst::mov_r_m(4, rdi, Addr::imm_reg(99, r15)), + Inst::mov_r_m(4, rdi, Amode::imm_reg(99, r15)), "41897F63", "movl %edi, 99(%r15)", )); insns.push(( - Inst::mov_r_m(4, rsp, Addr::imm_reg(99, rcx)), + Inst::mov_r_m(4, rsp, Amode::imm_reg(99, rcx)), "896163", "movl %esp, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(4, rbp, Addr::imm_reg(99, r14)), + Inst::mov_r_m(4, rbp, Amode::imm_reg(99, r14)), "41896E63", "movl %ebp, 99(%r14)", )); insns.push(( - Inst::mov_r_m(4, r8, Addr::imm_reg(99, rdi)), + Inst::mov_r_m(4, r8, Amode::imm_reg(99, rdi)), "44894763", "movl %r8d, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(4, r9, Addr::imm_reg(99, r8)), + Inst::mov_r_m(4, r9, Amode::imm_reg(99, r8)), "45894863", "movl %r9d, 99(%r8)", )); insns.push(( - Inst::mov_r_m(4, r10, Addr::imm_reg(99, rsi)), + Inst::mov_r_m(4, r10, Amode::imm_reg(99, rsi)), "44895663", "movl %r10d, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(4, r11, Addr::imm_reg(99, r9)), + Inst::mov_r_m(4, r11, Amode::imm_reg(99, r9)), "45895963", "movl %r11d, 99(%r9)", )); insns.push(( - Inst::mov_r_m(4, r12, Addr::imm_reg(99, rax)), + Inst::mov_r_m(4, r12, Amode::imm_reg(99, rax)), "44896063", "movl %r12d, 99(%rax)", )); insns.push(( - Inst::mov_r_m(4, r13, Addr::imm_reg(99, r15)), + Inst::mov_r_m(4, r13, Amode::imm_reg(99, r15)), "45896F63", "movl %r13d, 99(%r15)", )); insns.push(( - Inst::mov_r_m(4, r14, Addr::imm_reg(99, rcx)), + Inst::mov_r_m(4, r14, Amode::imm_reg(99, rcx)), "44897163", "movl %r14d, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(4, r15, Addr::imm_reg(99, r14)), + Inst::mov_r_m(4, r15, Amode::imm_reg(99, r14)), "45897E63", "movl %r15d, 99(%r14)", )); // insns.push(( - Inst::mov_r_m(2, rax, Addr::imm_reg(99, rdi)), + Inst::mov_r_m(2, rax, Amode::imm_reg(99, rdi)), "66894763", "movw %ax, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(2, rbx, Addr::imm_reg(99, r8)), + Inst::mov_r_m(2, rbx, Amode::imm_reg(99, r8)), "6641895863", "movw %bx, 99(%r8)", )); insns.push(( - Inst::mov_r_m(2, rcx, Addr::imm_reg(99, rsi)), + Inst::mov_r_m(2, rcx, Amode::imm_reg(99, rsi)), "66894E63", "movw %cx, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(2, rdx, Addr::imm_reg(99, r9)), + Inst::mov_r_m(2, rdx, Amode::imm_reg(99, r9)), "6641895163", "movw %dx, 99(%r9)", )); insns.push(( - Inst::mov_r_m(2, rsi, Addr::imm_reg(99, rax)), + Inst::mov_r_m(2, rsi, Amode::imm_reg(99, rax)), "66897063", "movw %si, 99(%rax)", )); insns.push(( - Inst::mov_r_m(2, rdi, Addr::imm_reg(99, r15)), + Inst::mov_r_m(2, rdi, Amode::imm_reg(99, r15)), "6641897F63", "movw %di, 99(%r15)", )); insns.push(( - Inst::mov_r_m(2, rsp, Addr::imm_reg(99, rcx)), + Inst::mov_r_m(2, rsp, Amode::imm_reg(99, rcx)), "66896163", "movw %sp, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(2, rbp, Addr::imm_reg(99, r14)), + Inst::mov_r_m(2, rbp, Amode::imm_reg(99, r14)), "6641896E63", "movw %bp, 99(%r14)", )); insns.push(( - Inst::mov_r_m(2, r8, Addr::imm_reg(99, rdi)), + Inst::mov_r_m(2, r8, Amode::imm_reg(99, rdi)), "6644894763", "movw %r8w, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(2, r9, Addr::imm_reg(99, r8)), + Inst::mov_r_m(2, r9, Amode::imm_reg(99, r8)), "6645894863", "movw %r9w, 99(%r8)", )); insns.push(( - Inst::mov_r_m(2, r10, Addr::imm_reg(99, rsi)), + Inst::mov_r_m(2, r10, Amode::imm_reg(99, rsi)), "6644895663", "movw %r10w, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(2, r11, Addr::imm_reg(99, r9)), + Inst::mov_r_m(2, r11, Amode::imm_reg(99, r9)), "6645895963", "movw %r11w, 99(%r9)", )); insns.push(( - Inst::mov_r_m(2, r12, Addr::imm_reg(99, rax)), + Inst::mov_r_m(2, r12, Amode::imm_reg(99, rax)), "6644896063", "movw %r12w, 99(%rax)", )); insns.push(( - Inst::mov_r_m(2, r13, Addr::imm_reg(99, r15)), + Inst::mov_r_m(2, r13, Amode::imm_reg(99, r15)), "6645896F63", "movw %r13w, 99(%r15)", )); insns.push(( - Inst::mov_r_m(2, r14, Addr::imm_reg(99, rcx)), + Inst::mov_r_m(2, r14, Amode::imm_reg(99, rcx)), "6644897163", "movw %r14w, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(2, r15, Addr::imm_reg(99, r14)), + Inst::mov_r_m(2, r15, Amode::imm_reg(99, r14)), "6645897E63", "movw %r15w, 99(%r14)", )); // insns.push(( - Inst::mov_r_m(1, rax, Addr::imm_reg(99, rdi)), + Inst::mov_r_m(1, rax, Amode::imm_reg(99, rdi)), "884763", "movb %al, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(1, rbx, Addr::imm_reg(99, r8)), + Inst::mov_r_m(1, rbx, Amode::imm_reg(99, r8)), "41885863", "movb %bl, 99(%r8)", )); insns.push(( - Inst::mov_r_m(1, rcx, Addr::imm_reg(99, rsi)), + Inst::mov_r_m(1, rcx, Amode::imm_reg(99, rsi)), "884E63", "movb %cl, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(1, rdx, Addr::imm_reg(99, r9)), + Inst::mov_r_m(1, rdx, Amode::imm_reg(99, r9)), "41885163", "movb %dl, 99(%r9)", )); insns.push(( - Inst::mov_r_m(1, rsi, Addr::imm_reg(99, rax)), + Inst::mov_r_m(1, rsi, Amode::imm_reg(99, rax)), "40887063", "movb %sil, 99(%rax)", )); insns.push(( - Inst::mov_r_m(1, rdi, Addr::imm_reg(99, r15)), + Inst::mov_r_m(1, rdi, Amode::imm_reg(99, r15)), "41887F63", "movb %dil, 99(%r15)", )); insns.push(( - Inst::mov_r_m(1, rsp, Addr::imm_reg(99, rcx)), + Inst::mov_r_m(1, rsp, Amode::imm_reg(99, rcx)), "40886163", "movb %spl, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(1, rbp, Addr::imm_reg(99, r14)), + Inst::mov_r_m(1, rbp, Amode::imm_reg(99, r14)), "41886E63", "movb %bpl, 99(%r14)", )); insns.push(( - Inst::mov_r_m(1, r8, Addr::imm_reg(99, rdi)), + Inst::mov_r_m(1, r8, Amode::imm_reg(99, rdi)), "44884763", "movb %r8b, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(1, r9, Addr::imm_reg(99, r8)), + Inst::mov_r_m(1, r9, Amode::imm_reg(99, r8)), "45884863", "movb %r9b, 99(%r8)", )); insns.push(( - Inst::mov_r_m(1, r10, Addr::imm_reg(99, rsi)), + Inst::mov_r_m(1, r10, Amode::imm_reg(99, rsi)), "44885663", "movb %r10b, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(1, r11, Addr::imm_reg(99, r9)), + Inst::mov_r_m(1, r11, Amode::imm_reg(99, r9)), "45885963", "movb %r11b, 99(%r9)", )); insns.push(( - Inst::mov_r_m(1, r12, Addr::imm_reg(99, rax)), + Inst::mov_r_m(1, r12, Amode::imm_reg(99, rax)), "44886063", "movb %r12b, 99(%rax)", )); insns.push(( - Inst::mov_r_m(1, r13, Addr::imm_reg(99, r15)), + Inst::mov_r_m(1, r13, Amode::imm_reg(99, r15)), "45886F63", "movb %r13b, 99(%r15)", )); insns.push(( - Inst::mov_r_m(1, r14, Addr::imm_reg(99, rcx)), + Inst::mov_r_m(1, r14, Amode::imm_reg(99, rcx)), "44887163", "movb %r14b, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(1, r15, Addr::imm_reg(99, r14)), + Inst::mov_r_m(1, r15, Amode::imm_reg(99, r14)), "45887E63", "movb %r15b, 99(%r14)", )); @@ -1837,314 +2200,325 @@ fn test_x64_emit() { // ======================================================== // CmpRMIR insns.push(( - Inst::cmp_rmi_r(8, RMI::reg(r15), rdx), + Inst::cmp_rmi_r(8, RegMemImm::reg(r15), rdx), "4C39FA", "cmpq %r15, %rdx", )); insns.push(( - Inst::cmp_rmi_r(8, RMI::reg(rcx), r8), + Inst::cmp_rmi_r(8, RegMemImm::reg(rcx), r8), "4939C8", "cmpq %rcx, %r8", )); insns.push(( - Inst::cmp_rmi_r(8, RMI::reg(rcx), rsi), + Inst::cmp_rmi_r(8, RegMemImm::reg(rcx), rsi), "4839CE", "cmpq %rcx, %rsi", )); insns.push(( - Inst::cmp_rmi_r(8, RMI::mem(Addr::imm_reg(99, rdi)), rdx), + Inst::cmp_rmi_r(8, RegMemImm::mem(Amode::imm_reg(99, rdi)), rdx), "483B5763", "cmpq 99(%rdi), %rdx", )); insns.push(( - Inst::cmp_rmi_r(8, RMI::mem(Addr::imm_reg(99, rdi)), r8), + Inst::cmp_rmi_r(8, RegMemImm::mem(Amode::imm_reg(99, rdi)), r8), "4C3B4763", "cmpq 99(%rdi), %r8", )); insns.push(( - Inst::cmp_rmi_r(8, RMI::mem(Addr::imm_reg(99, rdi)), rsi), + Inst::cmp_rmi_r(8, RegMemImm::mem(Amode::imm_reg(99, rdi)), rsi), "483B7763", "cmpq 99(%rdi), %rsi", )); insns.push(( - Inst::cmp_rmi_r(8, RMI::imm(76543210), rdx), + Inst::cmp_rmi_r(8, RegMemImm::imm(76543210), rdx), "4881FAEAF48F04", "cmpq $76543210, %rdx", )); insns.push(( - Inst::cmp_rmi_r(8, RMI::imm(-76543210i32 as u32), r8), + Inst::cmp_rmi_r(8, RegMemImm::imm(-76543210i32 as u32), r8), "4981F8160B70FB", "cmpq $-76543210, %r8", )); insns.push(( - Inst::cmp_rmi_r(8, RMI::imm(76543210), rsi), + Inst::cmp_rmi_r(8, RegMemImm::imm(76543210), rsi), "4881FEEAF48F04", "cmpq $76543210, %rsi", )); // insns.push(( - Inst::cmp_rmi_r(4, RMI::reg(r15), rdx), + Inst::cmp_rmi_r(4, RegMemImm::reg(r15), rdx), "4439FA", "cmpl %r15d, %edx", )); insns.push(( - Inst::cmp_rmi_r(4, RMI::reg(rcx), r8), + Inst::cmp_rmi_r(4, RegMemImm::reg(rcx), r8), "4139C8", "cmpl %ecx, %r8d", )); insns.push(( - Inst::cmp_rmi_r(4, RMI::reg(rcx), rsi), + Inst::cmp_rmi_r(4, RegMemImm::reg(rcx), rsi), "39CE", "cmpl %ecx, %esi", )); insns.push(( - Inst::cmp_rmi_r(4, RMI::mem(Addr::imm_reg(99, rdi)), rdx), + Inst::cmp_rmi_r(4, RegMemImm::mem(Amode::imm_reg(99, rdi)), rdx), "3B5763", "cmpl 99(%rdi), %edx", )); insns.push(( - Inst::cmp_rmi_r(4, RMI::mem(Addr::imm_reg(99, rdi)), r8), + Inst::cmp_rmi_r(4, RegMemImm::mem(Amode::imm_reg(99, rdi)), r8), "443B4763", "cmpl 99(%rdi), %r8d", )); insns.push(( - Inst::cmp_rmi_r(4, RMI::mem(Addr::imm_reg(99, rdi)), rsi), + Inst::cmp_rmi_r(4, RegMemImm::mem(Amode::imm_reg(99, rdi)), rsi), "3B7763", "cmpl 99(%rdi), %esi", )); insns.push(( - Inst::cmp_rmi_r(4, RMI::imm(76543210), rdx), + Inst::cmp_rmi_r(4, RegMemImm::imm(76543210), rdx), "81FAEAF48F04", "cmpl $76543210, %edx", )); insns.push(( - Inst::cmp_rmi_r(4, RMI::imm(-76543210i32 as u32), r8), + Inst::cmp_rmi_r(4, RegMemImm::imm(-76543210i32 as u32), r8), "4181F8160B70FB", "cmpl $-76543210, %r8d", )); insns.push(( - Inst::cmp_rmi_r(4, RMI::imm(76543210), rsi), + Inst::cmp_rmi_r(4, RegMemImm::imm(76543210), rsi), "81FEEAF48F04", "cmpl $76543210, %esi", )); // insns.push(( - Inst::cmp_rmi_r(2, RMI::reg(r15), rdx), + Inst::cmp_rmi_r(2, RegMemImm::reg(r15), rdx), "664439FA", "cmpw %r15w, %dx", )); insns.push(( - Inst::cmp_rmi_r(2, RMI::reg(rcx), r8), + Inst::cmp_rmi_r(2, RegMemImm::reg(rcx), r8), "664139C8", "cmpw %cx, %r8w", )); insns.push(( - Inst::cmp_rmi_r(2, RMI::reg(rcx), rsi), + Inst::cmp_rmi_r(2, RegMemImm::reg(rcx), rsi), "6639CE", "cmpw %cx, %si", )); insns.push(( - Inst::cmp_rmi_r(2, RMI::mem(Addr::imm_reg(99, rdi)), rdx), + Inst::cmp_rmi_r(2, RegMemImm::mem(Amode::imm_reg(99, rdi)), rdx), "663B5763", "cmpw 99(%rdi), %dx", )); insns.push(( - Inst::cmp_rmi_r(2, RMI::mem(Addr::imm_reg(99, rdi)), r8), + Inst::cmp_rmi_r(2, RegMemImm::mem(Amode::imm_reg(99, rdi)), r8), "66443B4763", "cmpw 99(%rdi), %r8w", )); insns.push(( - Inst::cmp_rmi_r(2, RMI::mem(Addr::imm_reg(99, rdi)), rsi), + Inst::cmp_rmi_r(2, RegMemImm::mem(Amode::imm_reg(99, rdi)), rsi), "663B7763", "cmpw 99(%rdi), %si", )); insns.push(( - Inst::cmp_rmi_r(2, RMI::imm(23210), rdx), + Inst::cmp_rmi_r(2, RegMemImm::imm(23210), rdx), "6681FAAA5A", "cmpw $23210, %dx", )); insns.push(( - Inst::cmp_rmi_r(2, RMI::imm(-7654i32 as u32), r8), + Inst::cmp_rmi_r(2, RegMemImm::imm(-7654i32 as u32), r8), "664181F81AE2", "cmpw $-7654, %r8w", )); insns.push(( - Inst::cmp_rmi_r(2, RMI::imm(7654), rsi), + Inst::cmp_rmi_r(2, RegMemImm::imm(7654), rsi), "6681FEE61D", "cmpw $7654, %si", )); // insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r15), rdx), + Inst::cmp_rmi_r(1, RegMemImm::reg(r15), rdx), "4438FA", "cmpb %r15b, %dl", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rcx), r8), + Inst::cmp_rmi_r(1, RegMemImm::reg(rcx), r8), "4138C8", "cmpb %cl, %r8b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rcx), rsi), + Inst::cmp_rmi_r(1, RegMemImm::reg(rcx), rsi), "4038CE", "cmpb %cl, %sil", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::mem(Addr::imm_reg(99, rdi)), rdx), + Inst::cmp_rmi_r(1, RegMemImm::mem(Amode::imm_reg(99, rdi)), rdx), "3A5763", "cmpb 99(%rdi), %dl", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::mem(Addr::imm_reg(99, rdi)), r8), + Inst::cmp_rmi_r(1, RegMemImm::mem(Amode::imm_reg(99, rdi)), r8), "443A4763", "cmpb 99(%rdi), %r8b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::mem(Addr::imm_reg(99, rdi)), rsi), + Inst::cmp_rmi_r(1, RegMemImm::mem(Amode::imm_reg(99, rdi)), rsi), "403A7763", "cmpb 99(%rdi), %sil", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::imm(70), rdx), + Inst::cmp_rmi_r(1, RegMemImm::imm(70), rdx), "80FA46", "cmpb $70, %dl", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::imm(-76i32 as u32), r8), + Inst::cmp_rmi_r(1, RegMemImm::imm(-76i32 as u32), r8), "4180F8B4", "cmpb $-76, %r8b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::imm(76), rsi), + Inst::cmp_rmi_r(1, RegMemImm::imm(76), rsi), "4080FE4C", "cmpb $76, %sil", )); // Extra byte-cases (paranoia!) for cmp_rmi_r for first operand = R insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rax), rbx), + Inst::cmp_rmi_r(1, RegMemImm::reg(rax), rbx), "38C3", "cmpb %al, %bl", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rbx), rax), + Inst::cmp_rmi_r(1, RegMemImm::reg(rbx), rax), "38D8", "cmpb %bl, %al", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rcx), rdx), + Inst::cmp_rmi_r(1, RegMemImm::reg(rcx), rdx), "38CA", "cmpb %cl, %dl", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rcx), rsi), + Inst::cmp_rmi_r(1, RegMemImm::reg(rcx), rsi), "4038CE", "cmpb %cl, %sil", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rcx), r10), + Inst::cmp_rmi_r(1, RegMemImm::reg(rcx), r10), "4138CA", "cmpb %cl, %r10b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rcx), r14), + Inst::cmp_rmi_r(1, RegMemImm::reg(rcx), r14), "4138CE", "cmpb %cl, %r14b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rbp), rdx), + Inst::cmp_rmi_r(1, RegMemImm::reg(rbp), rdx), "4038EA", "cmpb %bpl, %dl", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rbp), rsi), + Inst::cmp_rmi_r(1, RegMemImm::reg(rbp), rsi), "4038EE", "cmpb %bpl, %sil", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rbp), r10), + Inst::cmp_rmi_r(1, RegMemImm::reg(rbp), r10), "4138EA", "cmpb %bpl, %r10b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(rbp), r14), + Inst::cmp_rmi_r(1, RegMemImm::reg(rbp), r14), "4138EE", "cmpb %bpl, %r14b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r9), rdx), + Inst::cmp_rmi_r(1, RegMemImm::reg(r9), rdx), "4438CA", "cmpb %r9b, %dl", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r9), rsi), + Inst::cmp_rmi_r(1, RegMemImm::reg(r9), rsi), "4438CE", "cmpb %r9b, %sil", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r9), r10), + Inst::cmp_rmi_r(1, RegMemImm::reg(r9), r10), "4538CA", "cmpb %r9b, %r10b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r9), r14), + Inst::cmp_rmi_r(1, RegMemImm::reg(r9), r14), "4538CE", "cmpb %r9b, %r14b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r13), rdx), + Inst::cmp_rmi_r(1, RegMemImm::reg(r13), rdx), "4438EA", "cmpb %r13b, %dl", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r13), rsi), + Inst::cmp_rmi_r(1, RegMemImm::reg(r13), rsi), "4438EE", "cmpb %r13b, %sil", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r13), r10), + Inst::cmp_rmi_r(1, RegMemImm::reg(r13), r10), "4538EA", "cmpb %r13b, %r10b", )); insns.push(( - Inst::cmp_rmi_r(1, RMI::reg(r13), r14), + Inst::cmp_rmi_r(1, RegMemImm::reg(r13), r14), "4538EE", "cmpb %r13b, %r14b", )); + // ======================================================== + // SetCC + insns.push((Inst::setcc(CC::O, w_rsi), "400F90C6", "seto %sil")); + insns.push((Inst::setcc(CC::NLE, w_rsi), "400F9FC6", "setnle %sil")); + insns.push((Inst::setcc(CC::Z, w_r14), "410F94C6", "setz %r14b")); + insns.push((Inst::setcc(CC::LE, w_r14), "410F9EC6", "setle %r14b")); + // ======================================================== // Push64 - insns.push((Inst::push64(RMI::reg(rdi)), "57", "pushq %rdi")); - insns.push((Inst::push64(RMI::reg(r8)), "4150", "pushq %r8")); + insns.push((Inst::push64(RegMemImm::reg(rdi)), "57", "pushq %rdi")); + insns.push((Inst::push64(RegMemImm::reg(r8)), "4150", "pushq %r8")); insns.push(( - Inst::push64(RMI::mem(Addr::imm_reg_reg_shift(321, rsi, rcx, 3))), + Inst::push64(RegMemImm::mem(Amode::imm_reg_reg_shift(321, rsi, rcx, 3))), "FFB4CE41010000", "pushq 321(%rsi,%rcx,8)", )); insns.push(( - Inst::push64(RMI::mem(Addr::imm_reg_reg_shift(321, r9, rbx, 2))), + Inst::push64(RegMemImm::mem(Amode::imm_reg_reg_shift(321, r9, rbx, 2))), "41FFB49941010000", "pushq 321(%r9,%rbx,4)", )); - insns.push((Inst::push64(RMI::imm(0)), "6A00", "pushq $0")); - insns.push((Inst::push64(RMI::imm(127)), "6A7F", "pushq $127")); - insns.push((Inst::push64(RMI::imm(128)), "6880000000", "pushq $128")); + insns.push((Inst::push64(RegMemImm::imm(0)), "6A00", "pushq $0")); + insns.push((Inst::push64(RegMemImm::imm(127)), "6A7F", "pushq $127")); insns.push(( - Inst::push64(RMI::imm(0x31415927)), + Inst::push64(RegMemImm::imm(128)), + "6880000000", + "pushq $128", + )); + insns.push(( + Inst::push64(RegMemImm::imm(0x31415927)), "6827594131", "pushq $826366247", )); insns.push(( - Inst::push64(RMI::imm(-128i32 as u32)), + Inst::push64(RegMemImm::imm(-128i32 as u32)), "6A80", "pushq $-128", )); insns.push(( - Inst::push64(RMI::imm(-129i32 as u32)), + Inst::push64(RegMemImm::imm(-129i32 as u32)), "687FFFFFFF", "pushq $-129", )); insns.push(( - Inst::push64(RMI::imm(-0x75c4e8a1i32 as u32)), + Inst::push64(RegMemImm::imm(-0x75c4e8a1i32 as u32)), "685F173B8A", "pushq $-1975838881", )); @@ -2157,19 +2531,43 @@ fn test_x64_emit() { insns.push((Inst::pop64(w_r15), "415F", "popq %r15")); // ======================================================== - // CallKnown skipped for now + // CallKnown + insns.push(( + Inst::call_known( + ExternalName::User { + namespace: 0, + index: 0, + }, + Vec::new(), + Vec::new(), + SourceLoc::default(), + Opcode::Call, + ), + "E800000000", + "call User { namespace: 0, index: 0 }", + )); // ======================================================== // CallUnknown - insns.push((Inst::call_unknown(RM::reg(rbp)), "FFD5", "call *%rbp")); - insns.push((Inst::call_unknown(RM::reg(r11)), "41FFD3", "call *%r11")); + fn call_unknown(rm: RegMem) -> Inst { + Inst::call_unknown( + rm, + Vec::new(), + Vec::new(), + SourceLoc::default(), + Opcode::CallIndirect, + ) + } + + insns.push((call_unknown(RegMem::reg(rbp)), "FFD5", "call *%rbp")); + insns.push((call_unknown(RegMem::reg(r11)), "41FFD3", "call *%r11")); insns.push(( - Inst::call_unknown(RM::mem(Addr::imm_reg_reg_shift(321, rsi, rcx, 3))), + call_unknown(RegMem::mem(Amode::imm_reg_reg_shift(321, rsi, rcx, 3))), "FF94CE41010000", "call *321(%rsi,%rcx,8)", )); insns.push(( - Inst::call_unknown(RM::mem(Addr::imm_reg_reg_shift(321, r10, rdx, 2))), + call_unknown(RegMem::mem(Amode::imm_reg_reg_shift(321, r10, rdx, 2))), "41FF949241010000", "call *321(%r10,%rdx,4)", )); @@ -2192,15 +2590,19 @@ fn test_x64_emit() { // ======================================================== // JmpUnknown - insns.push((Inst::jmp_unknown(RM::reg(rbp)), "FFE5", "jmp *%rbp")); - insns.push((Inst::jmp_unknown(RM::reg(r11)), "41FFE3", "jmp *%r11")); + insns.push((Inst::jmp_unknown(RegMem::reg(rbp)), "FFE5", "jmp *%rbp")); insns.push(( - Inst::jmp_unknown(RM::mem(Addr::imm_reg_reg_shift(321, rsi, rcx, 3))), + Inst::jmp_unknown(RegMem::reg(r11)), + "41FFE3", + "jmp *%r11", + )); + insns.push(( + Inst::jmp_unknown(RegMem::mem(Amode::imm_reg_reg_shift(321, rsi, rcx, 3))), "FFA4CE41010000", "jmp *321(%rsi,%rcx,8)", )); insns.push(( - Inst::jmp_unknown(RM::mem(Addr::imm_reg_reg_shift(321, r10, rdx, 2))), + Inst::jmp_unknown(RegMem::mem(Amode::imm_reg_reg_shift(321, r10, rdx, 2))), "41FFA49241010000", "jmp *321(%r10,%rdx,4)", )); @@ -2209,62 +2611,125 @@ fn test_x64_emit() { // XMM_RM_R insns.push(( - Inst::xmm_rm_r(SSE_Op::SSE_Addss, RM::reg(xmm1), w_xmm0), + Inst::xmm_rm_r(SseOpcode::Addss, RegMem::reg(xmm1), w_xmm0), "F30F58C1", "addss %xmm1, %xmm0", )); insns.push(( - Inst::xmm_rm_r(SSE_Op::SSE_Subss, RM::reg(xmm0), w_xmm1), + Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm0), w_xmm1), "F30F5CC8", "subss %xmm0, %xmm1", )); - insns.push(( - Inst::xmm_rm_r(SSE_Op::SSE_Addss, RM::reg(xmm11), w_xmm13), + Inst::xmm_rm_r(SseOpcode::Addss, RegMem::reg(xmm11), w_xmm13), "F3450F58EB", "addss %xmm11, %xmm13", )); - insns.push(( - Inst::xmm_rm_r(SSE_Op::SSE_Subss, RM::reg(xmm12), w_xmm1), + Inst::xmm_rm_r(SseOpcode::Subss, RegMem::reg(xmm12), w_xmm1), "F3410F5CCC", "subss %xmm12, %xmm1", )); - insns.push(( Inst::xmm_rm_r( - SSE_Op::SSE_Addss, - RM::mem(Addr::imm_reg_reg_shift(123, r10, rdx, 2)), + SseOpcode::Addss, + RegMem::mem(Amode::imm_reg_reg_shift(123, r10, rdx, 2)), w_xmm0, ), "F3410F5844927B", "addss 123(%r10,%rdx,4), %xmm0", )); - insns.push(( Inst::xmm_rm_r( - SSE_Op::SSE_Subss, - RM::mem(Addr::imm_reg_reg_shift(321, r10, rax, 3)), + SseOpcode::Subss, + RegMem::mem(Amode::imm_reg_reg_shift(321, r10, rax, 3)), w_xmm10, ), "F3450F5C94C241010000", "subss 321(%r10,%rax,8), %xmm10", )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Mulss, RegMem::reg(xmm5), w_xmm4), + "F30F59E5", + "mulss %xmm5, %xmm4", + )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Divss, RegMem::reg(xmm8), w_xmm7), + "F3410F5EF8", + "divss %xmm8, %xmm7", + )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Sqrtss, RegMem::reg(xmm7), w_xmm8), + "F3440F51C7", + "sqrtss %xmm7, %xmm8", + )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Andps, RegMem::reg(xmm3), w_xmm12), + "440F54E3", + "andps %xmm3, %xmm12", + )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Andnps, RegMem::reg(xmm4), w_xmm11), + "440F55DC", + "andnps %xmm4, %xmm11", + )); + insns.push(( + Inst::xmm_mov_rm_r(SseOpcode::Movaps, RegMem::reg(xmm5), w_xmm14), + "440F28F5", + "movaps %xmm5, %xmm14", + )); + insns.push(( + Inst::xmm_mov_rm_r(SseOpcode::Movd, RegMem::reg(rax), w_xmm15), + "66440F6EF8", + "movd %eax, %xmm15", + )); + insns.push(( + Inst::xmm_rm_r(SseOpcode::Orps, RegMem::reg(xmm1), w_xmm15), + "440F56F9", + "orps %xmm1, %xmm15", + )); + + insns.push(( + Inst::xmm_mov_r_m(SseOpcode::Movd, xmm0, Amode::imm_reg(321, rbx)), + "660F7E8341010000", + "movd %xmm0, 321(%rbx)", + )); + + insns.push(( + Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12)), + "F3450F11BC2480000000", + "movss %xmm15, 128(%r12)", + )); + + insns.push(( + Inst::xmm_mov_rm_r(SseOpcode::Movd, RegMem::mem(Amode::imm_reg(2, r10)), w_xmm9), + "66450F6E4A02", + "movd 2(%r10), %xmm9", + )); + + insns.push(( + Inst::xmm_rm_r(SseOpcode::Orps, RegMem::reg(xmm5), w_xmm4), + "0F56E5", + "orps %xmm5, %xmm4", + )); + insns.push(( + Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(xmm13), w_xmm2), + "F3410F10D5", + "movss %xmm13, %xmm2", + )); + insns.push(( + Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(xmm14), w_xmm3), + "F2410F10DE", + "movsd %xmm14, %xmm3", + )); // ======================================================== - // XMM_R_R + // Misc instructions. - insns.push(( - Inst::xmm_r_r(SSE_Op::SSE_Movss, xmm3, w_xmm2), - "F30F10D3", - "movss %xmm3, %xmm2", - )); + insns.push((Inst::Hlt, "CC", "hlt")); - insns.push(( - Inst::xmm_r_r(SSE_Op::SSE2_Movsd, xmm4, w_xmm3), - "F20F10DC", - "movsd %xmm4, %xmm3", - )); + let trap_info = (SourceLoc::default(), TrapCode::UnreachableCodeReached); + insns.push((Inst::Ud2 { trap_info }, "0F0B", "ud2 unreachable")); // ======================================================== // Actually run the tests! @@ -2280,6 +2745,6 @@ fn test_x64_emit() { let buffer = buffer.finish(); buffer.emit(&mut sink); let actual_encoding = &sink.stringify(); - assert_eq!(expected_encoding, actual_encoding); + assert_eq!(expected_encoding, actual_encoding, "{}", expected_printing); } } diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/inst/mod.rs b/third_party/rust/cranelift-codegen/src/isa/x64/inst/mod.rs index 132d93e16609..bf778a1e4013 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/inst/mod.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/inst/mod.rs @@ -4,19 +4,17 @@ #![allow(non_snake_case)] #![allow(non_camel_case_types)] -use core::convert::TryFrom; +use alloc::vec::Vec; use smallvec::SmallVec; use std::fmt; use std::string::{String, ToString}; use regalloc::RegUsageCollector; -use regalloc::Set; use regalloc::{RealRegUniverse, Reg, RegClass, RegUsageMapper, SpillSlot, VirtualReg, Writable}; use crate::binemit::CodeOffset; use crate::ir::types::{B1, B128, B16, B32, B64, B8, F32, F64, I128, I16, I32, I64, I8}; -use crate::ir::ExternalName; -use crate::ir::Type; +use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type}; use crate::machinst::*; use crate::settings::Flags; use crate::{settings, CodegenError, CodegenResult}; @@ -37,61 +35,71 @@ use regs::{create_reg_universe_systemv, show_ireg_sized}; /// Instructions. Destinations are on the RIGHT (a la AT&T syntax). #[derive(Clone)] -pub(crate) enum Inst { +pub enum Inst { /// nops of various sizes, including zero Nop { len: u8 }, - /// (add sub and or xor mul adc? sbb?) (32 64) (reg addr imm) reg + // ===================================== + // Integer instructions. + /// Integer arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr imm) reg Alu_RMI_R { is_64: bool, - op: RMI_R_Op, - src: RMI, + op: AluRmiROpcode, + src: RegMemImm, dst: Writable, }, - /// (imm32 imm64) reg. - /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32 + /// Constant materialization: (imm32 imm64) reg. + /// Either: movl $imm32, %reg32 or movabsq $imm64, %reg32. Imm_R { dst_is_64: bool, simm64: u64, dst: Writable, }, - /// mov (64 32) reg reg + /// GPR to GPR move: mov (64 32) reg reg. Mov_R_R { is_64: bool, src: Reg, dst: Writable, }, - /// movz (bl bq wl wq lq) addr reg (good for all ZX loads except 64->64). - /// Note that the lq variant doesn't really exist since the default - /// zero-extend rule makes it unnecessary. For that case we emit the - /// equivalent "movl AM, reg32". - MovZX_M_R { - extMode: ExtMode, - addr: Addr, + /// Zero-extended loads, except for 64 bits: movz (bl bq wl wq lq) addr reg. + /// Note that the lq variant doesn't really exist since the default zero-extend rule makes it + /// unnecessary. For that case we emit the equivalent "movl AM, reg32". + MovZX_RM_R { + ext_mode: ExtMode, + src: RegMem, dst: Writable, }, - /// A plain 64-bit integer load, since MovZX_M_R can't represent that - Mov64_M_R { addr: Addr, dst: Writable }, - - /// movs (bl bq wl wq lq) addr reg (good for all SX loads) - MovSX_M_R { - extMode: ExtMode, - addr: Addr, + /// A plain 64-bit integer load, since MovZX_RM_R can't represent that. + Mov64_M_R { + src: SyntheticAmode, dst: Writable, }, - /// mov (b w l q) reg addr (good for all integer stores) + /// Loads the memory address of addr into dst. + LoadEffectiveAddress { + addr: SyntheticAmode, + dst: Writable, + }, + + /// Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. + MovSX_RM_R { + ext_mode: ExtMode, + src: RegMem, + dst: Writable, + }, + + /// Integer stores: mov (b w l q) reg addr. Mov_R_M { - size: u8, // 1, 2, 4 or 8 + size: u8, // 1, 2, 4 or 8. src: Reg, - addr: Addr, + dst: SyntheticAmode, }, - /// (shl shr sar) (l q) imm reg + /// Arithmetic shifts: (shl shr sar) (l q) imm reg. Shift_R { is_64: bool, kind: ShiftKind, @@ -100,70 +108,102 @@ pub(crate) enum Inst { dst: Writable, }, - /// cmp (b w l q) (reg addr imm) reg + /// Integer comparisons/tests: cmp (b w l q) (reg addr imm) reg. Cmp_RMI_R { size: u8, // 1, 2, 4 or 8 - src: RMI, + src: RegMemImm, dst: Reg, }, + /// Materializes the requested condition code in the destination reg. + Setcc { cc: CC, dst: Writable }, + + // ===================================== + // Stack manipulation. /// pushq (reg addr imm) - Push64 { src: RMI }, + Push64 { src: RegMemImm }, /// popq reg Pop64 { dst: Writable }, - /// call simm32 + // ===================================== + // Floating-point operations. + /// Float arithmetic/bit-twiddling: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg + XMM_RM_R { + op: SseOpcode, + src: RegMem, + dst: Writable, + }, + + /// mov between XMM registers (32 64) (reg addr) reg XMM_Mov_RM_R differs from XMM_RM_R in + /// that the dst register of XMM_MOV_RM_R is not used in the computation of the instruction + /// dst value and so does not have to be a previously valid value. This is characteristic of + /// mov instructions. + XMM_Mov_RM_R { + op: SseOpcode, + src: RegMem, + dst: Writable, + }, + + /// mov reg addr (good for all memory stores from xmm registers) + XMM_Mov_R_M { + op: SseOpcode, + src: Reg, + dst: SyntheticAmode, + }, + + // ===================================== + // Control flow instructions. + /// Direct call: call simm32. CallKnown { dest: ExternalName, - uses: Set, - defs: Set>, + uses: Vec, + defs: Vec>, + loc: SourceLoc, + opcode: Opcode, }, - /// callq (reg mem) + /// Indirect call: callq (reg mem). CallUnknown { - dest: RM, - //uses: Set, - //defs: Set>, + dest: RegMem, + uses: Vec, + defs: Vec>, + loc: SourceLoc, + opcode: Opcode, }, - // ---- branches (exactly one must appear at end of BB) ---- - /// ret + /// Return. Ret, /// A placeholder instruction, generating no code, meaning that a function epilogue must be /// inserted there. EpiloguePlaceholder, - /// jmp simm32 - JmpKnown { dest: BranchTarget }, + /// Jump to a known target: jmp simm32. + JmpKnown { dst: BranchTarget }, - /// jcond cond target target - /// Symmetrical two-way conditional branch. - /// Emitted as a compound sequence; the MachBuffer will shrink it - /// as appropriate. - JmpCondSymm { + /// Two-way conditional branch: jcond cond target target. + /// Emitted as a compound sequence; the MachBuffer will shrink it as appropriate. + JmpCond { cc: CC, taken: BranchTarget, not_taken: BranchTarget, }, - /// jmpq (reg mem) - JmpUnknown { target: RM }, + /// Indirect jump: jmpq (reg mem). + JmpUnknown { target: RegMem }, - /// (add sub and or xor mul adc? sbb?) (32 64) (reg addr imm) reg - XMM_RM_R { - op: SSE_Op, - src: RM, - dst: Writable, - }, + /// A debug trap. + Hlt, - /// mov (64 32) reg reg - XMM_R_R { - op: SSE_Op, - src: Reg, - dst: Writable, - }, + /// An instruction that will always trigger the illegal instruction exception. + Ud2 { trap_info: (SourceLoc, TrapCode) }, + + // ===================================== + // Meta-instructions generating no code. + /// Marker, no-op in generated code: SP "virtual offset" is adjusted. This + /// controls how MemArg::NominalSPOffset args are lowered. + VirtualSPOffsetAdj { offset: i64 }, } // Handy constructors for Insts. @@ -181,7 +221,12 @@ impl Inst { Self::Nop { len } } - pub(crate) fn alu_rmi_r(is_64: bool, op: RMI_R_Op, src: RMI, dst: Writable) -> Self { + pub(crate) fn alu_rmi_r( + is_64: bool, + op: AluRmiROpcode, + src: RegMemImm, + dst: Writable, + ) -> Self { debug_assert!(dst.to_reg().get_class() == RegClass::I64); Self::Alu_RMI_R { is_64, @@ -209,40 +254,63 @@ impl Inst { Inst::Mov_R_R { is_64, src, dst } } - pub(crate) fn xmm_r_r(op: SSE_Op, src: Reg, dst: Writable) -> Inst { + pub(crate) fn xmm_mov_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { + debug_assert!(dst.to_reg().get_class() == RegClass::V128); + Inst::XMM_Mov_RM_R { op, src, dst } + } + + pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { + debug_assert!(dst.to_reg().get_class() == RegClass::V128); + Inst::XMM_RM_R { op, src, dst } + } + + pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into) -> Inst { debug_assert!(src.get_class() == RegClass::V128); - debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XMM_R_R { op, src, dst } + Inst::XMM_Mov_R_M { + op, + src, + dst: dst.into(), + } } - pub(crate) fn xmm_rm_r(op: SSE_Op, src: RM, dst: Writable) -> Self { - debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Self::XMM_RM_R { op, src, dst } - } - - pub(crate) fn movzx_m_r(extMode: ExtMode, addr: Addr, dst: Writable) -> Inst { + pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); - Inst::MovZX_M_R { extMode, addr, dst } + Inst::MovZX_RM_R { ext_mode, src, dst } } - pub(crate) fn mov64_m_r(addr: Addr, dst: Writable) -> Inst { + pub(crate) fn mov64_m_r(src: impl Into, dst: Writable) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); - Inst::Mov64_M_R { addr, dst } + Inst::Mov64_M_R { + src: src.into(), + dst, + } } - pub(crate) fn movsx_m_r(extMode: ExtMode, addr: Addr, dst: Writable) -> Inst { + pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); - Inst::MovSX_M_R { extMode, addr, dst } + Inst::MovSX_RM_R { ext_mode, src, dst } } pub(crate) fn mov_r_m( size: u8, // 1, 2, 4 or 8 src: Reg, - addr: Addr, + dst: impl Into, ) -> Inst { debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); debug_assert!(src.get_class() == RegClass::I64); - Inst::Mov_R_M { size, src, addr } + Inst::Mov_R_M { + size, + src, + dst: dst.into(), + } + } + + pub(crate) fn lea(addr: impl Into, dst: Writable) -> Inst { + debug_assert!(dst.to_reg().get_class() == RegClass::I64); + Inst::LoadEffectiveAddress { + addr: addr.into(), + dst, + } } pub(crate) fn shift_r( @@ -265,9 +333,11 @@ impl Inst { } } + /// Does a comparison of dst - src for operands of size `size`, as stated by the machine + /// instruction semantics. Be careful with the order of parameters! pub(crate) fn cmp_rmi_r( size: u8, // 1, 2, 4 or 8 - src: RMI, + src: RegMemImm, dst: Reg, ) -> Inst { debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); @@ -275,7 +345,12 @@ impl Inst { Inst::Cmp_RMI_R { size, src, dst } } - pub(crate) fn push64(src: RMI) -> Inst { + pub(crate) fn setcc(cc: CC, dst: Writable) -> Inst { + debug_assert!(dst.to_reg().get_class() == RegClass::I64); + Inst::Setcc { cc, dst } + } + + pub(crate) fn push64(src: RegMemImm) -> Inst { Inst::Push64 { src } } @@ -283,8 +358,36 @@ impl Inst { Inst::Pop64 { dst } } - pub(crate) fn call_unknown(dest: RM) -> Inst { - Inst::CallUnknown { dest } + pub(crate) fn call_known( + dest: ExternalName, + uses: Vec, + defs: Vec>, + loc: SourceLoc, + opcode: Opcode, + ) -> Inst { + Inst::CallKnown { + dest, + uses, + defs, + loc, + opcode, + } + } + + pub(crate) fn call_unknown( + dest: RegMem, + uses: Vec, + defs: Vec>, + loc: SourceLoc, + opcode: Opcode, + ) -> Inst { + Inst::CallUnknown { + dest, + uses, + defs, + loc, + opcode, + } } pub(crate) fn ret() -> Inst { @@ -295,19 +398,19 @@ impl Inst { Inst::EpiloguePlaceholder } - pub(crate) fn jmp_known(dest: BranchTarget) -> Inst { - Inst::JmpKnown { dest } + pub(crate) fn jmp_known(dst: BranchTarget) -> Inst { + Inst::JmpKnown { dst } } - pub(crate) fn jmp_cond_symm(cc: CC, taken: BranchTarget, not_taken: BranchTarget) -> Inst { - Inst::JmpCondSymm { + pub(crate) fn jmp_cond(cc: CC, taken: BranchTarget, not_taken: BranchTarget) -> Inst { + Inst::JmpCond { cc, taken, not_taken, } } - pub(crate) fn jmp_unknown(target: RM) -> Inst { + pub(crate) fn jmp_unknown(target: RegMem) -> Inst { Inst::JmpUnknown { target } } } @@ -366,6 +469,18 @@ impl ShowWithRRU for Inst { src.show_rru_sized(mb_rru, sizeLQ(*is_64)), show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)), ), + Inst::XMM_Mov_RM_R { op, src, dst } => format!( + "{} {}, {}", + ljustify(op.to_string()), + src.show_rru_sized(mb_rru, op.src_size()), + show_ireg_sized(dst.to_reg(), mb_rru, 8), + ), + Inst::XMM_Mov_R_M { op, src, dst } => format!( + "{} {}, {}", + ljustify(op.to_string()), + show_ireg_sized(*src, mb_rru, 8), + dst.show_rru(mb_rru) + ), Inst::XMM_RM_R { op, src, dst } => format!( "{} {}, {}", ljustify(op.to_string()), @@ -399,46 +514,46 @@ impl ShowWithRRU for Inst { show_ireg_sized(*src, mb_rru, sizeLQ(*is_64)), show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)) ), - Inst::XMM_R_R { op, src, dst } => format!( - "{} {}, {}", - ljustify(op.to_string()), - show_ireg_sized(*src, mb_rru, 8), - show_ireg_sized(dst.to_reg(), mb_rru, 8) - ), - Inst::MovZX_M_R { extMode, addr, dst } => { - if *extMode == ExtMode::LQ { + Inst::MovZX_RM_R { ext_mode, src, dst } => { + if *ext_mode == ExtMode::LQ { format!( "{} {}, {}", ljustify("movl".to_string()), - addr.show_rru(mb_rru), + src.show_rru_sized(mb_rru, ext_mode.src_size()), show_ireg_sized(dst.to_reg(), mb_rru, 4) ) } else { format!( "{} {}, {}", - ljustify2("movz".to_string(), extMode.to_string()), - addr.show_rru(mb_rru), - show_ireg_sized(dst.to_reg(), mb_rru, extMode.dst_size()) + ljustify2("movz".to_string(), ext_mode.to_string()), + src.show_rru_sized(mb_rru, ext_mode.src_size()), + show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size()) ) } } - Inst::Mov64_M_R { addr, dst } => format!( + Inst::Mov64_M_R { src, dst } => format!( "{} {}, {}", ljustify("movq".to_string()), + src.show_rru(mb_rru), + dst.show_rru(mb_rru) + ), + Inst::LoadEffectiveAddress { addr, dst } => format!( + "{} {}, {}", + ljustify("lea".to_string()), addr.show_rru(mb_rru), dst.show_rru(mb_rru) ), - Inst::MovSX_M_R { extMode, addr, dst } => format!( + Inst::MovSX_RM_R { ext_mode, src, dst } => format!( "{} {}, {}", - ljustify2("movs".to_string(), extMode.to_string()), - addr.show_rru(mb_rru), - show_ireg_sized(dst.to_reg(), mb_rru, extMode.dst_size()) + ljustify2("movs".to_string(), ext_mode.to_string()), + src.show_rru_sized(mb_rru, ext_mode.src_size()), + show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size()) ), - Inst::Mov_R_M { size, src, addr } => format!( + Inst::Mov_R_M { size, src, dst } => format!( "{} {}, {}", ljustify2("mov".to_string(), suffixBWLQ(*size)), show_ireg_sized(*src, mb_rru, *size), - addr.show_rru(mb_rru) + dst.show_rru(mb_rru) ), Inst::Shift_R { is_64, @@ -465,25 +580,29 @@ impl ShowWithRRU for Inst { src.show_rru_sized(mb_rru, *size), show_ireg_sized(*dst, mb_rru, *size) ), + Inst::Setcc { cc, dst } => format!( + "{} {}", + ljustify2("set".to_string(), cc.to_string()), + show_ireg_sized(dst.to_reg(), mb_rru, 1) + ), Inst::Push64 { src } => { format!("{} {}", ljustify("pushq".to_string()), src.show_rru(mb_rru)) } Inst::Pop64 { dst } => { format!("{} {}", ljustify("popq".to_string()), dst.show_rru(mb_rru)) } - //Inst::CallKnown { target } => format!("{} {:?}", ljustify("call".to_string()), target), - Inst::CallKnown { .. } => "**CallKnown**".to_string(), - Inst::CallUnknown { dest } => format!( + Inst::CallKnown { dest, .. } => format!("{} {:?}", ljustify("call".to_string()), dest), + Inst::CallUnknown { dest, .. } => format!( "{} *{}", ljustify("call".to_string()), dest.show_rru(mb_rru) ), Inst::Ret => "ret".to_string(), Inst::EpiloguePlaceholder => "epilogue placeholder".to_string(), - Inst::JmpKnown { dest } => { - format!("{} {}", ljustify("jmp".to_string()), dest.show_rru(mb_rru)) + Inst::JmpKnown { dst } => { + format!("{} {}", ljustify("jmp".to_string()), dst.show_rru(mb_rru)) } - Inst::JmpCondSymm { + Inst::JmpCond { cc, taken, not_taken, @@ -499,6 +618,9 @@ impl ShowWithRRU for Inst { ljustify("jmp".to_string()), target.show_rru(mb_rru) ), + Inst::VirtualSPOffsetAdj { offset } => format!("virtual_sp_offset_adjust {}", offset), + Inst::Hlt => "hlt".into(), + Inst::Ud2 { trap_info } => format!("ud2 {}", trap_info.1), } } } @@ -517,7 +639,6 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { // regalloc.rs will "fix" this for us by removing the the modified set from the use and def // sets. match inst { - // ** Nop Inst::Alu_RMI_R { is_64: _, op: _, @@ -527,48 +648,40 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { src.get_regs_as_uses(collector); collector.add_mod(*dst); } - Inst::XMM_RM_R { op: _, src, dst } => { + Inst::XMM_Mov_RM_R { src, dst, .. } => { + src.get_regs_as_uses(collector); + collector.add_def(*dst); + } + Inst::XMM_RM_R { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_mod(*dst); } - Inst::Imm_R { - dst_is_64: _, - simm64: _, - dst, - } => { + Inst::XMM_Mov_R_M { src, dst, .. } => { + collector.add_use(*src); + dst.get_regs_as_uses(collector); + } + Inst::Imm_R { dst, .. } => { collector.add_def(*dst); } - Inst::Mov_R_R { is_64: _, src, dst } => { + Inst::Mov_R_R { src, dst, .. } => { collector.add_use(*src); collector.add_def(*dst); } - Inst::XMM_R_R { op: _, src, dst } => { + Inst::MovZX_RM_R { src, dst, .. } => { + src.get_regs_as_uses(collector); + collector.add_def(*dst); + } + Inst::Mov64_M_R { src, dst } | Inst::LoadEffectiveAddress { addr: src, dst } => { + src.get_regs_as_uses(collector); + collector.add_def(*dst) + } + Inst::MovSX_RM_R { src, dst, .. } => { + src.get_regs_as_uses(collector); + collector.add_def(*dst); + } + Inst::Mov_R_M { src, dst, .. } => { collector.add_use(*src); - collector.add_def(*dst); - } - Inst::MovZX_M_R { - extMode: _, - addr, - dst, - } => { - addr.get_regs_as_uses(collector); - collector.add_def(*dst); - } - Inst::Mov64_M_R { addr, dst } => { - addr.get_regs_as_uses(collector); - collector.add_def(*dst); - } - Inst::MovSX_M_R { - extMode: _, - addr, - dst, - } => { - addr.get_regs_as_uses(collector); - collector.add_def(*dst); - } - Inst::Mov_R_M { size: _, src, addr } => { - collector.add_use(*src); - addr.get_regs_as_uses(collector); + dst.get_regs_as_uses(collector); } Inst::Shift_R { is_64: _, @@ -585,6 +698,9 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { src.get_regs_as_uses(collector); collector.add_use(*dst); // yes, really `add_use` } + Inst::Setcc { dst, .. } => { + collector.add_def(*dst); + } Inst::Push64 { src } => { src.get_regs_as_uses(collector); collector.add_mod(Writable::from_reg(regs::rsp())); @@ -592,29 +708,36 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { Inst::Pop64 { dst } => { collector.add_def(*dst); } + Inst::CallKnown { - dest: _, - uses: _, - defs: _, + ref uses, ref defs, .. } => { - // FIXME add arg regs (iru.used) and caller-saved regs (iru.defined) - unimplemented!(); + collector.add_uses(uses); + collector.add_defs(defs); } - Inst::CallUnknown { dest } => { + + Inst::CallUnknown { + ref uses, + ref defs, + dest, + .. + } => { + collector.add_uses(uses); + collector.add_defs(defs); dest.get_regs_as_uses(collector); } - Inst::Ret => {} - Inst::EpiloguePlaceholder => {} - Inst::JmpKnown { dest: _ } => {} - Inst::JmpCondSymm { - cc: _, - taken: _, - not_taken: _, - } => {} - //Inst::JmpUnknown { target } => { - // target.get_regs_as_uses(collector); - //} - Inst::Nop { .. } | Inst::JmpUnknown { .. } => unimplemented!("x64_get_regs inst"), + + Inst::Ret + | Inst::EpiloguePlaceholder + | Inst::JmpKnown { .. } + | Inst::JmpCond { .. } + | Inst::Nop { .. } + | Inst::JmpUnknown { .. } + | Inst::VirtualSPOffsetAdj { .. } + | Inst::Hlt + | Inst::Ud2 { .. } => { + // No registers are used. + } } } @@ -622,34 +745,34 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { // Instructions and subcomponents: map_regs fn map_use(m: &RUM, r: &mut Reg) { - if r.is_virtual() { - let new = m.get_use(r.to_virtual_reg()).unwrap().to_reg(); + if let Some(reg) = r.as_virtual_reg() { + let new = m.get_use(reg).unwrap().to_reg(); *r = new; } } fn map_def(m: &RUM, r: &mut Writable) { - if r.to_reg().is_virtual() { - let new = m.get_def(r.to_reg().to_virtual_reg()).unwrap().to_reg(); + if let Some(reg) = r.to_reg().as_virtual_reg() { + let new = m.get_def(reg).unwrap().to_reg(); *r = Writable::from_reg(new); } } fn map_mod(m: &RUM, r: &mut Writable) { - if r.to_reg().is_virtual() { - let new = m.get_mod(r.to_reg().to_virtual_reg()).unwrap().to_reg(); + if let Some(reg) = r.to_reg().as_virtual_reg() { + let new = m.get_mod(reg).unwrap().to_reg(); *r = Writable::from_reg(new); } } -impl Addr { +impl Amode { fn map_uses(&mut self, map: &RUM) { match self { - Addr::IR { + Amode::ImmReg { simm32: _, ref mut base, } => map_use(map, base), - Addr::IRRS { + Amode::ImmRegRegShift { simm32: _, ref mut base, ref mut index, @@ -662,21 +785,21 @@ impl Addr { } } -impl RMI { +impl RegMemImm { fn map_uses(&mut self, map: &RUM) { match self { - RMI::R { ref mut reg } => map_use(map, reg), - RMI::M { ref mut addr } => addr.map_uses(map), - RMI::I { simm32: _ } => {} + RegMemImm::Reg { ref mut reg } => map_use(map, reg), + RegMemImm::Mem { ref mut addr } => addr.map_uses(map), + RegMemImm::Imm { simm32: _ } => {} } } } -impl RM { +impl RegMem { fn map_uses(&mut self, map: &RUM) { match self { - RM::R { ref mut reg } => map_use(map, reg), - RM::M { ref mut addr } => addr.map_uses(map), + RegMem::Reg { ref mut reg } => map_use(map, reg), + RegMem::Mem { ref mut addr } => addr.map_uses(map), } } } @@ -694,14 +817,30 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { src.map_uses(mapper); map_mod(mapper, dst); } - Inst::XMM_RM_R { - op: _, + Inst::XMM_Mov_RM_R { ref mut src, ref mut dst, + .. + } => { + src.map_uses(mapper); + map_def(mapper, dst); + } + Inst::XMM_RM_R { + ref mut src, + ref mut dst, + .. } => { src.map_uses(mapper); map_mod(mapper, dst); } + Inst::XMM_Mov_R_M { + ref mut src, + ref mut dst, + .. + } => { + map_use(mapper, src); + dst.map_uses(mapper); + } Inst::Imm_R { dst_is_64: _, simm64: _, @@ -715,41 +854,33 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { map_use(mapper, src); map_def(mapper, dst); } - Inst::XMM_R_R { - op: _, + Inst::MovZX_RM_R { ref mut src, ref mut dst, + .. } => { - map_use(mapper, src); + src.map_uses(mapper); map_def(mapper, dst); } - Inst::MovZX_M_R { - extMode: _, - ref mut addr, + Inst::Mov64_M_R { src, dst } | Inst::LoadEffectiveAddress { addr: src, dst } => { + src.map_uses(mapper); + map_def(mapper, dst); + } + Inst::MovSX_RM_R { + ref mut src, ref mut dst, + .. } => { - addr.map_uses(mapper); - map_def(mapper, dst); - } - Inst::Mov64_M_R { addr, dst } => { - addr.map_uses(mapper); - map_def(mapper, dst); - } - Inst::MovSX_M_R { - extMode: _, - ref mut addr, - ref mut dst, - } => { - addr.map_uses(mapper); + src.map_uses(mapper); map_def(mapper, dst); } Inst::Mov_R_M { - size: _, ref mut src, - ref mut addr, + ref mut dst, + .. } => { map_use(mapper, src); - addr.map_uses(mapper); + dst.map_uses(mapper); } Inst::Shift_R { is_64: _, @@ -767,28 +898,51 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { src.map_uses(mapper); map_use(mapper, dst); } + Inst::Setcc { ref mut dst, .. } => map_def(mapper, dst), Inst::Push64 { ref mut src } => src.map_uses(mapper), Inst::Pop64 { ref mut dst } => { map_def(mapper, dst); } + Inst::CallKnown { - dest: _, - uses: _, - defs: _, - } => {} - Inst::CallUnknown { dest } => dest.map_uses(mapper), - Inst::Ret => {} - Inst::EpiloguePlaceholder => {} - Inst::JmpKnown { dest: _ } => {} - Inst::JmpCondSymm { - cc: _, - taken: _, - not_taken: _, - } => {} - //Inst::JmpUnknown { target } => { - // target.apply_map(mapper); - //} - Inst::Nop { .. } | Inst::JmpUnknown { .. } => unimplemented!("x64_map_regs opcode"), + ref mut uses, + ref mut defs, + .. + } => { + for r in uses.iter_mut() { + map_use(mapper, r); + } + for r in defs.iter_mut() { + map_def(mapper, r); + } + } + + Inst::CallUnknown { + ref mut uses, + ref mut defs, + ref mut dest, + .. + } => { + for r in uses.iter_mut() { + map_use(mapper, r); + } + for r in defs.iter_mut() { + map_def(mapper, r); + } + dest.map_uses(mapper); + } + + Inst::Ret + | Inst::EpiloguePlaceholder + | Inst::JmpKnown { .. } + | Inst::JmpCond { .. } + | Inst::Nop { .. } + | Inst::JmpUnknown { .. } + | Inst::VirtualSPOffsetAdj { .. } + | Inst::Ud2 { .. } + | Inst::Hlt => { + // No registers are used. + } } } @@ -811,6 +965,17 @@ impl MachInst for Inst { // %reg. match self { Self::Mov_R_R { is_64, src, dst } if *is_64 => Some((*dst, *src)), + Self::XMM_Mov_RM_R { op, src, dst } + if *op == SseOpcode::Movss + || *op == SseOpcode::Movsd + || *op == SseOpcode::Movaps => + { + if let RegMem::Reg { reg } = src { + Some((*dst, *reg)) + } else { + None + } + } _ => None, } } @@ -827,8 +992,8 @@ impl MachInst for Inst { match self { // Interesting cases. &Self::Ret | &Self::EpiloguePlaceholder => MachTerminator::Ret, - &Self::JmpKnown { dest } => MachTerminator::Uncond(dest.as_label().unwrap()), - &Self::JmpCondSymm { + &Self::JmpKnown { dst } => MachTerminator::Uncond(dst.as_label().unwrap()), + &Self::JmpCond { cc: _, taken, not_taken, @@ -838,21 +1003,24 @@ impl MachInst for Inst { } } - fn gen_move(dst_reg: Writable, src_reg: Reg, _ty: Type) -> Inst { + fn gen_move(dst_reg: Writable, src_reg: Reg, ty: Type) -> Inst { let rc_dst = dst_reg.to_reg().get_class(); let rc_src = src_reg.get_class(); // If this isn't true, we have gone way off the rails. debug_assert!(rc_dst == rc_src); match rc_dst { RegClass::I64 => Inst::mov_r_r(true, src_reg, dst_reg), - // TODO: How do you just move 32 bits? - RegClass::V128 => Inst::xmm_r_r(SSE_Op::SSE2_Movsd, src_reg, dst_reg), + RegClass::V128 => match ty { + F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(src_reg), dst_reg), + F64 => Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(src_reg), dst_reg), + _ => panic!("unexpected V128 type in gen_move"), + }, _ => panic!("gen_move(x64): unhandled regclass"), } } fn gen_zero_len_nop() -> Inst { - unimplemented!() + Inst::Nop { len: 0 } } fn gen_nop(_preferred_size: usize) -> Inst { @@ -896,20 +1064,27 @@ impl MachInst for Inst { type LabelUse = LabelUse; } -impl MachInstEmit for Inst { - type State = (); +/// State carried between emissions of a sequence of instructions. +#[derive(Default, Clone, Debug)] +pub struct EmitState { + virtual_sp_offset: i64, +} - fn emit(&self, sink: &mut MachBuffer, _flags: &settings::Flags, _: &mut Self::State) { - emit::emit(self, sink); +impl MachInstEmit for Inst { + type State = EmitState; + + fn emit(&self, sink: &mut MachBuffer, flags: &settings::Flags, state: &mut Self::State) { + emit::emit(self, sink, flags, state); } } /// A label-use (internal relocation) in generated code. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum LabelUse { - /// A 32-bit offset from location of relocation itself, added to the - /// existing value at that location. - Rel32, +pub enum LabelUse { + /// A 32-bit offset from location of relocation itself, added to the existing value at that + /// location. Used for control flow instructions which consider an offset from the start of the + /// next instruction (so the size of the payload -- 4 bytes -- is subtracted from the payload). + JmpRel32, } impl MachInstLabelUse for LabelUse { @@ -917,30 +1092,31 @@ impl MachInstLabelUse for LabelUse { fn max_pos_range(self) -> CodeOffset { match self { - LabelUse::Rel32 => 0x7fff_ffff, + LabelUse::JmpRel32 => 0x7fff_ffff, } } fn max_neg_range(self) -> CodeOffset { match self { - LabelUse::Rel32 => 0x8000_0000, + LabelUse::JmpRel32 => 0x8000_0000, } } fn patch_size(self) -> CodeOffset { match self { - LabelUse::Rel32 => 4, + LabelUse::JmpRel32 => 4, } } fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) { + let pc_rel = (label_offset as i64) - (use_offset as i64); + debug_assert!(pc_rel <= self.max_pos_range() as i64); + debug_assert!(pc_rel >= -(self.max_neg_range() as i64)); + let pc_rel = pc_rel as u32; match self { - LabelUse::Rel32 => { - let addend = i32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]); - let value = i32::try_from(label_offset) - .unwrap() - .wrapping_sub(i32::try_from(use_offset).unwrap()) - .wrapping_add(addend); + LabelUse::JmpRel32 => { + let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]); + let value = pc_rel.wrapping_add(addend).wrapping_sub(4); buffer.copy_from_slice(&value.to_le_bytes()[..]); } } @@ -948,20 +1124,20 @@ impl MachInstLabelUse for LabelUse { fn supports_veneer(self) -> bool { match self { - LabelUse::Rel32 => false, + LabelUse::JmpRel32 => false, } } fn veneer_size(self) -> CodeOffset { match self { - LabelUse::Rel32 => 0, + LabelUse::JmpRel32 => 0, } } fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) { match self { - LabelUse::Rel32 => { - panic!("Veneer not supported for Rel32 label-use."); + LabelUse::JmpRel32 => { + panic!("Veneer not supported for JumpRel32 label-use."); } } } diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/lower.rs b/third_party/rust/cranelift-codegen/src/isa/x64/lower.rs index f306d867b020..24e375289505 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/lower.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/lower.rs @@ -1,19 +1,22 @@ //! Lowering rules for X64. -#![allow(dead_code)] #![allow(non_snake_case)] +use log::trace; use regalloc::{Reg, RegClass, Writable}; +use smallvec::SmallVec; +use std::convert::TryFrom; -use crate::ir::condcodes::IntCC; use crate::ir::types; +use crate::ir::types::*; use crate::ir::Inst as IRInst; -use crate::ir::{InstructionData, Opcode, Type}; +use crate::ir::{condcodes::IntCC, InstructionData, Opcode, TrapCode, Type}; use crate::machinst::lower::*; use crate::machinst::*; use crate::result::CodegenResult; +use crate::isa::x64::abi::*; use crate::isa::x64::inst::args::*; use crate::isa::x64::inst::*; use crate::isa::x64::X64Backend; @@ -31,6 +34,20 @@ fn is_int_ty(ty: Type) -> bool { } } +fn is_bool_ty(ty: Type) -> bool { + match ty { + types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true, + _ => false, + } +} + +fn is_float_ty(ty: Type) -> bool { + match ty { + types::F32 | types::F64 => true, + _ => false, + } +} + fn int_ty_is_64(ty: Type) -> bool { match ty { types::I8 | types::I16 | types::I32 => false, @@ -47,29 +64,17 @@ fn flt_ty_is_64(ty: Type) -> bool { } } -fn int_ty_to_sizeB(ty: Type) -> u8 { - match ty { - types::I8 => 1, - types::I16 => 2, - types::I32 => 4, - types::I64 => 8, - _ => panic!("ity_to_sizeB"), - } +fn iri_to_u64_imm(ctx: Ctx, inst: IRInst) -> Option { + ctx.get_constant(inst) } -fn iri_to_u64_immediate<'a>(ctx: Ctx<'a>, iri: IRInst) -> Option { - let inst_data = ctx.data(iri); - if inst_data.opcode() == Opcode::Null { - Some(0) - } else { - match inst_data { - &InstructionData::UnaryImm { opcode: _, imm } => { - // Only has Into for i64; we use u64 elsewhere, so we cast. - let imm: i64 = imm.into(); - Some(imm as u64) - } - _ => None, - } +fn inst_trapcode(data: &InstructionData) -> Option { + match data { + &InstructionData::Trap { code, .. } + | &InstructionData::CondTrap { code, .. } + | &InstructionData::IntCondTrap { code, .. } + | &InstructionData::FloatCondTrap { code, .. } => Some(code), + _ => None, } } @@ -86,131 +91,514 @@ fn inst_condcode(data: &InstructionData) -> IntCC { } } -fn intCC_to_x64_CC(cc: IntCC) -> CC { - match cc { - IntCC::Equal => CC::Z, - IntCC::NotEqual => CC::NZ, - IntCC::SignedGreaterThanOrEqual => CC::NL, - IntCC::SignedGreaterThan => CC::NLE, - IntCC::SignedLessThanOrEqual => CC::LE, - IntCC::SignedLessThan => CC::L, - IntCC::UnsignedGreaterThanOrEqual => CC::NB, - IntCC::UnsignedGreaterThan => CC::NBE, - IntCC::UnsignedLessThanOrEqual => CC::BE, - IntCC::UnsignedLessThan => CC::B, - IntCC::Overflow => CC::O, - IntCC::NotOverflow => CC::NO, +fn ldst_offset(data: &InstructionData) -> Option { + match data { + &InstructionData::Load { offset, .. } + | &InstructionData::StackLoad { offset, .. } + | &InstructionData::LoadComplex { offset, .. } + | &InstructionData::Store { offset, .. } + | &InstructionData::StackStore { offset, .. } + | &InstructionData::StoreComplex { offset, .. } => Some(offset.into()), + _ => None, } } -fn input_to_reg<'a>(ctx: Ctx<'a>, iri: IRInst, input: usize) -> Reg { - let inputs = ctx.get_input(iri, input); +/// Identifier for a particular input of an instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct InsnInput { + insn: IRInst, + input: usize, +} + +/// Identifier for a particular output of an instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct InsnOutput { + insn: IRInst, + output: usize, +} + +fn input_to_reg<'a>(ctx: Ctx<'a>, spec: InsnInput) -> Reg { + let inputs = ctx.get_input(spec.insn, spec.input); ctx.use_input_reg(inputs); inputs.reg } -fn output_to_reg<'a>(ctx: Ctx<'a>, iri: IRInst, output: usize) -> Writable { - ctx.get_output(iri, output) +/// Try to use an immediate for constant inputs, and a register otherwise. +/// TODO: handle memory as well! +fn input_to_reg_mem_imm(ctx: Ctx, spec: InsnInput) -> RegMemImm { + let imm = ctx.get_input(spec.insn, spec.input).constant.and_then(|x| { + let as_u32 = x as u32; + let extended = as_u32 as u64; + // If the truncation and sign-extension don't change the value, use it. + if extended == x { + Some(as_u32) + } else { + None + } + }); + match imm { + Some(x) => RegMemImm::imm(x), + None => RegMemImm::reg(input_to_reg(ctx, spec)), + } +} + +fn output_to_reg<'a>(ctx: Ctx<'a>, spec: InsnOutput) -> Writable { + ctx.get_output(spec.insn, spec.output) } //============================================================================= // Top-level instruction lowering entry point, for one instruction. /// Actually codegen an instruction's results into registers. -fn lower_insn_to_regs<'a>(ctx: Ctx<'a>, iri: IRInst) { - let op = ctx.data(iri).opcode(); - let ty = if ctx.num_outputs(iri) == 1 { - Some(ctx.output_ty(iri, 0)) +fn lower_insn_to_regs>(ctx: &mut C, insn: IRInst) -> CodegenResult<()> { + let op = ctx.data(insn).opcode(); + + let inputs: SmallVec<[InsnInput; 4]> = (0..ctx.num_inputs(insn)) + .map(|i| InsnInput { insn, input: i }) + .collect(); + let outputs: SmallVec<[InsnOutput; 2]> = (0..ctx.num_outputs(insn)) + .map(|i| InsnOutput { insn, output: i }) + .collect(); + + let ty = if outputs.len() > 0 { + Some(ctx.output_ty(insn, 0)) } else { None }; - // This is all outstandingly feeble. TODO: much better! match op { Opcode::Iconst => { - if let Some(w64) = iri_to_u64_immediate(ctx, iri) { + if let Some(w64) = iri_to_u64_imm(ctx, insn) { // Get exactly the bit pattern in 'w64' into the dest. No // monkeying with sign extension etc. - let dstIs64 = w64 > 0xFFFF_FFFF; - let regD = output_to_reg(ctx, iri, 0); - ctx.emit(Inst::imm_r(dstIs64, w64, regD)); + let dst_is_64 = w64 > 0xFFFF_FFFF; + let dst = output_to_reg(ctx, outputs[0]); + ctx.emit(Inst::imm_r(dst_is_64, w64, dst)); } else { unimplemented!(); } } Opcode::Iadd | Opcode::Isub => { - let regD = output_to_reg(ctx, iri, 0); - let regL = input_to_reg(ctx, iri, 0); - let regR = input_to_reg(ctx, iri, 1); - let is64 = int_ty_is_64(ty.unwrap()); - let how = if op == Opcode::Iadd { - RMI_R_Op::Add + let lhs = input_to_reg(ctx, inputs[0]); + let rhs = input_to_reg_mem_imm(ctx, inputs[1]); + let dst = output_to_reg(ctx, outputs[0]); + + // TODO For add, try to commute the operands if one is an immediate. + + let is_64 = int_ty_is_64(ty.unwrap()); + let alu_op = if op == Opcode::Iadd { + AluRmiROpcode::Add } else { - RMI_R_Op::Sub + AluRmiROpcode::Sub }; - ctx.emit(Inst::mov_r_r(true, regL, regD)); - ctx.emit(Inst::alu_rmi_r(is64, how, RMI::reg(regR), regD)); + + ctx.emit(Inst::mov_r_r(true, lhs, dst)); + ctx.emit(Inst::alu_rmi_r(is_64, alu_op, rhs, dst)); } Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => { // TODO: implement imm shift value into insn - let tySL = ctx.input_ty(iri, 0); - let tyD = ctx.output_ty(iri, 0); // should be the same as tySL - let regSL = input_to_reg(ctx, iri, 0); - let regSR = input_to_reg(ctx, iri, 1); - let regD = output_to_reg(ctx, iri, 0); - if tyD == tySL && (tyD == types::I32 || tyD == types::I64) { - let how = match op { - Opcode::Ishl => ShiftKind::Left, - Opcode::Ushr => ShiftKind::RightZ, - Opcode::Sshr => ShiftKind::RightS, - _ => unreachable!(), - }; - let is64 = tyD == types::I64; - let r_rcx = regs::rcx(); - let w_rcx = Writable::::from_reg(r_rcx); - ctx.emit(Inst::mov_r_r(true, regSL, regD)); - ctx.emit(Inst::mov_r_r(true, regSR, w_rcx)); - ctx.emit(Inst::shift_r(is64, how, None /*%cl*/, regD)); + let dst_ty = ctx.output_ty(insn, 0); + assert_eq!(ctx.input_ty(insn, 0), dst_ty); + assert!(dst_ty == types::I32 || dst_ty == types::I64); + + let lhs = input_to_reg(ctx, inputs[0]); + let rhs = input_to_reg(ctx, inputs[1]); + let dst = output_to_reg(ctx, outputs[0]); + + let shift_kind = match op { + Opcode::Ishl => ShiftKind::Left, + Opcode::Ushr => ShiftKind::RightZ, + Opcode::Sshr => ShiftKind::RightS, + _ => unreachable!(), + }; + + let is_64 = dst_ty == types::I64; + let w_rcx = Writable::from_reg(regs::rcx()); + ctx.emit(Inst::mov_r_r(true, lhs, dst)); + ctx.emit(Inst::mov_r_r(true, rhs, w_rcx)); + ctx.emit(Inst::shift_r(is_64, shift_kind, None /*%cl*/, dst)); + } + + Opcode::Uextend + | Opcode::Sextend + | Opcode::Bint + | Opcode::Breduce + | Opcode::Bextend + | Opcode::Ireduce => { + let src_ty = ctx.input_ty(insn, 0); + let dst_ty = ctx.output_ty(insn, 0); + + // TODO: if the source operand is a load, incorporate that. + let src = input_to_reg(ctx, inputs[0]); + let dst = output_to_reg(ctx, outputs[0]); + + let ext_mode = match (src_ty.bits(), dst_ty.bits()) { + (1, 32) | (8, 32) => ExtMode::BL, + (1, 64) | (8, 64) => ExtMode::BQ, + (16, 32) => ExtMode::WL, + (16, 64) => ExtMode::WQ, + (32, 64) => ExtMode::LQ, + _ => unreachable!( + "unexpected extension kind from {:?} to {:?}", + src_ty, dst_ty + ), + }; + + if op == Opcode::Sextend { + ctx.emit(Inst::movsx_rm_r(ext_mode, RegMem::reg(src), dst)); } else { - unimplemented!() + // All of these other opcodes are simply a move from a zero-extended source. Here + // is why this works, in each case: + // + // - Bint: Bool-to-int. We always represent a bool as a 0 or 1, so we + // merely need to zero-extend here. + // + // - Breduce, Bextend: changing width of a boolean. We represent a + // bool as a 0 or 1, so again, this is a zero-extend / no-op. + // + // - Ireduce: changing width of an integer. Smaller ints are stored + // with undefined high-order bits, so we can simply do a copy. + ctx.emit(Inst::movzx_rm_r(ext_mode, RegMem::reg(src), dst)); } } - Opcode::Uextend | Opcode::Sextend => { - // TODO: this is all extremely lame, all because Mov{ZX,SX}_M_R - // don't accept a register source operand. They should be changed - // so as to have _RM_R form. - // TODO2: if the source operand is a load, incorporate that. - let isZX = op == Opcode::Uextend; - let tyS = ctx.input_ty(iri, 0); - let tyD = ctx.output_ty(iri, 0); - let regS = input_to_reg(ctx, iri, 0); - let regD = output_to_reg(ctx, iri, 0); - ctx.emit(Inst::mov_r_r(true, regS, regD)); - match (tyS, tyD, isZX) { - (types::I8, types::I64, false) => { - ctx.emit(Inst::shift_r(true, ShiftKind::Left, Some(56), regD)); - ctx.emit(Inst::shift_r(true, ShiftKind::RightS, Some(56), regD)); - } - _ => unimplemented!(), - } + Opcode::Icmp => { + let condcode = inst_condcode(ctx.data(insn)); + let cc = CC::from_intcc(condcode); + let ty = ctx.input_ty(insn, 0); + + // TODO Try to commute the operands (and invert the condition) if one is an immediate. + let lhs = input_to_reg(ctx, inputs[0]); + let rhs = input_to_reg_mem_imm(ctx, inputs[1]); + let dst = output_to_reg(ctx, outputs[0]); + + // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives + // us dst - src at the machine instruction level, so invert operands. + ctx.emit(Inst::cmp_rmi_r(ty.bytes() as u8, rhs, lhs)); + ctx.emit(Inst::setcc(cc, dst)); } Opcode::FallthroughReturn | Opcode::Return => { - for i in 0..ctx.num_inputs(iri) { - let src_reg = input_to_reg(ctx, iri, i); + for i in 0..ctx.num_inputs(insn) { + let src_reg = input_to_reg(ctx, inputs[i]); let retval_reg = ctx.retval(i); if src_reg.get_class() == RegClass::I64 { ctx.emit(Inst::mov_r_r(true, src_reg, retval_reg)); } else if src_reg.get_class() == RegClass::V128 { - ctx.emit(Inst::xmm_r_r(SSE_Op::SSE2_Movsd, src_reg, retval_reg)); + ctx.emit(Inst::xmm_mov_rm_r( + SseOpcode::Movsd, + RegMem::reg(src_reg), + retval_reg, + )); } } // N.B.: the Ret itself is generated by the ABI. } + Opcode::Call | Opcode::CallIndirect => { + let loc = ctx.srcloc(insn); + let (mut abi, inputs) = match op { + Opcode::Call => { + let (extname, dist) = ctx.call_target(insn).unwrap(); + let sig = ctx.call_sig(insn).unwrap(); + assert!(inputs.len() == sig.params.len()); + assert!(outputs.len() == sig.returns.len()); + ( + X64ABICall::from_func(sig, &extname, dist, loc)?, + &inputs[..], + ) + } + + Opcode::CallIndirect => { + let ptr = input_to_reg(ctx, inputs[0]); + let sig = ctx.call_sig(insn).unwrap(); + assert!(inputs.len() - 1 == sig.params.len()); + assert!(outputs.len() == sig.returns.len()); + (X64ABICall::from_ptr(sig, ptr, loc, op)?, &inputs[1..]) + } + + _ => unreachable!(), + }; + + abi.emit_stack_pre_adjust(ctx); + assert!(inputs.len() == abi.num_args()); + for (i, input) in inputs.iter().enumerate() { + let arg_reg = input_to_reg(ctx, *input); + abi.emit_copy_reg_to_arg(ctx, i, arg_reg); + } + abi.emit_call(ctx); + for (i, output) in outputs.iter().enumerate() { + let retval_reg = output_to_reg(ctx, *output); + abi.emit_copy_retval_to_reg(ctx, i, retval_reg); + } + abi.emit_stack_post_adjust(ctx); + } + + Opcode::Debugtrap => { + ctx.emit(Inst::Hlt); + } + + Opcode::Trap => { + let trap_info = (ctx.srcloc(insn), inst_trapcode(ctx.data(insn)).unwrap()); + ctx.emit(Inst::Ud2 { trap_info }) + } + + Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv => { + let lhs = input_to_reg(ctx, inputs[0]); + let rhs = input_to_reg(ctx, inputs[1]); + let dst = output_to_reg(ctx, outputs[0]); + let is_64 = flt_ty_is_64(ty.unwrap()); + if !is_64 { + let sse_op = match op { + Opcode::Fadd => SseOpcode::Addss, + Opcode::Fsub => SseOpcode::Subss, + Opcode::Fmul => SseOpcode::Mulss, + Opcode::Fdiv => SseOpcode::Divss, + // TODO Fmax, Fmin. + _ => unimplemented!(), + }; + ctx.emit(Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(lhs), dst)); + ctx.emit(Inst::xmm_rm_r(sse_op, RegMem::reg(rhs), dst)); + } else { + unimplemented!("unimplemented lowering for opcode {:?}", op); + } + } + + Opcode::Fcopysign => { + let dst = output_to_reg(ctx, outputs[0]); + let lhs = input_to_reg(ctx, inputs[0]); + let rhs = input_to_reg(ctx, inputs[1]); + if !flt_ty_is_64(ty.unwrap()) { + // movabs 0x8000_0000, tmp_gpr1 + // movd tmp_gpr1, tmp_xmm1 + // movaps tmp_xmm1, dst + // andnps src_1, dst + // movss src_2, tmp_xmm2 + // andps tmp_xmm1, tmp_xmm2 + // orps tmp_xmm2, dst + let tmp_gpr1 = ctx.alloc_tmp(RegClass::I64, I32); + let tmp_xmm1 = ctx.alloc_tmp(RegClass::V128, F32); + let tmp_xmm2 = ctx.alloc_tmp(RegClass::V128, F32); + ctx.emit(Inst::imm_r(true, 0x8000_0000, tmp_gpr1)); + ctx.emit(Inst::xmm_mov_rm_r( + SseOpcode::Movd, + RegMem::reg(tmp_gpr1.to_reg()), + tmp_xmm1, + )); + ctx.emit(Inst::xmm_mov_rm_r( + SseOpcode::Movaps, + RegMem::reg(tmp_xmm1.to_reg()), + dst, + )); + ctx.emit(Inst::xmm_rm_r(SseOpcode::Andnps, RegMem::reg(lhs), dst)); + ctx.emit(Inst::xmm_mov_rm_r( + SseOpcode::Movss, + RegMem::reg(rhs), + tmp_xmm2, + )); + ctx.emit(Inst::xmm_rm_r( + SseOpcode::Andps, + RegMem::reg(tmp_xmm1.to_reg()), + tmp_xmm2, + )); + ctx.emit(Inst::xmm_rm_r( + SseOpcode::Orps, + RegMem::reg(tmp_xmm2.to_reg()), + dst, + )); + } else { + unimplemented!("{:?} for non 32-bit destination is not supported", op); + } + } + + Opcode::Load + | Opcode::Uload8 + | Opcode::Sload8 + | Opcode::Uload16 + | Opcode::Sload16 + | Opcode::Uload32 + | Opcode::Sload32 + | Opcode::LoadComplex + | Opcode::Uload8Complex + | Opcode::Sload8Complex + | Opcode::Uload16Complex + | Opcode::Sload16Complex + | Opcode::Uload32Complex + | Opcode::Sload32Complex => { + let offset = ldst_offset(ctx.data(insn)).unwrap(); + + let elem_ty = match op { + Opcode::Sload8 | Opcode::Uload8 | Opcode::Sload8Complex | Opcode::Uload8Complex => { + types::I8 + } + Opcode::Sload16 + | Opcode::Uload16 + | Opcode::Sload16Complex + | Opcode::Uload16Complex => types::I16, + Opcode::Sload32 + | Opcode::Uload32 + | Opcode::Sload32Complex + | Opcode::Uload32Complex => types::I32, + Opcode::Load | Opcode::LoadComplex => ctx.output_ty(insn, 0), + _ => unimplemented!(), + }; + + let ext_mode = match elem_ty.bytes() { + 1 => Some(ExtMode::BQ), + 2 => Some(ExtMode::WQ), + 4 => Some(ExtMode::LQ), + _ => None, + }; + + let sign_extend = match op { + Opcode::Sload8 + | Opcode::Sload8Complex + | Opcode::Sload16 + | Opcode::Sload16Complex + | Opcode::Sload32 + | Opcode::Sload32Complex => true, + _ => false, + }; + + let is_float = is_float_ty(elem_ty); + + let addr = match op { + Opcode::Load + | Opcode::Uload8 + | Opcode::Sload8 + | Opcode::Uload16 + | Opcode::Sload16 + | Opcode::Uload32 + | Opcode::Sload32 => { + assert!(inputs.len() == 1, "only one input for load operands"); + let base = input_to_reg(ctx, inputs[0]); + Amode::imm_reg(offset as u32, base) + } + + Opcode::LoadComplex + | Opcode::Uload8Complex + | Opcode::Sload8Complex + | Opcode::Uload16Complex + | Opcode::Sload16Complex + | Opcode::Uload32Complex + | Opcode::Sload32Complex => { + assert!( + inputs.len() == 2, + "can't handle more than two inputs in complex load" + ); + let base = input_to_reg(ctx, inputs[0]); + let index = input_to_reg(ctx, inputs[1]); + let shift = 0; + Amode::imm_reg_reg_shift(offset as u32, base, index, shift) + } + + _ => unreachable!(), + }; + + let dst = output_to_reg(ctx, outputs[0]); + match (sign_extend, is_float) { + (true, false) => { + // The load is sign-extended only when the output size is lower than 64 bits, + // so ext-mode is defined in this case. + ctx.emit(Inst::movsx_rm_r(ext_mode.unwrap(), RegMem::mem(addr), dst)); + } + (false, false) => { + if elem_ty.bytes() == 8 { + // Use a plain load. + ctx.emit(Inst::mov64_m_r(addr, dst)) + } else { + // Use a zero-extended load. + ctx.emit(Inst::movzx_rm_r(ext_mode.unwrap(), RegMem::mem(addr), dst)) + } + } + (_, true) => { + ctx.emit(match elem_ty { + F32 => Inst::xmm_mov_rm_r(SseOpcode::Movd, RegMem::mem(addr), dst), + _ => unimplemented!("FP load not 32-bit"), + }); + } + } + } + + Opcode::Store + | Opcode::Istore8 + | Opcode::Istore16 + | Opcode::Istore32 + | Opcode::StoreComplex + | Opcode::Istore8Complex + | Opcode::Istore16Complex + | Opcode::Istore32Complex => { + let offset = ldst_offset(ctx.data(insn)).unwrap(); + + let elem_ty = match op { + Opcode::Istore8 | Opcode::Istore8Complex => types::I8, + Opcode::Istore16 | Opcode::Istore16Complex => types::I16, + Opcode::Istore32 | Opcode::Istore32Complex => types::I32, + Opcode::Store | Opcode::StoreComplex => ctx.input_ty(insn, 0), + _ => unreachable!(), + }; + let is_float = is_float_ty(elem_ty); + + let addr = match op { + Opcode::Store | Opcode::Istore8 | Opcode::Istore16 | Opcode::Istore32 => { + assert!( + inputs.len() == 2, + "only one input for store memory operands" + ); + let base = input_to_reg(ctx, inputs[1]); + // TODO sign? + Amode::imm_reg(offset as u32, base) + } + + Opcode::StoreComplex + | Opcode::Istore8Complex + | Opcode::Istore16Complex + | Opcode::Istore32Complex => { + assert!( + inputs.len() == 3, + "can't handle more than two inputs in complex load" + ); + let base = input_to_reg(ctx, inputs[1]); + let index = input_to_reg(ctx, inputs[2]); + let shift = 0; + Amode::imm_reg_reg_shift(offset as u32, base, index, shift) + } + + _ => unreachable!(), + }; + + let src = input_to_reg(ctx, inputs[0]); + + if is_float { + ctx.emit(match elem_ty { + F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr), + _ => unimplemented!("FP store not 32-bit"), + }); + } else { + ctx.emit(Inst::mov_r_m(elem_ty.bytes() as u8, src, addr)); + } + } + + Opcode::StackAddr => { + let (stack_slot, offset) = match *ctx.data(insn) { + InstructionData::StackLoad { + opcode: Opcode::StackAddr, + stack_slot, + offset, + } => (stack_slot, offset), + _ => unreachable!(), + }; + let dst = output_to_reg(ctx, outputs[0]); + let offset: i32 = offset.into(); + println!("stackslot_addr: {:?} @ off{}", stack_slot, offset); + let inst = ctx + .abi() + .stackslot_addr(stack_slot, u32::try_from(offset).unwrap(), dst); + ctx.emit(inst); + } + Opcode::IaddImm | Opcode::ImulImm | Opcode::UdivImm @@ -240,25 +628,10 @@ fn lower_insn_to_regs<'a>(ctx: Ctx<'a>, iri: IRInst) { | Opcode::SshrImm => { panic!("ALU+imm and ALU+carry ops should not appear here!"); } - Opcode::Fadd | Opcode::Fsub => { - let regD = output_to_reg(ctx, iri, 0); - let regL = input_to_reg(ctx, iri, 0); - let regR = input_to_reg(ctx, iri, 1); - let is64 = flt_ty_is_64(ty.unwrap()); - if !is64 { - let inst = if op == Opcode::Fadd { - SSE_Op::SSE_Addss - } else { - SSE_Op::SSE_Subss - }; - ctx.emit(Inst::xmm_r_r(SSE_Op::SSE_Movss, regL, regD)); - ctx.emit(Inst::xmm_rm_r(inst, RM::reg(regR), regD)); - } else { - unimplemented!("unimplemented lowering for opcode {:?}", op); - } - } _ => unimplemented!("unimplemented lowering for opcode {:?}", op), } + + Ok(()) } //============================================================================= @@ -268,8 +641,7 @@ impl LowerBackend for X64Backend { type MInst = Inst; fn lower>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> { - lower_insn_to_regs(ctx, ir_inst); - Ok(()) + lower_insn_to_regs(ctx, ir_inst) } fn lower_branch_group>( @@ -287,60 +659,79 @@ impl LowerBackend for X64Backend { // verifier pass. assert!(branches.len() <= 2); - let mut unimplemented = false; - if branches.len() == 2 { // Must be a conditional branch followed by an unconditional branch. let op0 = ctx.data(branches[0]).opcode(); let op1 = ctx.data(branches[1]).opcode(); - println!( - "QQQQ lowering two-branch group: opcodes are {:?} and {:?}", - op0, op1 + trace!( + "lowering two-branch group: opcodes are {:?} and {:?}", + op0, + op1 ); - assert!(op1 == Opcode::Jump || op1 == Opcode::Fallthrough); + let taken = BranchTarget::Label(targets[0]); let not_taken = match op1 { Opcode::Jump => BranchTarget::Label(targets[1]), Opcode::Fallthrough => BranchTarget::Label(fallthrough.unwrap()), _ => unreachable!(), // assert above. }; + match op0 { Opcode::Brz | Opcode::Brnz => { - let tyS = ctx.input_ty(branches[0], 0); - if is_int_ty(tyS) { - let rS = input_to_reg(ctx, branches[0], 0); + let src_ty = ctx.input_ty(branches[0], 0); + if is_int_ty(src_ty) || is_bool_ty(src_ty) { + let src = input_to_reg( + ctx, + InsnInput { + insn: branches[0], + input: 0, + }, + ); let cc = match op0 { Opcode::Brz => CC::Z, Opcode::Brnz => CC::NZ, _ => unreachable!(), }; - let sizeB = int_ty_to_sizeB(tyS); - ctx.emit(Inst::cmp_rmi_r(sizeB, RMI::imm(0), rS)); - ctx.emit(Inst::jmp_cond_symm(cc, taken, not_taken)); + let size_bytes = src_ty.bytes() as u8; + ctx.emit(Inst::cmp_rmi_r(size_bytes, RegMemImm::imm(0), src)); + ctx.emit(Inst::jmp_cond(cc, taken, not_taken)); } else { - unimplemented = true; + unimplemented!("brz/brnz with non-int type {:?}", src_ty); } } + Opcode::BrIcmp => { - let tyS = ctx.input_ty(branches[0], 0); - if is_int_ty(tyS) { - let rSL = input_to_reg(ctx, branches[0], 0); - let rSR = input_to_reg(ctx, branches[0], 1); - let cc = intCC_to_x64_CC(inst_condcode(ctx.data(branches[0]))); - let sizeB = int_ty_to_sizeB(tyS); - // FIXME verify rSR vs rSL ordering - ctx.emit(Inst::cmp_rmi_r(sizeB, RMI::reg(rSR), rSL)); - ctx.emit(Inst::jmp_cond_symm(cc, taken, not_taken)); + let src_ty = ctx.input_ty(branches[0], 0); + if is_int_ty(src_ty) || is_bool_ty(src_ty) { + let lhs = input_to_reg( + ctx, + InsnInput { + insn: branches[0], + input: 0, + }, + ); + let rhs = input_to_reg_mem_imm( + ctx, + InsnInput { + insn: branches[0], + input: 1, + }, + ); + let cc = CC::from_intcc(inst_condcode(ctx.data(branches[0]))); + let byte_size = src_ty.bytes() as u8; + // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives + // us dst - src at the machine instruction level, so invert operands. + ctx.emit(Inst::cmp_rmi_r(byte_size, rhs, lhs)); + ctx.emit(Inst::jmp_cond(cc, taken, not_taken)); } else { - unimplemented = true; + unimplemented!("bricmp with non-int type {:?}", src_ty); } } + // TODO: Brif/icmp, Brff/icmp, jump tables - _ => { - unimplemented = true; - } + _ => unimplemented!("branch opcode"), } } else { assert!(branches.len() == 1); @@ -348,23 +739,13 @@ impl LowerBackend for X64Backend { // Must be an unconditional branch or trap. let op = ctx.data(branches[0]).opcode(); match op { - Opcode::Jump => { + Opcode::Jump | Opcode::Fallthrough => { ctx.emit(Inst::jmp_known(BranchTarget::Label(targets[0]))); } - Opcode::Fallthrough => { - ctx.emit(Inst::jmp_known(BranchTarget::Label(targets[0]))); - } - Opcode::Trap => { - unimplemented = true; - } _ => panic!("Unknown branch type!"), } } - if unimplemented { - unimplemented!("lower_branch_group(x64): can't handle: {:?}", branches); - } - Ok(()) } } diff --git a/third_party/rust/cranelift-codegen/src/isa/x64/mod.rs b/third_party/rust/cranelift-codegen/src/isa/x64/mod.rs index 3b1652cb1093..7666875a0eec 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x64/mod.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x64/mod.rs @@ -40,7 +40,7 @@ impl X64Backend { fn compile_vcode(&self, func: &Function, flags: Flags) -> CodegenResult> { // This performs lowering to VCode, register-allocates the code, computes // block layout and finalizes branches. The result is ready for binary emission. - let abi = Box::new(abi::X64ABIBody::new(&func, flags)); + let abi = Box::new(abi::X64ABIBody::new(&func, flags)?); compile::compile::(&func, self, abi) } } diff --git a/third_party/rust/cranelift-codegen/src/isa/x86/enc_tables.rs b/third_party/rust/cranelift-codegen/src/isa/x86/enc_tables.rs index 0786d375783c..b64140cfa896 100644 --- a/third_party/rust/cranelift-codegen/src/isa/x86/enc_tables.rs +++ b/third_party/rust/cranelift-codegen/src/isa/x86/enc_tables.rs @@ -596,6 +596,100 @@ fn expand_minmax( cfg.recompute_block(pos.func, done); } +/// This legalization converts a minimum/maximum operation into a sequence that matches the +/// non-x86-friendly WebAssembly semantics of NaN handling. This logic is kept separate from +/// [expand_minmax] above (the scalar version) for code clarity. +fn expand_minmax_vector( + inst: ir::Inst, + func: &mut ir::Function, + _cfg: &mut ControlFlowGraph, + _isa: &dyn TargetIsa, +) { + let ty = func.dfg.ctrl_typevar(inst); + debug_assert!(ty.is_vector()); + let (x, y, x86_opcode, is_max) = match func.dfg[inst] { + ir::InstructionData::Binary { + opcode: ir::Opcode::Fmin, + args, + } => (args[0], args[1], ir::Opcode::X86Fmin, false), + ir::InstructionData::Binary { + opcode: ir::Opcode::Fmax, + args, + } => (args[0], args[1], ir::Opcode::X86Fmax, true), + _ => panic!("Expected fmin/fmax: {}", func.dfg.display_inst(inst, None)), + }; + + let mut pos = FuncCursor::new(func).at_inst(inst); + pos.use_srcloc(inst); + + // This sequence is complex due to how x86 handles NaNs and +0/-0. If x86 finds a NaN in + // either lane it returns the second operand; likewise, if both operands are in {+0.0, -0.0} + // it returns the second operand. To match the behavior of "return the minimum of the + // operands or a canonical NaN if either operand is NaN," we must compare in both + // directions. + let (forward_inst, dfg) = pos.ins().Binary(x86_opcode, ty, x, y); + let forward = dfg.first_result(forward_inst); + let (backward_inst, dfg) = pos.ins().Binary(x86_opcode, ty, y, x); + let backward = dfg.first_result(backward_inst); + + let (value, mask) = if is_max { + // For maximum: + // Find any differences between the forward and backward `max` operation. + let difference = pos.ins().bxor(forward, backward); + // Merge in the differences. + let propagate_nans_and_plus_zero = pos.ins().bor(backward, difference); + let value = pos.ins().fsub(propagate_nans_and_plus_zero, difference); + // Discover which lanes have NaNs in them. + let find_nan_lanes_mask = pos.ins().fcmp(FloatCC::Unordered, difference, value); + (value, find_nan_lanes_mask) + } else { + // For minimum: + // If either lane is a NaN, we want to use these bits, not the second operand bits. + let propagate_nans = pos.ins().bor(backward, forward); + // Find which lanes contain a NaN with an unordered comparison, filling the mask with + // 1s. + let find_nan_lanes_mask = pos.ins().fcmp(FloatCC::Unordered, forward, propagate_nans); + let bitcast_find_nan_lanes_mask = pos.ins().raw_bitcast(ty, find_nan_lanes_mask); + // Then flood the value lane with all 1s if that lane is a NaN. This causes all NaNs + // along this code path to be quieted and negative: after the upcoming shift and and_not, + // all upper bits (sign, exponent, and payload MSB) will be 1s. + let tmp = pos.ins().bor(propagate_nans, bitcast_find_nan_lanes_mask); + (tmp, bitcast_find_nan_lanes_mask) + }; + + // During this lowering we will need to know how many bits to shift by and what type to + // convert to when using an integer shift. Recall that an IEEE754 number looks like: + // `[sign bit] [exponent bits] [significand bits]` + // A quiet NaN has all exponent bits set to 1 and the most significant bit of the + // significand set to 1; a signaling NaN has the same exponent but the MSB of the + // significand is set to 0. The payload of the NaN is the remaining significand bits, and + // WebAssembly assumes a canonical NaN is quiet and has 0s in its payload. To compute this + // canonical NaN, we create a mask for the top 10 bits on F32X4 (1 sign + 8 exp. + 1 MSB + // sig.) and the top 13 bits on F64X2 (1 sign + 11 exp. + 1 MSB sig.). This means that all + // NaNs produced with the mask will be negative (`-NaN`) which is allowed by the sign + // non-determinism in the spec: https://webassembly.github.io/spec/core/bikeshed/index.html#nan-propagation%E2%91%A0 + let (shift_by, ty_as_int) = match ty { + F32X4 => (10, I32X4), + F64X2 => (13, I64X2), + _ => unimplemented!("this legalization only understands 128-bit floating point types"), + }; + + // In order to clear the NaN payload for canonical NaNs, we shift right the NaN lanes (all + // 1s) leaving 0s in the top bits. Remember that non-NaN lanes are all 0s so this has + // little effect. + let mask_as_int = pos.ins().raw_bitcast(ty_as_int, mask); + let shift_mask = pos.ins().ushr_imm(mask_as_int, shift_by); + let shift_mask_as_float = pos.ins().raw_bitcast(ty, shift_mask); + + // Finally, we replace the value with `value & ~shift_mask`. For non-NaN lanes, this is + // equivalent to `... & 1111...` but for NaN lanes this will only have 1s in the top bits, + // clearing the payload. + pos.func + .dfg + .replace(inst) + .band_not(value, shift_mask_as_float); +} + /// x86 has no unsigned-to-float conversions. We handle the easy case of zero-extending i32 to /// i64 with a pattern, the rest needs more code. /// @@ -964,6 +1058,61 @@ fn expand_fcvt_to_sint_sat( cfg.recompute_block(pos.func, done_block); } +/// This legalization converts a vector of 32-bit floating point lanes to signed integer lanes +/// using CVTTPS2DQ (see encoding of `x86_cvtt2si`). This logic is separate from [expand_fcvt_to_sint_sat] +/// above (the scalar version), only due to how the transform groups are set up; TODO if we change +/// the SIMD legalization groups, then this logic could be merged into [expand_fcvt_to_sint_sat] +/// (see https://github.com/bytecodealliance/wasmtime/issues/1745). +fn expand_fcvt_to_sint_sat_vector( + inst: ir::Inst, + func: &mut ir::Function, + _cfg: &mut ControlFlowGraph, + _isa: &dyn TargetIsa, +) { + let mut pos = FuncCursor::new(func).at_inst(inst); + pos.use_srcloc(inst); + + if let ir::InstructionData::Unary { + opcode: ir::Opcode::FcvtToSintSat, + arg, + } = pos.func.dfg[inst] + { + let controlling_type = pos.func.dfg.ctrl_typevar(inst); + if controlling_type == I32X4 { + debug_assert_eq!(pos.func.dfg.value_type(arg), F32X4); + // We must both quiet any NaNs--setting that lane to 0--and saturate any + // lanes that might overflow during conversion to the highest/lowest signed integer + // allowed in that lane. + + // Saturate NaNs: `fcmp eq` will not match if a lane contains a NaN. We use ANDPS to + // avoid doing the comparison twice (we need the zeroed lanes to find differences). + let zeroed_nans = pos.ins().fcmp(FloatCC::Equal, arg, arg); + let zeroed_nans_bitcast = pos.ins().raw_bitcast(F32X4, zeroed_nans); + let zeroed_nans_copy = pos.ins().band(arg, zeroed_nans_bitcast); + + // Find differences with the zeroed lanes (we will only use the MSB: 1 if positive or + // NaN, 0 otherwise). + let differences = pos.ins().bxor(zeroed_nans_bitcast, arg); + let differences_bitcast = pos.ins().raw_bitcast(I32X4, differences); + + // Convert the numeric lanes. CVTTPS2DQ will mark overflows with 0x80000000 (MSB set). + let converted = pos.ins().x86_cvtt2si(I32X4, zeroed_nans_copy); + + // Create a mask of all 1s only on positive overflow, 0s otherwise. This uses the MSB + // of `differences` (1 when positive or NaN) and the MSB of `converted` (1 on positive + // overflow). + let tmp = pos.ins().band(differences_bitcast, converted); + let mask = pos.ins().sshr_imm(tmp, 31); + + // Apply the mask to create 0x7FFFFFFF for positive overflow. XOR of all 0s (all other + // cases) has no effect. + pos.func.dfg.replace(inst).bxor(converted, mask); + } else { + unimplemented!("cannot legalize {}", pos.func.dfg.display_inst(inst, None)) + } + } +} + fn expand_fcvt_to_uint( inst: ir::Inst, func: &mut ir::Function, diff --git a/third_party/rust/cranelift-codegen/src/legalizer/heap.rs b/third_party/rust/cranelift-codegen/src/legalizer/heap.rs index 663736793b5a..26a0640f0836 100644 --- a/third_party/rust/cranelift-codegen/src/legalizer/heap.rs +++ b/third_party/rust/cranelift-codegen/src/legalizer/heap.rs @@ -66,19 +66,14 @@ fn dynamic_addr( // Start with the bounds check. Trap if `offset + access_size > bound`. let bound = pos.ins().global_value(offset_ty, bound_gv); - let oob; - if access_size == 1 { + let (cc, lhs, bound) = if access_size == 1 { // `offset > bound - 1` is the same as `offset >= bound`. - oob = pos - .ins() - .icmp(IntCC::UnsignedGreaterThanOrEqual, offset, bound); + (IntCC::UnsignedGreaterThanOrEqual, offset, bound) } else if access_size <= min_size { // We know that bound >= min_size, so here we can compare `offset > bound - access_size` // without wrapping. let adj_bound = pos.ins().iadd_imm(bound, -(access_size as i64)); - oob = pos - .ins() - .icmp(IntCC::UnsignedGreaterThan, offset, adj_bound); + (IntCC::UnsignedGreaterThan, offset, adj_bound) } else { // We need an overflow check for the adjusted offset. let access_size_val = pos.ins().iconst(offset_ty, access_size as i64); @@ -88,13 +83,27 @@ fn dynamic_addr( overflow, ir::TrapCode::HeapOutOfBounds, ); - oob = pos - .ins() - .icmp(IntCC::UnsignedGreaterThan, adj_offset, bound); - } + (IntCC::UnsignedGreaterThan, adj_offset, bound) + }; + let oob = pos.ins().icmp(cc, lhs, bound); pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds); - compute_addr(isa, inst, heap, addr_ty, offset, offset_ty, pos.func); + let spectre_oob_comparison = if isa.flags().enable_heap_access_spectre_mitigation() { + Some((cc, lhs, bound)) + } else { + None + }; + + compute_addr( + isa, + inst, + heap, + addr_ty, + offset, + offset_ty, + pos.func, + spectre_oob_comparison, + ); } /// Expand a `heap_addr` for a static heap. @@ -146,20 +155,35 @@ fn static_addr( // With that we have an optimization here where with 32-bit offsets and // `bound - access_size >= 4GB` we can omit a bounds check. let limit = bound - access_size; + let mut spectre_oob_comparison = None; if offset_ty != ir::types::I32 || limit < 0xffff_ffff { - let oob = if limit & 1 == 1 { + let (cc, lhs, limit_imm) = if limit & 1 == 1 { // Prefer testing `offset >= limit - 1` when limit is odd because an even number is // likely to be a convenient constant on ARM and other RISC architectures. - pos.ins() - .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, offset, limit as i64 - 1) + let limit = limit as i64 - 1; + (IntCC::UnsignedGreaterThanOrEqual, offset, limit) } else { - pos.ins() - .icmp_imm(IntCC::UnsignedGreaterThan, offset, limit as i64) + let limit = limit as i64; + (IntCC::UnsignedGreaterThan, offset, limit) }; + let oob = pos.ins().icmp_imm(cc, lhs, limit_imm); pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds); + if isa.flags().enable_heap_access_spectre_mitigation() { + let limit = pos.ins().iconst(offset_ty, limit_imm); + spectre_oob_comparison = Some((cc, lhs, limit)); + } } - compute_addr(isa, inst, heap, addr_ty, offset, offset_ty, pos.func); + compute_addr( + isa, + inst, + heap, + addr_ty, + offset, + offset_ty, + pos.func, + spectre_oob_comparison, + ); } /// Emit code for the base address computation of a `heap_addr` instruction. @@ -171,6 +195,11 @@ fn compute_addr( mut offset: ir::Value, offset_ty: ir::Type, func: &mut ir::Function, + // If we are performing Spectre mitigation with conditional selects, the + // values to compare and the condition code that indicates an out-of bounds + // condition; on this condition, the conditional move will choose a + // speculatively safe address (a zero / null pointer) instead. + spectre_oob_comparison: Option<(IntCC, ir::Value, ir::Value)>, ) { let mut pos = FuncCursor::new(func).at_inst(inst); pos.use_srcloc(inst); @@ -198,5 +227,15 @@ fn compute_addr( pos.ins().global_value(addr_ty, base_gv) }; - pos.func.dfg.replace(inst).iadd(base, offset); + if let Some((cc, a, b)) = spectre_oob_comparison { + let final_addr = pos.ins().iadd(base, offset); + let zero = pos.ins().iconst(addr_ty, 0); + let flags = pos.ins().ifcmp(a, b); + pos.func + .dfg + .replace(inst) + .selectif_spectre_guard(addr_ty, cc, flags, zero, final_addr); + } else { + pos.func.dfg.replace(inst).iadd(base, offset); + } } diff --git a/third_party/rust/cranelift-codegen/src/machinst/buffer.rs b/third_party/rust/cranelift-codegen/src/machinst/buffer.rs index 03b4ab075096..5f54b1ba77e7 100644 --- a/third_party/rust/cranelift-codegen/src/machinst/buffer.rs +++ b/third_party/rust/cranelift-codegen/src/machinst/buffer.rs @@ -12,13 +12,13 @@ //! from the branch itself. //! //! - The lowering of control flow from the CFG-with-edges produced by -//! [BlockLoweringOrder], combined with many empty edge blocks when the register -//! allocator does not need to insert any spills/reloads/moves in edge blocks, -//! results in many suboptimal branch patterns. The lowering also pays no -//! attention to block order, and so two-target conditional forms (cond-br -//! followed by uncond-br) can often by avoided because one of the targets is -//! the fallthrough. There are several cases here where we can simplify to use -//! fewer branches. +//! [BlockLoweringOrder](super::BlockLoweringOrder), combined with many empty +//! edge blocks when the register allocator does not need to insert any +//! spills/reloads/moves in edge blocks, results in many suboptimal branch +//! patterns. The lowering also pays no attention to block order, and so +//! two-target conditional forms (cond-br followed by uncond-br) can often by +//! avoided because one of the targets is the fallthrough. There are several +//! cases here where we can simplify to use fewer branches. //! //! This "buffer" implements a single-pass code emission strategy (with a later //! "fixup" pass, but only through recorded fixups, not all instructions). The @@ -41,7 +41,7 @@ //! by the emitter (e.g., vcode iterating over instruction structs). The emitter //! has some awareness of this: it either asks for an island between blocks, so //! it is not accidentally executed, or else it emits a branch around the island -//! when all other options fail (see [Inst::EmitIsland] meta-instruction). +//! when all other options fail (see `Inst::EmitIsland` meta-instruction). //! //! - A "veneer" is an instruction (or sequence of instructions) in an "island" //! that implements a longer-range reference to a label. The idea is that, for @@ -1024,7 +1024,7 @@ impl MachBuffer { let veneer_offset = self.cur_offset(); trace!("making a veneer at {}", veneer_offset); let slice = &mut self.data[start..end]; - // Patch the original label use to refer to teh veneer. + // Patch the original label use to refer to the veneer. trace!( "patching original at offset {} to veneer offset {}", offset, diff --git a/third_party/rust/cranelift-codegen/src/machinst/lower.rs b/third_party/rust/cranelift-codegen/src/machinst/lower.rs index 6b1ff0a0939d..ea97607dfe07 100644 --- a/third_party/rust/cranelift-codegen/src/machinst/lower.rs +++ b/third_party/rust/cranelift-codegen/src/machinst/lower.rs @@ -132,7 +132,11 @@ pub trait LowerCtx { fn get_input(&self, ir_inst: Inst, idx: usize) -> LowerInput; /// Get the `idx`th output register of the given IR instruction. When /// `backend.lower_inst_to_regs(ctx, inst)` is called, it is expected that - /// the backend will write results to these output register(s). + /// the backend will write results to these output register(s). This + /// register will always be "fresh"; it is guaranteed not to overlap with + /// any of the inputs, and can be freely used as a scratch register within + /// the lowered instruction sequence, as long as its final value is the + /// result of the computation. fn get_output(&self, ir_inst: Inst, idx: usize) -> Writable; // Codegen primitives: allocate temps, emit instructions, set result registers, diff --git a/third_party/rust/cranelift-codegen/src/machinst/vcode.rs b/third_party/rust/cranelift-codegen/src/machinst/vcode.rs index 81c59f5f2a49..74c99b3af19a 100644 --- a/third_party/rust/cranelift-codegen/src/machinst/vcode.rs +++ b/third_party/rust/cranelift-codegen/src/machinst/vcode.rs @@ -137,13 +137,10 @@ impl VCodeBuilder { /// Set the type of a VReg. pub fn set_vreg_type(&mut self, vreg: VirtualReg, ty: Type) { if self.vcode.vreg_types.len() <= vreg.get_index() { - self.vcode.vreg_types.resize( - self.vcode.vreg_types.len() - + ((vreg.get_index() + 1) - self.vcode.vreg_types.len()), - ir::types::I8, - ) + self.vcode + .vreg_types + .resize(vreg.get_index() + 1, ir::types::I8); } - self.vcode.vreg_types[vreg.get_index()] = ty; } diff --git a/third_party/rust/cranelift-codegen/src/postopt.rs b/third_party/rust/cranelift-codegen/src/postopt.rs index 426aab00b8a7..ada14e1ff8ee 100644 --- a/third_party/rust/cranelift-codegen/src/postopt.rs +++ b/third_party/rust/cranelift-codegen/src/postopt.rs @@ -386,7 +386,11 @@ fn optimize_complex_addresses(pos: &mut EncCursor, inst: Inst, isa: &dyn TargetI } let ok = pos.func.update_encoding(inst, isa).is_ok(); - debug_assert!(ok); + debug_assert!( + ok, + "failed to update encoding for `{}`", + pos.func.dfg.display_inst(inst, isa) + ); } //---------------------------------------------------------------------- diff --git a/third_party/rust/cranelift-codegen/src/preopt.serialized b/third_party/rust/cranelift-codegen/src/preopt.serialized index f016c6d32d260ec758239ee2c0bfdb630735d80e..bae915dab008d2ecd380e99793768123a1317186 100644 GIT binary patch delta 13 VcmZ3Wy-#aG#Uv4yjrR_T001aY1c delta 92 zcmdm|wLp79#bi4!4o@Zq4h9HdVGw|_*cc?BELH{uD2tQ90LtQEuz<4I8U8~7GXv-3 MgRJ5k>kfzj01Mm@KmY&$ diff --git a/third_party/rust/cranelift-codegen/src/redundant_reload_remover.rs b/third_party/rust/cranelift-codegen/src/redundant_reload_remover.rs index 79b505da8620..501c67ab6bd0 100644 --- a/third_party/rust/cranelift-codegen/src/redundant_reload_remover.rs +++ b/third_party/rust/cranelift-codegen/src/redundant_reload_remover.rs @@ -635,7 +635,11 @@ impl RedundantReloadRemover { // Load is completely redundant. Convert it to a no-op. dfg.replace(inst).fill_nop(arg); let ok = func.update_encoding(inst, isa).is_ok(); - debug_assert!(ok, "fill_nop encoding missing for this type"); + debug_assert!( + ok, + "fill_nop encoding missing for this type: `{}`", + func.dfg.display_inst(inst, isa) + ); } Transform::ChangeToCopyToSSA(ty, reg) => { // We already have the relevant value in some other register. Convert the diff --git a/third_party/rust/cranelift-codegen/src/result.rs b/third_party/rust/cranelift-codegen/src/result.rs index 5ea91f8a0545..493545c15199 100644 --- a/third_party/rust/cranelift-codegen/src/result.rs +++ b/third_party/rust/cranelift-codegen/src/result.rs @@ -21,7 +21,7 @@ pub enum CodegenError { /// Cranelift can compile very large and complicated functions, but the [implementation has /// limits][limits] that cause compilation to fail when they are exceeded. /// - /// [limits]: https://github.com/bytecodealliance/wasmtime/blob/master/cranelift/docs/ir.md#implementation-limits + /// [limits]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/docs/ir.md#implementation-limits #[error("Implementation limit exceeded")] ImplLimitExceeded, diff --git a/third_party/rust/cranelift-codegen/src/settings.rs b/third_party/rust/cranelift-codegen/src/settings.rs index bd6d7bc36071..4879d5c2a89e 100644 --- a/third_party/rust/cranelift-codegen/src/settings.rs +++ b/third_party/rust/cranelift-codegen/src/settings.rs @@ -399,6 +399,7 @@ emit_all_ones_funcaddrs = false enable_probestack = true probestack_func_adjusts_sp = false enable_jump_tables = true +enable_heap_access_spectre_mitigation = true "# ); assert_eq!(f.opt_level(), super::OptLevel::None); diff --git a/third_party/rust/cranelift-frontend/.cargo-checksum.json b/third_party/rust/cranelift-frontend/.cargo-checksum.json index 18f491da5636..ee547e43a6a6 100644 --- a/third_party/rust/cranelift-frontend/.cargo-checksum.json +++ b/third_party/rust/cranelift-frontend/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"2633e2c61491f80fbeea54dcf8763ff7c4b91510da00c32fdba8425cf5267a74","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"dea43e8044284df50f8b8772e9b48ba8b109b45c74111ff73619775d57ad8d67","src/frontend.rs":"d1d8477572f70cc28f71424af272d9eec0adf58af657ff153c4acbbb39822a50","src/lib.rs":"5197f467d1625ee2b117a168f4b1886b4b69d4250faea6618360a5adc70b4e0c","src/ssa.rs":"650d26025706cfb63935f956bca6f166b0edfa32260cd2a8c27f9b49fcc743c3","src/switch.rs":"3bf1f11817565b95edfbc9393ef2bfdeacf534264c9d44b4f93d1432b353af6c","src/variable.rs":"399437bd7d2ac11a7a748bad7dd1f6dac58824d374ec318f36367a9d077cc225"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"2633e2c61491f80fbeea54dcf8763ff7c4b91510da00c32fdba8425cf5267a74","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"dea43e8044284df50f8b8772e9b48ba8b109b45c74111ff73619775d57ad8d67","src/frontend.rs":"ac3a1e3070b1ab0bdec84e4d73ec182b50d0b9a4017e6a95c37adab57571b827","src/lib.rs":"5197f467d1625ee2b117a168f4b1886b4b69d4250faea6618360a5adc70b4e0c","src/ssa.rs":"650d26025706cfb63935f956bca6f166b0edfa32260cd2a8c27f9b49fcc743c3","src/switch.rs":"3bf1f11817565b95edfbc9393ef2bfdeacf534264c9d44b4f93d1432b353af6c","src/variable.rs":"399437bd7d2ac11a7a748bad7dd1f6dac58824d374ec318f36367a9d077cc225"},"package":null} \ No newline at end of file diff --git a/third_party/rust/cranelift-frontend/src/frontend.rs b/third_party/rust/cranelift-frontend/src/frontend.rs index b45166b38396..802a13ff6aa8 100644 --- a/third_party/rust/cranelift-frontend/src/frontend.rs +++ b/third_party/rust/cranelift-frontend/src/frontend.rs @@ -206,6 +206,11 @@ impl<'a> FunctionBuilder<'a> { } } + /// Get the block that this builder is currently at. + pub fn current_block(&self) -> Option { + self.position.expand() + } + /// Set the source location that should be assigned to all new instructions. pub fn set_srcloc(&mut self, srcloc: ir::SourceLoc) { self.srcloc = srcloc; @@ -223,6 +228,11 @@ impl<'a> FunctionBuilder<'a> { block } + /// Insert `block` in the layout *after* the existing block `after`. + pub fn insert_block_after(&mut self, block: Block, after: Block) { + self.func.layout.insert_block_after(block, after); + } + /// After the call to this function, new instructions will be inserted into the designated /// block, in the order they are declared. You must declare the types of the Block arguments /// you will use here. diff --git a/third_party/rust/cranelift-wasm/.cargo-checksum.json b/third_party/rust/cranelift-wasm/.cargo-checksum.json index a916a57a91e8..41ef430d4bb0 100644 --- a/third_party/rust/cranelift-wasm/.cargo-checksum.json +++ b/third_party/rust/cranelift-wasm/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"4a020cf3914cc5e863a120f2bde92354a063ffe827a28422a5ecf86433cf8cbc","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"cce724251d4abc08c6492e1e25c138ab5a0d11e9ac90bc573652b18e034f56ed","src/code_translator.rs":"132fe4f0ee579339f249270907617a1925b628c306659cdaab58df907b6a83ce","src/environ/dummy.rs":"07b6510a7141b92769c914e37386790486f92b691beb0876b8590f2ae5489ee4","src/environ/mod.rs":"692f35d75f125f9c071f7166252f427e4bac29401356f73307c6c36e23c667fb","src/environ/spec.rs":"2ff8524cd592efdef67e5f8d06d144f7d628dee8183848ff4f5e35850f3ce550","src/func_translator.rs":"eb1fcea970407eda872984808e9a3e3a3297c2dea6e3a600ee7116ca89c7b49f","src/lib.rs":"6d3662b3f219a3f7a26f6b44b7921a19da1d892cf78f5a4434fdced5753b069f","src/module_translator.rs":"bcdf5a84226b726a73f4be0acb0318ca89c82584460101378e73021d85bd4485","src/sections_translator.rs":"db567511e273a9e383b18a15fc47f74a1247cbe13f120d7656c21660be53ab78","src/state/func_state.rs":"b114522784984a7cc26a3549c7c17f842885e1232254de81d938f9d155f95aa6","src/state/mod.rs":"20014cb93615467b4d20321b52f67f66040417efcaa739a4804093bb559eed19","src/state/module_state.rs":"3cb3d9de26ec7ccc0ba81ed82163f27648794d4d1d1162eae8eee80a3c0ac05a","src/translation_utils.rs":"20082fded6a8d3637eccbda4465355d8d9fab0a1cd8222accb10cb3e06543689","tests/wasm_testsuite.rs":"da8dedfd11918946e9cf6af68fd4826f020ef90a4e22742b1a30e61a3fb4aedd"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"6c9d8563161a9803e876842482a1c34fd0ea740d5a7141fc51cec3c21ef60eec","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"c82c252fbeeaa101a0eef042b9a925eb1fa3d2b51d19481b9c22e593e6a8d772","src/code_translator.rs":"e8d525ae48f967ebda012981b10dd11fbb46d9223fd95d1e3409da528851fcf7","src/environ/dummy.rs":"922d029491a9f5c55d22fcc9fbeae9e8c6721fa6556527785494f1351874e9f3","src/environ/mod.rs":"692f35d75f125f9c071f7166252f427e4bac29401356f73307c6c36e23c667fb","src/environ/spec.rs":"026a145c1cf9cd25c77e7ea8e0bb43739769dfc4693fcf827f6cdb79acf398a1","src/func_translator.rs":"b4391a11df5c401c9ddd26698105548b7a861c8deb5f84215f0b88cba5327362","src/lib.rs":"7bdbcf638fa30fb05e8320439881f7536824f7f60a7db4f0c1b51ab369edf895","src/module_translator.rs":"47b575f0edbe8a2a3334261742870ce7424e13d91f8980609f9c963a2811e1f6","src/sections_translator.rs":"ebd08548e048c7f792da45aa8d710e7d6f047e9197bc86260743c97d00dd99f6","src/state/func_state.rs":"023e3eb4f69590167baecb3fa8e7b335d69a631fff68fa0ee249075699f71a30","src/state/mod.rs":"20014cb93615467b4d20321b52f67f66040417efcaa739a4804093bb559eed19","src/state/module_state.rs":"3cb3d9de26ec7ccc0ba81ed82163f27648794d4d1d1162eae8eee80a3c0ac05a","src/translation_utils.rs":"0a2a53a7f60a5192661ce4c95ee9bd6775e1eb7d32647e1c6e026b0f8849cd2c","tests/wasm_testsuite.rs":"da8dedfd11918946e9cf6af68fd4826f020ef90a4e22742b1a30e61a3fb4aedd"},"package":null} \ No newline at end of file diff --git a/third_party/rust/cranelift-wasm/Cargo.toml b/third_party/rust/cranelift-wasm/Cargo.toml index 2149d6f8b70a..4e8034979ff0 100644 --- a/third_party/rust/cranelift-wasm/Cargo.toml +++ b/third_party/rust/cranelift-wasm/Cargo.toml @@ -12,7 +12,7 @@ keywords = ["webassembly", "wasm"] edition = "2018" [dependencies] -wasmparser = { version = "0.57.0", default-features = false } +wasmparser = { version = "0.58.0", default-features = false } cranelift-codegen = { path = "../codegen", version = "0.65.0", default-features = false } cranelift-entity = { path = "../entity", version = "0.65.0" } cranelift-frontend = { path = "../frontend", version = "0.65.0", default-features = false } diff --git a/third_party/rust/cranelift-wasm/README.md b/third_party/rust/cranelift-wasm/README.md index b8bee2e098d9..556b4b9eded4 100644 --- a/third_party/rust/cranelift-wasm/README.md +++ b/third_party/rust/cranelift-wasm/README.md @@ -5,4 +5,4 @@ If you're looking for a complete WebAssembly implementation that uses this library, see [Wasmtime]. [Wasmtime]: https://github.com/bytecodealliance/wasmtime -[Cranelift IR]: https://github.com/bytecodealliance/wasmtime/blob/master/cranelift/docs/ir.md +[Cranelift IR]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/docs/ir.md diff --git a/third_party/rust/cranelift-wasm/src/code_translator.rs b/third_party/rust/cranelift-wasm/src/code_translator.rs index 20c6e3af8d5d..023961035b6e 100644 --- a/third_party/rust/cranelift-wasm/src/code_translator.rs +++ b/third_party/rust/cranelift-wasm/src/code_translator.rs @@ -533,7 +533,7 @@ pub fn translate_operator( // `index` is the index of the function's signature and `table_index` is the index of // the table to search the function in. let (sigref, num_args) = state.get_indirect_sig(builder.func, *index, environ)?; - let table = state.get_table(builder.func, *table_index, environ)?; + let table = state.get_or_create_table(builder.func, *table_index, environ)?; let callee = state.pop1(); // Bitcast any vector arguments to their default type, I8X16, before calling. @@ -1039,15 +1039,14 @@ pub fn translate_operator( Operator::F32Le | Operator::F64Le => { translate_fcmp(FloatCC::LessThanOrEqual, builder, state) } - Operator::RefNull { ty: _ } => state.push1(builder.ins().null(environ.reference_type())), + Operator::RefNull { ty } => state.push1(environ.translate_ref_null(builder.cursor(), *ty)?), Operator::RefIsNull { ty: _ } => { - let arg = state.pop1(); - let val = builder.ins().is_null(arg); - let val_int = builder.ins().bint(I32, val); - state.push1(val_int); + let value = state.pop1(); + state.push1(environ.translate_ref_is_null(builder.cursor(), value)?); } Operator::RefFunc { function_index } => { - state.push1(environ.translate_ref_func(builder.cursor(), *function_index)?); + let index = FuncIndex::from_u32(*function_index); + state.push1(environ.translate_ref_func(builder.cursor(), index)?); } Operator::AtomicNotify { .. } | Operator::I32AtomicWait { .. } @@ -1163,41 +1162,45 @@ pub fn translate_operator( environ.translate_data_drop(builder.cursor(), *segment)?; } Operator::TableSize { table: index } => { - let table = state.get_table(builder.func, *index, environ)?; + let table = state.get_or_create_table(builder.func, *index, environ)?; state.push1(environ.translate_table_size( builder.cursor(), TableIndex::from_u32(*index), table, )?); } - Operator::TableGrow { table } => { - let table_index = TableIndex::from_u32(*table); + Operator::TableGrow { table: index } => { + let table_index = TableIndex::from_u32(*index); + let table = state.get_or_create_table(builder.func, *index, environ)?; let delta = state.pop1(); let init_value = state.pop1(); state.push1(environ.translate_table_grow( builder.cursor(), table_index, + table, delta, init_value, )?); } - Operator::TableGet { table } => { - let table_index = TableIndex::from_u32(*table); + Operator::TableGet { table: index } => { + let table_index = TableIndex::from_u32(*index); + let table = state.get_or_create_table(builder.func, *index, environ)?; let index = state.pop1(); - state.push1(environ.translate_table_get(builder.cursor(), table_index, index)?); + state.push1(environ.translate_table_get(builder, table_index, table, index)?); } - Operator::TableSet { table } => { - let table_index = TableIndex::from_u32(*table); + Operator::TableSet { table: index } => { + let table_index = TableIndex::from_u32(*index); + let table = state.get_or_create_table(builder.func, *index, environ)?; let value = state.pop1(); let index = state.pop1(); - environ.translate_table_set(builder.cursor(), table_index, value, index)?; + environ.translate_table_set(builder, table_index, table, value, index)?; } Operator::TableCopy { dst_table: dst_table_index, src_table: src_table_index, } => { - let dst_table = state.get_table(builder.func, *dst_table_index, environ)?; - let src_table = state.get_table(builder.func, *src_table_index, environ)?; + let dst_table = state.get_or_create_table(builder.func, *dst_table_index, environ)?; + let src_table = state.get_or_create_table(builder.func, *src_table_index, environ)?; let len = state.pop1(); let src = state.pop1(); let dest = state.pop1(); @@ -1223,7 +1226,7 @@ pub fn translate_operator( segment, table: table_index, } => { - let table = state.get_table(builder.func, *table_index, environ)?; + let table = state.get_or_create_table(builder.func, *table_index, environ)?; let len = state.pop1(); let src = state.pop1(); let dest = state.pop1(); @@ -1383,6 +1386,10 @@ pub fn translate_operator( let a = pop1_with_bitcast(state, type_of(op), builder); state.push1(builder.ins().ineg(a)) } + Operator::I8x16Abs | Operator::I16x8Abs | Operator::I32x4Abs => { + let a = pop1_with_bitcast(state, type_of(op), builder); + state.push1(builder.ins().iabs(a)) + } Operator::I16x8Mul | Operator::I32x4Mul | Operator::I64x2Mul => { let (a, b) = pop2_with_bitcast(state, type_of(op), builder); state.push1(builder.ins().imul(a, b)) @@ -1548,11 +1555,11 @@ pub fn translate_operator( let a = pop1_with_bitcast(state, I32X4, builder); state.push1(builder.ins().fcvt_from_uint(F32X4, a)) } - Operator::I32x4TruncSatF32x4S - | Operator::I32x4TruncSatF32x4U - | Operator::I8x16Abs - | Operator::I16x8Abs - | Operator::I32x4Abs + Operator::I32x4TruncSatF32x4S => { + let a = pop1_with_bitcast(state, F32X4, builder); + state.push1(builder.ins().fcvt_to_sint_sat(I32X4, a)) + } + Operator::I32x4TruncSatF32x4U | Operator::I8x16NarrowI16x8S { .. } | Operator::I8x16NarrowI16x8U { .. } | Operator::I16x8NarrowI32x4S { .. } @@ -1564,7 +1571,10 @@ pub fn translate_operator( | Operator::I32x4WidenLowI16x8S { .. } | Operator::I32x4WidenHighI16x8S { .. } | Operator::I32x4WidenLowI16x8U { .. } - | Operator::I32x4WidenHighI16x8U { .. } => { + | Operator::I32x4WidenHighI16x8U { .. } + | Operator::I8x16Bitmask + | Operator::I16x8Bitmask + | Operator::I32x4Bitmask => { return Err(wasm_unsupported!("proposed SIMD operator {:?}", op)); } @@ -1981,6 +1991,7 @@ fn type_of(operator: &Operator) -> Type { | Operator::I8x16GeS | Operator::I8x16GeU | Operator::I8x16Neg + | Operator::I8x16Abs | Operator::I8x16AnyTrue | Operator::I8x16AllTrue | Operator::I8x16Shl @@ -1996,7 +2007,8 @@ fn type_of(operator: &Operator) -> Type { | Operator::I8x16MinU | Operator::I8x16MaxS | Operator::I8x16MaxU - | Operator::I8x16RoundingAverageU => I8X16, + | Operator::I8x16RoundingAverageU + | Operator::I8x16Bitmask => I8X16, Operator::I16x8Splat | Operator::V16x8LoadSplat { .. } @@ -2014,6 +2026,7 @@ fn type_of(operator: &Operator) -> Type { | Operator::I16x8GeS | Operator::I16x8GeU | Operator::I16x8Neg + | Operator::I16x8Abs | Operator::I16x8AnyTrue | Operator::I16x8AllTrue | Operator::I16x8Shl @@ -2030,7 +2043,8 @@ fn type_of(operator: &Operator) -> Type { | Operator::I16x8MaxS | Operator::I16x8MaxU | Operator::I16x8RoundingAverageU - | Operator::I16x8Mul => I16X8, + | Operator::I16x8Mul + | Operator::I16x8Bitmask => I16X8, Operator::I32x4Splat | Operator::V32x4LoadSplat { .. } @@ -2047,6 +2061,7 @@ fn type_of(operator: &Operator) -> Type { | Operator::I32x4GeS | Operator::I32x4GeU | Operator::I32x4Neg + | Operator::I32x4Abs | Operator::I32x4AnyTrue | Operator::I32x4AllTrue | Operator::I32x4Shl @@ -2060,7 +2075,8 @@ fn type_of(operator: &Operator) -> Type { | Operator::I32x4MaxS | Operator::I32x4MaxU | Operator::F32x4ConvertI32x4S - | Operator::F32x4ConvertI32x4U => I32X4, + | Operator::F32x4ConvertI32x4U + | Operator::I32x4Bitmask => I32X4, Operator::I64x2Splat | Operator::V64x2LoadSplat { .. } diff --git a/third_party/rust/cranelift-wasm/src/environ/dummy.rs b/third_party/rust/cranelift-wasm/src/environ/dummy.rs index 671414bcb6b8..25f5c3a5929e 100644 --- a/third_party/rust/cranelift-wasm/src/environ/dummy.rs +++ b/third_party/rust/cranelift-wasm/src/environ/dummy.rs @@ -22,6 +22,7 @@ use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{self, InstBuilder}; use cranelift_codegen::isa::TargetFrontendConfig; use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap}; +use cranelift_frontend::FunctionBuilder; use std::boxed::Box; use std::string::String; use std::vec::Vec; @@ -197,6 +198,14 @@ impl<'dummy_environment> DummyFuncEnvironment<'dummy_environment> { )); sig } + + fn reference_type(&self) -> ir::Type { + match self.pointer_type() { + ir::types::I32 => ir::types::R32, + ir::types::I64 => ir::types::R64, + _ => panic!("unsupported pointer type"), + } + } } impl<'dummy_environment> TargetEnvironment for DummyFuncEnvironment<'dummy_environment> { @@ -435,6 +444,7 @@ impl<'dummy_environment> FuncEnvironment for DummyFuncEnvironment<'dummy_environ &mut self, mut pos: FuncCursor, _table_index: TableIndex, + _table: ir::Table, _delta: ir::Value, _init_value: ir::Value, ) -> WasmResult { @@ -443,17 +453,19 @@ impl<'dummy_environment> FuncEnvironment for DummyFuncEnvironment<'dummy_environ fn translate_table_get( &mut self, - mut pos: FuncCursor, + builder: &mut FunctionBuilder, _table_index: TableIndex, + _table: ir::Table, _index: ir::Value, ) -> WasmResult { - Ok(pos.ins().null(self.reference_type())) + Ok(builder.ins().null(self.reference_type())) } fn translate_table_set( &mut self, - _pos: FuncCursor, + _builder: &mut FunctionBuilder, _table_index: TableIndex, + _table: ir::Table, _value: ir::Value, _index: ir::Value, ) -> WasmResult<()> { @@ -505,7 +517,7 @@ impl<'dummy_environment> FuncEnvironment for DummyFuncEnvironment<'dummy_environ fn translate_ref_func( &mut self, mut pos: FuncCursor, - _func_index: u32, + _func_index: FuncIndex, ) -> WasmResult { Ok(pos.ins().null(self.reference_type())) } diff --git a/third_party/rust/cranelift-wasm/src/environ/spec.rs b/third_party/rust/cranelift-wasm/src/environ/spec.rs index 0473ad3c5a73..eacc90233f33 100644 --- a/third_party/rust/cranelift-wasm/src/environ/spec.rs +++ b/third_party/rust/cranelift-wasm/src/environ/spec.rs @@ -78,7 +78,7 @@ pub enum WasmError { /// Cranelift can compile very large and complicated functions, but the [implementation has /// limits][limits] that cause compilation to fail when they are exceeded. /// - /// [limits]: https://github.com/bytecodealliance/wasmtime/blob/master/cranelift/docs/ir.md#implementation-limits + /// [limits]: https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/docs/ir.md#implementation-limits #[error("Implementation limit exceeded")] ImplLimitExceeded, @@ -133,10 +133,15 @@ pub trait TargetEnvironment { self.target_config().pointer_bytes() } - /// Get the Cranelift reference type to use for native references. + /// Get the Cranelift reference type to use for the given Wasm reference + /// type. /// - /// This returns `R64` for 64-bit architectures and `R32` for 32-bit architectures. - fn reference_type(&self) -> ir::Type { + /// By default, this returns `R64` for 64-bit architectures and `R32` for + /// 32-bit architectures. If you override this, then you should also + /// override `FuncEnvironment::{translate_ref_null, translate_ref_is_null}` + /// as well. + fn reference_type(&self, ty: WasmType) -> ir::Type { + let _ = ty; match self.pointer_type() { ir::types::I32 => ir::types::R32, ir::types::I64 => ir::types::R64, @@ -355,6 +360,7 @@ pub trait FuncEnvironment: TargetEnvironment { &mut self, pos: FuncCursor, table_index: TableIndex, + table: ir::Table, delta: ir::Value, init_value: ir::Value, ) -> WasmResult; @@ -362,16 +368,18 @@ pub trait FuncEnvironment: TargetEnvironment { /// Translate a `table.get` WebAssembly instruction. fn translate_table_get( &mut self, - pos: FuncCursor, + builder: &mut FunctionBuilder, table_index: TableIndex, + table: ir::Table, index: ir::Value, ) -> WasmResult; /// Translate a `table.set` WebAssembly instruction. fn translate_table_set( &mut self, - pos: FuncCursor, + builder: &mut FunctionBuilder, table_index: TableIndex, + table: ir::Table, value: ir::Value, index: ir::Value, ) -> WasmResult<()>; @@ -416,8 +424,43 @@ pub trait FuncEnvironment: TargetEnvironment { /// Translate a `elem.drop` WebAssembly instruction. fn translate_elem_drop(&mut self, pos: FuncCursor, seg_index: u32) -> WasmResult<()>; + /// Translate a `ref.null T` WebAssembly instruction. + /// + /// By default, translates into a null reference type. + /// + /// Override this if you don't use Cranelift reference types for all Wasm + /// reference types (e.g. you use a raw pointer for `funcref`s) or if the + /// null sentinel is not a null reference type pointer for your type. If you + /// override this method, then you should also override + /// `translate_ref_is_null` as well. + fn translate_ref_null(&mut self, mut pos: FuncCursor, ty: WasmType) -> WasmResult { + let _ = ty; + Ok(pos.ins().null(self.reference_type(ty))) + } + + /// Translate a `ref.is_null` WebAssembly instruction. + /// + /// By default, assumes that `value` is a Cranelift reference type, and that + /// a null Cranelift reference type is the null value for all Wasm reference + /// types. + /// + /// If you override this method, you probably also want to override + /// `translate_ref_null` as well. + fn translate_ref_is_null( + &mut self, + mut pos: FuncCursor, + value: ir::Value, + ) -> WasmResult { + let is_null = pos.ins().is_null(value); + Ok(pos.ins().bint(ir::types::I32, is_null)) + } + /// Translate a `ref.func` WebAssembly instruction. - fn translate_ref_func(&mut self, pos: FuncCursor, func_index: u32) -> WasmResult; + fn translate_ref_func( + &mut self, + pos: FuncCursor, + func_index: FuncIndex, + ) -> WasmResult; /// Translate a `global.get` WebAssembly instruction at `pos` for a global /// that is custom. diff --git a/third_party/rust/cranelift-wasm/src/func_translator.rs b/third_party/rust/cranelift-wasm/src/func_translator.rs index c446dd948855..601996d6df11 100644 --- a/third_party/rust/cranelift-wasm/src/func_translator.rs +++ b/third_party/rust/cranelift-wasm/src/func_translator.rs @@ -196,8 +196,7 @@ fn declare_locals( let constant_handle = builder.func.dfg.constants.insert([0; 16].to_vec().into()); builder.ins().vconst(ir::types::I8X16, constant_handle) } - ExternRef => builder.ins().null(environ.reference_type()), - FuncRef => builder.ins().null(environ.reference_type()), + ExternRef | FuncRef => environ.translate_ref_null(builder.cursor(), wasm_type)?, ty => return Err(wasm_unsupported!("unsupported local type {:?}", ty)), }; diff --git a/third_party/rust/cranelift-wasm/src/lib.rs b/third_party/rust/cranelift-wasm/src/lib.rs index b3836fd251ba..661bfe8dd4df 100644 --- a/third_party/rust/cranelift-wasm/src/lib.rs +++ b/third_party/rust/cranelift-wasm/src/lib.rs @@ -70,6 +70,7 @@ pub use crate::translation_utils::{ DefinedTableIndex, ElemIndex, FuncIndex, Global, GlobalIndex, GlobalInit, Memory, MemoryIndex, SignatureIndex, Table, TableElementType, TableIndex, }; +pub use cranelift_frontend::FunctionBuilder; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/third_party/rust/cranelift-wasm/src/module_translator.rs b/third_party/rust/cranelift-wasm/src/module_translator.rs index e31e6f09f851..d34a2c15575e 100644 --- a/third_party/rust/cranelift-wasm/src/module_translator.rs +++ b/third_party/rust/cranelift-wasm/src/module_translator.rs @@ -71,6 +71,11 @@ pub fn translate_module<'data>( environ.reserve_passive_data(count)?; } + SectionContent::Module(_) + | SectionContent::ModuleCode(_) + | SectionContent::Instance(_) + | SectionContent::Alias(_) => unimplemented!("module linking not implemented yet"), + SectionContent::Custom { name, binary, diff --git a/third_party/rust/cranelift-wasm/src/sections_translator.rs b/third_party/rust/cranelift-wasm/src/sections_translator.rs index 967612178a3c..7075fbc19bf8 100644 --- a/third_party/rust/cranelift-wasm/src/sections_translator.rs +++ b/third_party/rust/cranelift-wasm/src/sections_translator.rs @@ -26,7 +26,7 @@ use wasmparser::{ ElementKind, ElementSectionReader, Export, ExportSectionReader, ExternalKind, FunctionSectionReader, GlobalSectionReader, GlobalType, ImportSectionEntryType, ImportSectionReader, MemorySectionReader, MemoryType, NameSectionReader, Naming, NamingReader, - Operator, TableSectionReader, Type, TypeSectionReader, + Operator, TableSectionReader, Type, TypeDef, TypeSectionReader, }; /// Parses the Type section of the wasm module. @@ -40,22 +40,26 @@ pub fn parse_type_section( environ.reserve_signatures(count)?; for entry in types { - let wasm_func_ty = entry?; - let mut sig = Signature::new(ModuleEnvironment::target_config(environ).default_call_conv); - sig.params.extend(wasm_func_ty.params.iter().map(|ty| { - let cret_arg: ir::Type = type_to_type(*ty, environ) - .expect("only numeric types are supported in function signatures"); - AbiParam::new(cret_arg) - })); - sig.returns.extend(wasm_func_ty.returns.iter().map(|ty| { - let cret_arg: ir::Type = type_to_type(*ty, environ) - .expect("only numeric types are supported in function signatures"); - AbiParam::new(cret_arg) - })); - environ.declare_signature(&wasm_func_ty, sig)?; - module_translation_state - .wasm_types - .push((wasm_func_ty.params, wasm_func_ty.returns)); + if let Ok(TypeDef::Func(wasm_func_ty)) = entry { + let mut sig = + Signature::new(ModuleEnvironment::target_config(environ).default_call_conv); + sig.params.extend(wasm_func_ty.params.iter().map(|ty| { + let cret_arg: ir::Type = type_to_type(*ty, environ) + .expect("only numeric types are supported in function signatures"); + AbiParam::new(cret_arg) + })); + sig.returns.extend(wasm_func_ty.returns.iter().map(|ty| { + let cret_arg: ir::Type = type_to_type(*ty, environ) + .expect("only numeric types are supported in function signatures"); + AbiParam::new(cret_arg) + })); + environ.declare_signature(&wasm_func_ty, sig)?; + module_translation_state + .wasm_types + .push((wasm_func_ty.params, wasm_func_ty.returns)); + } else { + unimplemented!("module linking not implemented yet") + } } Ok(()) } @@ -70,7 +74,7 @@ pub fn parse_import_section<'data>( for entry in imports { let import = entry?; let module_name = import.module; - let field_name = import.field; + let field_name = import.field.unwrap(); // TODO Handle error when module linking is implemented. match import.ty { ImportSectionEntryType::Function(sig) => { @@ -80,6 +84,9 @@ pub fn parse_import_section<'data>( field_name, )?; } + ImportSectionEntryType::Module(_sig) | ImportSectionEntryType::Instance(_sig) => { + unimplemented!("module linking not implemented yet") + } ImportSectionEntryType::Memory(MemoryType { limits: ref memlimits, shared, @@ -97,6 +104,7 @@ pub fn parse_import_section<'data>( ImportSectionEntryType::Global(ref ty) => { environ.declare_global_import( Global { + wasm_ty: ty.content_type, ty: type_to_type(ty.content_type, environ).unwrap(), mutability: ty.mutable, initializer: GlobalInit::Import, @@ -108,6 +116,7 @@ pub fn parse_import_section<'data>( ImportSectionEntryType::Table(ref tab) => { environ.declare_table_import( Table { + wasm_ty: tab.element_type, ty: match tabletype_to_type(tab.element_type, environ)? { Some(t) => TableElementType::Val(t), None => TableElementType::Func, @@ -157,6 +166,7 @@ pub fn parse_table_section( for entry in tables { let table = entry?; environ.declare_table(Table { + wasm_ty: table.element_type, ty: match tabletype_to_type(table.element_type, environ)? { Some(t) => TableElementType::Val(t), None => TableElementType::Func, @@ -227,6 +237,7 @@ pub fn parse_global_section( } }; let global = Global { + wasm_ty: content_type, ty: type_to_type(content_type, environ).unwrap(), mutability: mutable, initializer, @@ -264,6 +275,9 @@ pub fn parse_export_section<'data>( ExternalKind::Global => { environ.declare_global_export(GlobalIndex::new(index), field)? } + ExternalKind::Type | ExternalKind::Module | ExternalKind::Instance => { + unimplemented!("module linking not implemented yet") + } } } @@ -335,7 +349,9 @@ pub fn parse_element_section<'data>( let index = ElemIndex::from_u32(index as u32); environ.declare_passive_element(index, segments)?; } - ElementKind::Declared => return Err(wasm_unsupported!("element kind declared")), + ElementKind::Declared => { + // Nothing to do here. + } } } Ok(()) diff --git a/third_party/rust/cranelift-wasm/src/state/func_state.rs b/third_party/rust/cranelift-wasm/src/state/func_state.rs index 81a0d35e9741..24ef5ff0cba6 100644 --- a/third_party/rust/cranelift-wasm/src/state/func_state.rs +++ b/third_party/rust/cranelift-wasm/src/state/func_state.rs @@ -202,7 +202,7 @@ pub struct FuncTranslationState { heaps: HashMap, // Map of tables that have been created by `FuncEnvironment::make_table`. - tables: HashMap, + pub(crate) tables: HashMap, // Map of indirect call signatures that have been created by // `FuncEnvironment::make_indirect_sig()`. @@ -446,7 +446,7 @@ impl FuncTranslationState { /// Get the `Table` reference that should be used to access table `index`. /// Create the reference if necessary. - pub(crate) fn get_table( + pub(crate) fn get_or_create_table( &mut self, func: &mut ir::Function, index: u32, diff --git a/third_party/rust/cranelift-wasm/src/translation_utils.rs b/third_party/rust/cranelift-wasm/src/translation_utils.rs index 19b05f760c78..4bbab4e75ab4 100644 --- a/third_party/rust/cranelift-wasm/src/translation_utils.rs +++ b/third_party/rust/cranelift-wasm/src/translation_utils.rs @@ -1,5 +1,5 @@ //! Helper functions and structures for the translation. -use crate::environ::{TargetEnvironment, WasmResult}; +use crate::environ::{TargetEnvironment, WasmResult, WasmType}; use crate::state::ModuleTranslationState; use crate::wasm_unsupported; use core::u32; @@ -67,10 +67,18 @@ entity_impl!(DataIndex); pub struct ElemIndex(u32); entity_impl!(ElemIndex); -/// WebAssembly global. +/// A WebAssembly global. +/// +/// Note that we record both the original Wasm type and the Cranelift IR type +/// used to represent it. This is because multiple different kinds of Wasm types +/// might be represented with the same Cranelift IR type. For example, both a +/// Wasm `i64` and a `funcref` might be represented with a Cranelift `i64` on +/// 64-bit architectures, and when GC is not required for func refs. #[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] pub struct Global { - /// The type of the value stored in the global. + /// The Wasm type of the value stored in the global. + pub wasm_ty: crate::WasmType, + /// The Cranelift IR type of the value stored in the global. pub ty: ir::Type, /// A flag indicating whether the value may change at runtime. pub mutability: bool, @@ -104,7 +112,9 @@ pub enum GlobalInit { /// WebAssembly table. #[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] pub struct Table { - /// The type of data stored in elements of the table. + /// The table elements' Wasm type. + pub wasm_ty: WasmType, + /// The table elements' Cranelift type. pub ty: TableElementType, /// The minimum number of elements in the table. pub minimum: u32, @@ -143,7 +153,7 @@ pub fn type_to_type( wasmparser::Type::F32 => Ok(ir::types::F32), wasmparser::Type::F64 => Ok(ir::types::F64), wasmparser::Type::V128 => Ok(ir::types::I8X16), - wasmparser::Type::ExternRef | wasmparser::Type::FuncRef => Ok(environ.reference_type()), + wasmparser::Type::ExternRef | wasmparser::Type::FuncRef => Ok(environ.reference_type(ty)), ty => Err(wasm_unsupported!("type_to_type: wasm type {:?}", ty)), } } @@ -160,7 +170,7 @@ pub fn tabletype_to_type( wasmparser::Type::F32 => Ok(Some(ir::types::F32)), wasmparser::Type::F64 => Ok(Some(ir::types::F64)), wasmparser::Type::V128 => Ok(Some(ir::types::I8X16)), - wasmparser::Type::ExternRef => Ok(Some(environ.reference_type())), + wasmparser::Type::ExternRef => Ok(Some(environ.reference_type(ty))), wasmparser::Type::FuncRef => Ok(None), ty => Err(wasm_unsupported!( "tabletype_to_type: table wasm type {:?}", @@ -216,7 +226,7 @@ pub fn block_with_params( builder.append_block_param(block, ir::types::F64); } wasmparser::Type::ExternRef | wasmparser::Type::FuncRef => { - builder.append_block_param(block, environ.reference_type()); + builder.append_block_param(block, environ.reference_type(*ty)); } wasmparser::Type::V128 => { builder.append_block_param(block, ir::types::I8X16); diff --git a/third_party/rust/wasmparser/.cargo-checksum.json b/third_party/rust/wasmparser/.cargo-checksum.json index b609eba475d2..e51609f8e197 100644 --- a/third_party/rust/wasmparser/.cargo-checksum.json +++ b/third_party/rust/wasmparser/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.lock":"f0adbcf79ba9aa2b13594b7b7e712884b8261151380659501abc2c2f7d5e9be0","Cargo.toml":"a5fbf3dde37713f296bb466ed72421400adef4947a8a61aa0cb9c52e5f0e51c7","README.md":"2e252886759b5ee5137ec39efc0765850be2cb4242c68e6b44452b75d3191db3","benches/benchmark.rs":"5ed64bf1497769a8df2a6518d06c7ca8dfaef328869ffa4fef43414035a07255","compare-master.sh":"165490eab36ef4eceb2913a6c5cdeff479a05e1e0119a7f4551b03dbcda51ad4","examples/dump.rs":"40cf9492d58e196e462c37d610fd372a648df780143d1804ae4c536f03ac5254","examples/simple.rs":"d2d46f1a232e9b23fd56982a84379b741423916983e0fd1f2a1009d456f7f851","src/binary_reader.rs":"83b0d2c7eb3fa23e0e3ea474586a9437678eb89b18682cd9a8824e678b75fc45","src/lib.rs":"8bb5301d5d66746160466e38b0e956e14e4997bf2385431ec9070cea13fc632e","src/limits.rs":"34e5cda95fb67669011ba95ca60f48fc777f3e3fa279ff68a1f2a072032a4abd","src/module_resources.rs":"f2f3e2a00ca74df1f6218ecf0c3d9ed36529675febba3e8a825585514658480f","src/operators_validator.rs":"e0085694bbde6d6c9efa853cc6d81ba86be39a7f41bf0ccff38ddd1dbe0898ff","src/parser.rs":"4b18ae92d0c6f85581bf77896f457b34f47c75cbbf422cdd4edac48c5f48c0cb","src/primitives.rs":"3017e05e7379c98e99de7e0618a8dbd59f37dc3f7e6da5e42bf3c4ca56ac7457","src/readers/code_section.rs":"bfdd8d5f08ef357679d7bfe6f9735ff4f08925361e0771a6b1b5112a12c62f30","src/readers/data_count_section.rs":"e711720f8205a906794dc7020a656a2ae74e1d9c3823fcdcdbd9d2f3b206c7d7","src/readers/data_section.rs":"f572e7d2589f0bccf5e97d43c1ca3aac103cbd47d139ead6b84b39b5c9d47c0b","src/readers/element_section.rs":"0193c9b7be80a0c18cba9f2d2892dba83819339aaf39a44d44003fec5328196c","src/readers/export_section.rs":"7c74f7a11406a95c162f6ad4f77aafd0b1eee309f33b69f06bea12b23925e143","src/readers/function_section.rs":"57c0479ba8d7f61908ed74e86cbc26553fdd6d2d952f032ce29385a39f82efd3","src/readers/global_section.rs":"5fa18bed0fffadcc2dbdcbaedbe4e4398992fd1ce9e611b0319333a7681082ac","src/readers/import_section.rs":"1db4bf7290d04783d5cf526050d025b15a1daaf2bd97fca1a92ecb873d48f641","src/readers/init_expr.rs":"7020c80013dad4518a5f969c3ab4d624b46d778f03e632871cf343964f63441c","src/readers/linking_section.rs":"9df71f3ee5356f0d273c099212213353080001e261ca697caddf6b847fb5af09","src/readers/memory_section.rs":"83212f86cfc40d18fb392e9234c880afdf443f4af38a727ba346f9c740ef8718","src/readers/mod.rs":"b9f835365b9b04411d7b141a3c9b52695e9bf8ef1f07094a10a18b901d0ac420","src/readers/module.rs":"db292e3cebe55e5f2e9de8aff0a2074fa874d42058c6bc2a798c5b7e3c1ca81e","src/readers/name_section.rs":"4ff460766bbcd67f658086c8fa525cf2bbceea67b393c65edfddbb714de722fd","src/readers/operators.rs":"1defc15f364775018ffe8c7f010ff83342c46659f780be4ba88c58fad7606e03","src/readers/producers_section.rs":"674f402fc4545c94487f827153871b37adab44ed5eff4070a436eb18e514023a","src/readers/reloc_section.rs":"0ef818a8b83a4542c4c29c23642436a92d3e7c37bc0248e817ed5a9d65ec38ce","src/readers/section_reader.rs":"f27f017938bb8602954298d053cd3b79d8876f9fcbbe0e1a3380051b6aa4584a","src/readers/sourcemappingurl_section.rs":"eff317f6f2b728a98a5eb68eec7e6cf222d27158d0d5597fd1c84f09b1092a50","src/readers/start_section.rs":"012fe574a5b94ea34c9d689629fb0df2f5ba4c11c835147b39155f5a8c715e34","src/readers/table_section.rs":"e564876825a7b31df2b5dc850279b523e26dc50a08da935cc8d635a49e809951","src/readers/type_section.rs":"2fa33a7b793f3bfa01c259b5dbc38633b7343931886ab41f0cb96dd78db3bf6e","src/tests.rs":"5d47ec97d0a303d8cbe905f8ddcf11212a03607e0b245c9f52371464e7d08ee7","src/validator.rs":"b14b45f03e99789d3b08d9c60a5d250904af776fa68fb0a39d400f5fa659ffc9"},"package":"32fddd575d477c6e9702484139cf9f23dcd554b06d185ed0f56c857dd3a47aa6"} \ No newline at end of file +{"files":{"Cargo.lock":"66295ad9f17449e9ef5c16b64c9f0fca138ff07e31fb182bdd134099a7d049b4","Cargo.toml":"ddff8c2657f4fd0f83ce3b732cea03b8eb1f434fdce886fba2904cee5b0090d5","README.md":"2e252886759b5ee5137ec39efc0765850be2cb4242c68e6b44452b75d3191db3","benches/benchmark.rs":"a50793192bdc1729a786bb456e5ad1e567c7f4b6a0a13ab0e46754e965978e8f","compare-with-main.sh":"2ddfab71ba571055292a29a36c1ede05f64ba094c78495b945d1486bf32ab6d7","examples/dump.rs":"a5944669754d1093c048a3b2e959c8c22e485a8069582d532172a8207e54dce6","examples/simple.rs":"0bbf762ca214815d81a915106efca05a9fa642a7a250c704c188258ec15d2629","src/binary_reader.rs":"d209e8cf15db30cb06e4c23980de775a59db8654aeb7a69bbe432c09f5046f76","src/lib.rs":"62c4b60aae7b7c5018caf68da31f929956f188042fa0715781e6879148f79db1","src/limits.rs":"22649a707c3f894d0381d745f744876a106cacb72d8a9a608cfa7a6f3f1e5631","src/module_resources.rs":"3a2137adb9018a5d5ebcaf274f969e650e184f77e5db62cd9b655cc6e97fdee1","src/operators_validator.rs":"4d98039d738be26670f7fb399e0738dde6caa170c09139badb62190795c78593","src/parser.rs":"061ba728cbf044456c088255c4c633d5bcc630fe9035a21168f068e498e8255c","src/primitives.rs":"c5056a6f6f444cdd4b45d2b7bb332b776088d7b5bc323e3daddeb48110025b25","src/readers/alias_section.rs":"fa64491870d97577bad7f1891aab1be7597a940bc0e2ccfef0c84847e9df3d6d","src/readers/code_section.rs":"bfdd8d5f08ef357679d7bfe6f9735ff4f08925361e0771a6b1b5112a12c62f30","src/readers/data_count_section.rs":"e711720f8205a906794dc7020a656a2ae74e1d9c3823fcdcdbd9d2f3b206c7d7","src/readers/data_section.rs":"f572e7d2589f0bccf5e97d43c1ca3aac103cbd47d139ead6b84b39b5c9d47c0b","src/readers/element_section.rs":"0193c9b7be80a0c18cba9f2d2892dba83819339aaf39a44d44003fec5328196c","src/readers/export_section.rs":"7c74f7a11406a95c162f6ad4f77aafd0b1eee309f33b69f06bea12b23925e143","src/readers/function_section.rs":"57c0479ba8d7f61908ed74e86cbc26553fdd6d2d952f032ce29385a39f82efd3","src/readers/global_section.rs":"5fa18bed0fffadcc2dbdcbaedbe4e4398992fd1ce9e611b0319333a7681082ac","src/readers/import_section.rs":"236e754867ad7829b5a95221051daff3c5df971aff9f2339fa11256f2309d209","src/readers/init_expr.rs":"7020c80013dad4518a5f969c3ab4d624b46d778f03e632871cf343964f63441c","src/readers/instance_section.rs":"7b78bbca4b79ac7f9c42455815863a4d32dc074c5973ab1035dbfdf88b0d3c12","src/readers/linking_section.rs":"9df71f3ee5356f0d273c099212213353080001e261ca697caddf6b847fb5af09","src/readers/memory_section.rs":"83212f86cfc40d18fb392e9234c880afdf443f4af38a727ba346f9c740ef8718","src/readers/mod.rs":"d80ba76d763b06ae6e570f09a312b20006f0b83b5cd01d6baab496006fe9b7f1","src/readers/module.rs":"04f6e0bb7250f86037f30754d95a2941623272002184f372ed159db19f52dc7e","src/readers/module_code_section.rs":"aa9cf64f65e43ea5fcf9e695b7b1ba5a45b92d537f7ccef379a07162108ce9d9","src/readers/module_section.rs":"7a0c0b34478ec32030c5400df2366914c7e08ba799440a8c5ea999755c489e7f","src/readers/name_section.rs":"23b5106b17744833fb8cd61cb102e756bccb4a44594d34a5dd8b7930307ac4cb","src/readers/operators.rs":"1defc15f364775018ffe8c7f010ff83342c46659f780be4ba88c58fad7606e03","src/readers/producers_section.rs":"674f402fc4545c94487f827153871b37adab44ed5eff4070a436eb18e514023a","src/readers/reloc_section.rs":"0ef818a8b83a4542c4c29c23642436a92d3e7c37bc0248e817ed5a9d65ec38ce","src/readers/section_reader.rs":"f27f017938bb8602954298d053cd3b79d8876f9fcbbe0e1a3380051b6aa4584a","src/readers/sourcemappingurl_section.rs":"eff317f6f2b728a98a5eb68eec7e6cf222d27158d0d5597fd1c84f09b1092a50","src/readers/start_section.rs":"012fe574a5b94ea34c9d689629fb0df2f5ba4c11c835147b39155f5a8c715e34","src/readers/table_section.rs":"e564876825a7b31df2b5dc850279b523e26dc50a08da935cc8d635a49e809951","src/readers/type_section.rs":"c2f9d7b77a1315d323bebe94ced44dc10b77c0e75c1e367bb594a402c74933ba","src/tests.rs":"5d47ec97d0a303d8cbe905f8ddcf11212a03607e0b245c9f52371464e7d08ee7","src/validator.rs":"bec65fde1d8b98d80d082067a6ccf006f35e3f06062cac97887bd5a04ef75192"},"package":"721a8d79483738d7aef6397edcf8f04cd862640b1ad5973adf5bb50fc10e86db"} \ No newline at end of file diff --git a/third_party/rust/wasmparser/Cargo.lock b/third_party/rust/wasmparser/Cargo.lock index 076eefe06a3f..0f06e8ac67e0 100644 --- a/third_party/rust/wasmparser/Cargo.lock +++ b/third_party/rust/wasmparser/Cargo.lock @@ -43,9 +43,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5356f1d23ee24a1f785a56d1d1a5f0fd5b0f6a0c0fb2412ce11da71649ab78f6" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" [[package]] name = "byteorder" @@ -81,9 +81,9 @@ dependencies = [ [[package]] name = "criterion" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63f696897c88b57f4ffe3c69d8e1a0613c7d0e6c4833363c8560fbde9c47b966" +checksum = "70daa7ceec6cf143990669a04c7df13391d55fb27bd4079d252fca774ba244d8" dependencies = [ "atty", "cast", @@ -98,6 +98,7 @@ dependencies = [ "rayon", "regex", "serde", + "serde_cbor", "serde_derive", "serde_json", "tinytemplate", @@ -106,9 +107,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddeaf7989f00f2e1d871a26a110f3ed713632feac17f65f03ca938c542618b60" +checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d" dependencies = [ "cast", "itertools", @@ -142,12 +143,13 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ "cfg-if", "crossbeam-utils", + "maybe-uninit", ] [[package]] @@ -199,10 +201,16 @@ dependencies = [ ] [[package]] -name = "hermit-abi" -version = "0.1.13" +name = "half" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" +checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" + +[[package]] +name = "hermit-abi" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" dependencies = [ "libc", ] @@ -218,15 +226,15 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.39" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa5a448de267e7358beaf4a5d849518fe9a0c13fce7afd44b06e68550e5562a7" +checksum = "c4b9172132a62451e56142bff9afc91c8e4a4500aa5b847da36815b63bfda916" dependencies = [ "wasm-bindgen", ] @@ -239,9 +247,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.70" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f" +checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" [[package]] name = "log" @@ -275,9 +283,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" dependencies = [ "autocfg", ] @@ -294,15 +302,15 @@ dependencies = [ [[package]] name = "oorandom" -version = "11.1.1" +version = "11.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94af325bc33c7f60191be4e2c984d48aaa21e2854f473b85398344b60c9b6358" +checksum = "a170cebd8021a008ea92e4db85a72f80b35df514ec664b296fdcbb654eac0b2c" [[package]] name = "plotters" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9b1d9ca091d370ea3a78d5619145d1b59426ab0c9eedbad2514a4cee08bf389" +checksum = "0d1685fbe7beba33de0330629da9d955ac75bd54f33d7b79f9a895590124f6bb" dependencies = [ "js-sys", "num-traits", @@ -312,28 +320,29 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a50b9351bfa8d65a7d93ce712dc63d2fd15ddbf2c36990fc7cac344859c04f" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" dependencies = [ "unicode-xid", ] [[package]] name = "quote" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a21852a652ad6f610c9510194f398ff6f8692e334fd1145fed931f7fbe44ea" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ "proc-macro2", ] [[package]] name = "rayon" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098" +checksum = "62f02856753d04e03e26929f820d0a0a337ebe71f849801eea335d464b349080" dependencies = [ + "autocfg", "crossbeam-deque", "either", "rayon-core", @@ -341,9 +350,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" +checksum = "e92e15d89083484e11353891f1af602cc661426deb9564c298b270c726973280" dependencies = [ "crossbeam-deque", "crossbeam-queue", @@ -354,9 +363,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.7" +version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" dependencies = [ "regex-syntax", ] @@ -372,9 +381,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" [[package]] name = "rustc_version" @@ -387,9 +396,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "same-file" @@ -423,15 +432,25 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.110" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c" +checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" + +[[package]] +name = "serde_cbor" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +dependencies = [ + "half", + "serde", +] [[package]] name = "serde_derive" -version = "1.0.110" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" +checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" dependencies = [ "proc-macro2", "quote", @@ -440,9 +459,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.53" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" +checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" dependencies = [ "itoa", "ryu", @@ -451,9 +470,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.23" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b5f192649e48a5302a13f2feb224df883b98933222369e4b3b0fe2a5447269" +checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" dependencies = [ "proc-macro2", "quote", @@ -471,9 +490,9 @@ dependencies = [ [[package]] name = "tinytemplate" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e4bc5ac99433e0dcb8b9f309dd271a165ae37dde129b9e0ce1bfdd8bfe4891" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" dependencies = [ "serde", "serde_json", @@ -481,15 +500,15 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" [[package]] name = "walkdir" @@ -504,9 +523,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.62" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c7d40d09cdbf0f4895ae58cf57d92e1e57a9dd8ed2e8390514b54a47cc5551" +checksum = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -514,9 +533,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.62" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3972e137ebf830900db522d6c8fd74d1900dcfc733462e9a12e942b00b4ac94" +checksum = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" dependencies = [ "bumpalo", "lazy_static", @@ -529,9 +548,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.62" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd85aa2c579e8892442954685f0d801f9129de24fa2136b2c6a539c76b65776" +checksum = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -539,9 +558,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.62" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb197bd3a47553334907ffd2f16507b4f4f01bbec3ac921a7719e0decdfe72a" +checksum = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" dependencies = [ "proc-macro2", "quote", @@ -552,13 +571,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.62" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91c2916119c17a8e316507afaaa2dd94b47646048014bbdf6bef098c1bb58ad" +checksum = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" [[package]] name = "wasmparser" -version = "0.57.0" +version = "0.58.0" dependencies = [ "anyhow", "criterion", @@ -568,9 +587,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.39" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bc359e5dd3b46cb9687a051d50a2fdd228e4ba7cf6fcf861a5365c3d671a642" +checksum = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" dependencies = [ "js-sys", "wasm-bindgen", @@ -578,9 +597,9 @@ dependencies = [ [[package]] name = "winapi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", diff --git a/third_party/rust/wasmparser/Cargo.toml b/third_party/rust/wasmparser/Cargo.toml index daf565107575..1d4c28aa5b2a 100644 --- a/third_party/rust/wasmparser/Cargo.toml +++ b/third_party/rust/wasmparser/Cargo.toml @@ -13,13 +13,17 @@ [package] edition = "2018" name = "wasmparser" -version = "0.57.0" +version = "0.58.0" authors = ["Yury Delendik "] description = "A simple event-driven library for parsing WebAssembly binary files.\n" -homepage = "https://github.com/bytecodealliance/wasm-tools/tree/master/crates/wasmparser" +homepage = "https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasmparser" keywords = ["parser", "WebAssembly", "wasm"] license = "Apache-2.0 WITH LLVM-exception" -repository = "https://github.com/bytecodealliance/wasm-tools/tree/master/crates/wasmparser" +repository = "https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasmparser" + +[[bench]] +name = "benchmark" +harness = false [dev-dependencies.anyhow] version = "1.0" diff --git a/third_party/rust/wasmparser/benches/benchmark.rs b/third_party/rust/wasmparser/benches/benchmark.rs index 9f23cd7d1a07..621c8fa24dc0 100644 --- a/third_party/rust/wasmparser/benches/benchmark.rs +++ b/third_party/rust/wasmparser/benches/benchmark.rs @@ -1,9 +1,15 @@ -pub fn read_file_data(path: &PathBuf) -> Vec { - let mut data = Vec::new(); - let mut f = File::open(path).ok().unwrap(); - f.read_to_end(&mut data).unwrap(); - data -} +#[macro_use] +extern crate criterion; + +use anyhow::Result; +use criterion::{black_box, Criterion}; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use wasmparser::{ + validate, OperatorValidatorConfig, Parser, ParserState, ValidatingParser, + ValidatingParserConfig, WasmDecoder, +}; const VALIDATOR_CONFIG: Option = Some(ValidatingParserConfig { operator_config: OperatorValidatorConfig { @@ -12,81 +18,164 @@ const VALIDATOR_CONFIG: Option = Some(ValidatingParserCo enable_simd: true, enable_bulk_memory: true, enable_multi_value: true, + enable_tail_call: true, + enable_module_linking: true, }, }); -#[macro_use] -extern crate criterion; -extern crate wasmparser; +/// A benchmark input. +pub struct BenchmarkInput { + /// The path to the benchmark file important for handling errors. + pub path: PathBuf, + /// The encoded Wasm module that is run by the benchmark. + pub wasm: Vec, +} -use criterion::Criterion; -use wasmparser::{ - validate, OperatorValidatorConfig, Parser, ParserState, ValidatingParser, - ValidatingParserConfig, WasmDecoder, -}; +impl BenchmarkInput { + /// Creates a new benchmark input. + pub fn new(test_path: PathBuf, encoded_wasm: Vec) -> Self { + Self { + path: test_path, + wasm: encoded_wasm, + } + } +} -use std::fs::{read_dir, File}; -use std::io::Read; -use std::path::PathBuf; +/// Returns a vector of all found benchmark input files under the given directory. +/// +/// Benchmark input files can be `.wat` or `.wast` formatted files. +/// For `.wast` files we pull out all the module directives and run them in the benchmarks. +fn collect_test_files(path: &Path, list: &mut Vec) -> Result<()> { + for entry in path.read_dir()? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + collect_test_files(&path, list)?; + continue; + } + match path.extension().and_then(|ext| ext.to_str()) { + Some("wasm") => { + let wasm = fs::read(&path)?; + list.push(BenchmarkInput::new(path, wasm)); + } + Some("wat") | Some("txt") => { + if let Ok(wasm) = wat::parse_file(&path) { + list.push(BenchmarkInput::new(path, wasm)); + } + } + Some("wast") => { + let contents = fs::read_to_string(&path)?; + let buf = match wast::parser::ParseBuffer::new(&contents) { + Ok(buf) => buf, + Err(_) => continue, + }; + let wast: wast::Wast<'_> = match wast::parser::parse(&buf) { + Ok(wast) => wast, + Err(_) => continue, + }; + for directive in wast.directives { + match directive { + wast::WastDirective::Module(mut module) => { + let wasm = module.encode()?; + list.push(BenchmarkInput::new(path.clone(), wasm)); + } + _ => continue, + } + } + } + _ => (), + } + } + Ok(()) +} -fn read_all_wasm<'a, T>(mut d: T) +/// Reads the input given the Wasm parser or validator. +/// +/// The `path` specifies which benchmark input file we are currently operating on +/// so that we can report better errors in case of failures. +fn read_all_wasm<'a, T>(path: &PathBuf, mut d: T) where T: WasmDecoder<'a>, { loop { match *d.read() { - ParserState::Error(ref e) => panic!("unexpected error {}", e), + ParserState::Error(ref e) => { + panic!("unexpected error while reading Wasm from {:?}: {}", path, e) + } ParserState::EndWasm => return, _ => (), } } } +/// Returns the default benchmark inputs that are proper `wasmparser` benchmark +/// test inputs. +fn collect_benchmark_inputs() -> Vec { + let mut ret = Vec::new(); + collect_test_files("../../tests".as_ref(), &mut ret).unwrap(); + return ret; +} + fn it_works_benchmark(c: &mut Criterion) { - let mut data: Vec> = vec![]; - for entry in read_dir("tests").unwrap() { - let dir = entry.unwrap(); - if !dir.file_type().unwrap().is_file() { - continue; + let mut inputs = collect_benchmark_inputs(); + // Filter out all benchmark inputs that fail to parse via `wasmparser`. + inputs.retain(|input| { + let mut parser = Parser::new(input.wasm.as_slice()); + 'outer: loop { + match parser.read() { + ParserState::Error(_) => break 'outer false, + ParserState::EndWasm => break 'outer true, + _ => continue, + } } - data.push(read_file_data(&dir.path())); - } + }); c.bench_function("it works benchmark", move |b| { - for d in &mut data { - b.iter(|| read_all_wasm(Parser::new(d.as_slice()))); - } + b.iter(|| { + for input in &mut inputs { + let _ = black_box(read_all_wasm( + &input.path, + Parser::new(input.wasm.as_slice()), + )); + } + }) }); } fn validator_not_fails_benchmark(c: &mut Criterion) { - let mut data: Vec> = vec![]; - for entry in read_dir("tests").unwrap() { - let dir = entry.unwrap(); - if !dir.file_type().unwrap().is_file() { - continue; + let mut inputs = collect_benchmark_inputs(); + // Filter out all benchmark inputs that fail to validate via `wasmparser`. + inputs.retain(|input| { + let mut parser = ValidatingParser::new(input.wasm.as_slice(), VALIDATOR_CONFIG); + 'outer: loop { + match parser.read() { + ParserState::Error(_) => break 'outer false, + ParserState::EndWasm => break 'outer true, + _ => continue, + } } - data.push(read_file_data(&dir.path())); - } + }); c.bench_function("validator no fails benchmark", move |b| { - for d in &mut data { - b.iter(|| read_all_wasm(ValidatingParser::new(d.as_slice(), VALIDATOR_CONFIG))); - } + b.iter(|| { + for input in &mut inputs { + let _ = black_box(read_all_wasm( + &input.path, + ValidatingParser::new(input.wasm.as_slice(), VALIDATOR_CONFIG), + )); + } + }); }); } fn validate_benchmark(c: &mut Criterion) { - let mut data: Vec> = vec![vec![]]; - for entry in read_dir("tests").unwrap() { - let dir = entry.unwrap(); - if !dir.file_type().unwrap().is_file() { - continue; - } - data.push(read_file_data(&dir.path())); - } + let mut inputs = collect_benchmark_inputs(); + // Filter out all benchmark inputs that fail to validate via `wasmparser`. + inputs.retain(|input| validate(input.wasm.as_slice(), VALIDATOR_CONFIG).is_ok()); c.bench_function("validate benchmark", move |b| { - for d in &mut data { - b.iter(|| validate(&d, VALIDATOR_CONFIG)); - } + b.iter(|| { + for input in &mut inputs { + let _ = black_box(validate(input.wasm.as_slice(), VALIDATOR_CONFIG)); + } + }) }); } diff --git a/third_party/rust/wasmparser/compare-master.sh b/third_party/rust/wasmparser/compare-master.sh deleted file mode 100755 index e06c5e45a3f4..000000000000 --- a/third_party/rust/wasmparser/compare-master.sh +++ /dev/null @@ -1,12 +0,0 @@ -#/bin/bash - -# record current bench results -cargo bench --bench benchmark -- --noplot --save-baseline after - -# switch to master and record its bench results -git checkout master && \ -cargo bench --bench benchmark -- --noplot --save-baseline before - -# compare -cargo install critcmp --force && \ -critcmp before after diff --git a/third_party/rust/wasmparser/compare-with-main.sh b/third_party/rust/wasmparser/compare-with-main.sh new file mode 100755 index 000000000000..eed367bfab68 --- /dev/null +++ b/third_party/rust/wasmparser/compare-with-main.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e + +# switch to main branch and record its bench results +git checkout main && \ +cargo bench --bench benchmark -- --noplot --save-baseline before + +# record current bench results +git checkout - && \ +cargo bench --bench benchmark -- --noplot --baseline before diff --git a/third_party/rust/wasmparser/examples/dump.rs b/third_party/rust/wasmparser/examples/dump.rs index d651e3c19f41..af3b75cd9a93 100644 --- a/third_party/rust/wasmparser/examples/dump.rs +++ b/third_party/rust/wasmparser/examples/dump.rs @@ -38,7 +38,7 @@ fn main() { ref ty, } => { println!( - "ImportSectionEntry {{ module: \"{}\", field: \"{}\", ty: {:?} }}", + "ImportSectionEntry {{ module: \"{}\", field: {:?}, ty: {:?} }}", module, field, ty ); } diff --git a/third_party/rust/wasmparser/examples/simple.rs b/third_party/rust/wasmparser/examples/simple.rs index 8843957fda21..70dee2af9942 100644 --- a/third_party/rust/wasmparser/examples/simple.rs +++ b/third_party/rust/wasmparser/examples/simple.rs @@ -29,9 +29,11 @@ fn main() { } => { println!(" Export {} {:?}", field, kind); } - ParserState::ImportSectionEntry { module, field, .. } => { - println!(" Import {}::{}", module, field) - } + ParserState::ImportSectionEntry { + module, + field: Some(field), + .. + } => println!(" Import {}::{}", module, field), ParserState::EndWasm => break, ParserState::Error(ref err) => panic!("Error: {:?}", err), _ => ( /* println!(" Other {:?}", state); */ ), diff --git a/third_party/rust/wasmparser/src/binary_reader.rs b/third_party/rust/wasmparser/src/binary_reader.rs index 0b61eddbea58..2bcd35891d85 100644 --- a/third_party/rust/wasmparser/src/binary_reader.rs +++ b/third_party/rust/wasmparser/src/binary_reader.rs @@ -18,16 +18,14 @@ use std::convert::TryInto; use std::str; use std::vec::Vec; -use crate::limits::{ - MAX_WASM_FUNCTION_LOCALS, MAX_WASM_FUNCTION_PARAMS, MAX_WASM_FUNCTION_RETURNS, - MAX_WASM_FUNCTION_SIZE, MAX_WASM_STRING_SIZE, -}; +use crate::limits::*; use crate::primitives::{ BinaryReaderError, BrTable, CustomSectionKind, ExternalKind, FuncType, GlobalType, Ieee32, Ieee64, LinkingType, MemoryImmediate, MemoryType, NameType, Operator, RelocType, ResizableLimits, Result, SIMDLaneIndex, SectionCode, TableType, Type, TypeOrFuncType, V128, }; +use crate::{ExportType, Import, ImportSectionEntryType, InstanceType, ModuleType}; const MAX_WASM_BR_TABLE_SIZE: usize = MAX_WASM_FUNCTION_SIZE; @@ -43,6 +41,7 @@ const WASM_MAGIC_NUMBER: &[u8; 4] = b"\0asm"; const WASM_EXPERIMENTAL_VERSION: u32 = 0xd; const WASM_SUPPORTED_VERSION: u32 = 0x1; +#[derive(Clone)] pub(crate) struct SectionHeader<'a> { pub code: SectionCode<'a>, pub payload_start: usize, @@ -235,6 +234,9 @@ impl<'a> BinaryReader<'a> { 1 => Ok(ExternalKind::Table), 2 => Ok(ExternalKind::Memory), 3 => Ok(ExternalKind::Global), + 5 => Ok(ExternalKind::Module), + 6 => Ok(ExternalKind::Instance), + 7 => Ok(ExternalKind::Type), _ => Err(BinaryReaderError::new( "Invalid external kind", self.original_position() - 1, @@ -243,13 +245,6 @@ impl<'a> BinaryReader<'a> { } pub(crate) fn read_func_type(&mut self) -> Result { - if self.read_type()? != Type::Func { - return Err(BinaryReaderError::new( - "type signature is not a func", - self.original_position() - 1, - )); - } - let params_len = self.read_var_u32()? as usize; if params_len > MAX_WASM_FUNCTION_PARAMS { return Err(BinaryReaderError::new( @@ -278,6 +273,77 @@ impl<'a> BinaryReader<'a> { }) } + pub(crate) fn read_module_type(&mut self) -> Result> { + let pos = self.original_position(); + let imports_len = self.read_var_u32()? as usize; + if imports_len > MAX_WASM_IMPORTS { + return Err(BinaryReaderError::new("imports size is out of bounds", pos)); + } + Ok(ModuleType { + imports: (0..imports_len) + .map(|_| self.read_import()) + .collect::>()?, + exports: self.read_export_types()?, + }) + } + + pub(crate) fn read_instance_type(&mut self) -> Result> { + Ok(InstanceType { + exports: self.read_export_types()?, + }) + } + + fn read_export_types(&mut self) -> Result]>> { + let pos = self.original_position(); + let exports_len = self.read_var_u32()? as usize; + if exports_len > MAX_WASM_EXPORTS { + return Err(BinaryReaderError::new("exports size is out of bound", pos)); + } + (0..exports_len).map(|_| self.read_export_type()).collect() + } + + pub(crate) fn read_import(&mut self) -> Result> { + let module = self.read_string()?; + + // For the `field`, figure out if we're the experimental encoding of + // single-level imports for the module linking proposal (a single-byte + // string which is 0xc0, which is invalid utf-8) or if we have a second + // level of import. + let mut clone = self.clone(); + let field = if clone.read_var_u32()? == 1 && clone.read_u8()? == 0xc0 { + *self = clone; + None + } else { + Some(self.read_string()?) + }; + + let ty = self.read_import_desc()?; + Ok(Import { module, field, ty }) + } + + pub(crate) fn read_export_type(&mut self) -> Result> { + let name = self.read_string()?; + let ty = self.read_import_desc()?; + Ok(ExportType { name, ty }) + } + + pub(crate) fn read_import_desc(&mut self) -> Result { + Ok(match self.read_external_kind()? { + ExternalKind::Function => ImportSectionEntryType::Function(self.read_var_u32()?), + ExternalKind::Table => ImportSectionEntryType::Table(self.read_table_type()?), + ExternalKind::Memory => ImportSectionEntryType::Memory(self.read_memory_type()?), + ExternalKind::Global => ImportSectionEntryType::Global(self.read_global_type()?), + ExternalKind::Module => ImportSectionEntryType::Module(self.read_var_u32()?), + ExternalKind::Instance => ImportSectionEntryType::Instance(self.read_var_u32()?), + ExternalKind::Type => { + return Err(BinaryReaderError::new( + "cannot import types", + self.original_position() - 1, + )) + } + }) + } + fn read_resizable_limits(&mut self, max_present: bool) -> Result { let initial = self.read_var_u32()?; let maximum = if max_present { @@ -362,6 +428,10 @@ impl<'a> BinaryReader<'a> { 10 => Ok(SectionCode::Code), 11 => Ok(SectionCode::Data), 12 => Ok(SectionCode::DataCount), + 100 => Ok(SectionCode::Module), + 101 => Ok(SectionCode::Instance), + 102 => Ok(SectionCode::Alias), + 103 => Ok(SectionCode::ModuleCode), _ => Err(BinaryReaderError::new("Invalid section code", offset)), } } @@ -1526,6 +1596,7 @@ impl<'a> BinaryReader<'a> { 0x61 => Operator::I8x16Neg, 0x62 => Operator::I8x16AnyTrue, 0x63 => Operator::I8x16AllTrue, + 0x64 => Operator::I8x16Bitmask, 0x65 => Operator::I8x16NarrowI16x8S, 0x66 => Operator::I8x16NarrowI16x8U, 0x6b => Operator::I8x16Shl, @@ -1546,6 +1617,7 @@ impl<'a> BinaryReader<'a> { 0x81 => Operator::I16x8Neg, 0x82 => Operator::I16x8AnyTrue, 0x83 => Operator::I16x8AllTrue, + 0x84 => Operator::I16x8Bitmask, 0x85 => Operator::I16x8NarrowI32x4S, 0x86 => Operator::I16x8NarrowI32x4U, 0x87 => Operator::I16x8WidenLowI8x16S, @@ -1571,6 +1643,7 @@ impl<'a> BinaryReader<'a> { 0xa1 => Operator::I32x4Neg, 0xa2 => Operator::I32x4AnyTrue, 0xa3 => Operator::I32x4AllTrue, + 0xa4 => Operator::I32x4Bitmask, 0xa7 => Operator::I32x4WidenLowI16x8S, 0xa8 => Operator::I32x4WidenHighI16x8S, 0xa9 => Operator::I32x4WidenLowI16x8U, diff --git a/third_party/rust/wasmparser/src/lib.rs b/third_party/rust/wasmparser/src/lib.rs index e581b932daa5..b40210873bbc 100644 --- a/third_party/rust/wasmparser/src/lib.rs +++ b/third_party/rust/wasmparser/src/lib.rs @@ -39,15 +39,18 @@ pub use crate::parser::WasmDecoder; pub use crate::primitives::BinaryReaderError; pub use crate::primitives::BrTable; pub use crate::primitives::CustomSectionKind; +pub use crate::primitives::ExportType; pub use crate::primitives::ExternalKind; pub use crate::primitives::FuncType; pub use crate::primitives::GlobalType; pub use crate::primitives::Ieee32; pub use crate::primitives::Ieee64; pub use crate::primitives::ImportSectionEntryType; +pub use crate::primitives::InstanceType; pub use crate::primitives::LinkingType; pub use crate::primitives::MemoryImmediate; pub use crate::primitives::MemoryType; +pub use crate::primitives::ModuleType; pub use crate::primitives::NameType; pub use crate::primitives::Naming; pub use crate::primitives::Operator; @@ -57,6 +60,7 @@ pub use crate::primitives::Result; pub use crate::primitives::SectionCode; pub use crate::primitives::TableType; pub use crate::primitives::Type; +pub use crate::primitives::TypeDef; pub use crate::primitives::TypeOrFuncType; pub use crate::primitives::V128; @@ -72,11 +76,15 @@ pub use crate::module_resources::WasmMemoryType; pub use crate::module_resources::WasmModuleResources; pub use crate::module_resources::WasmTableType; pub use crate::module_resources::WasmType; +pub use crate::module_resources::WasmTypeDef; pub(crate) use crate::module_resources::{wasm_func_type_inputs, wasm_func_type_outputs}; pub use crate::operators_validator::OperatorValidatorConfig; +pub use crate::readers::Alias; +pub use crate::readers::AliasSectionReader; +pub use crate::readers::AliasedInstance; pub use crate::readers::CodeSectionReader; pub use crate::readers::CustomSectionContent; pub use crate::readers::Data; @@ -97,10 +105,13 @@ pub use crate::readers::GlobalSectionReader; pub use crate::readers::Import; pub use crate::readers::ImportSectionReader; pub use crate::readers::InitExpr; +pub use crate::readers::InstanceSectionReader; pub use crate::readers::LinkingSectionReader; pub use crate::readers::LocalsReader; pub use crate::readers::MemorySectionReader; +pub use crate::readers::ModuleCodeSectionReader; pub use crate::readers::ModuleReader; +pub use crate::readers::ModuleSectionReader; pub use crate::readers::Name; pub use crate::readers::NameSectionReader; pub use crate::readers::NamingReader; diff --git a/third_party/rust/wasmparser/src/limits.rs b/third_party/rust/wasmparser/src/limits.rs index 38594a6362c2..e0307d14f67c 100644 --- a/third_party/rust/wasmparser/src/limits.rs +++ b/third_party/rust/wasmparser/src/limits.rs @@ -17,8 +17,8 @@ // The limits are agreed upon with other engines for consistency. pub const MAX_WASM_TYPES: usize = 1_000_000; pub const MAX_WASM_FUNCTIONS: usize = 1_000_000; -pub const _MAX_WASM_IMPORTS: usize = 100_000; -pub const _MAX_WASM_EXPORTS: usize = 100_000; +pub const MAX_WASM_IMPORTS: usize = 100_000; +pub const MAX_WASM_EXPORTS: usize = 100_000; pub const MAX_WASM_GLOBALS: usize = 1_000_000; pub const _MAX_WASM_DATA_SEGMENTS: usize = 100_000; pub const MAX_WASM_MEMORY_PAGES: usize = 65536; @@ -32,3 +32,5 @@ pub const _MAX_WASM_TABLE_SIZE: usize = 10_000_000; pub const MAX_WASM_TABLE_ENTRIES: usize = 10_000_000; pub const MAX_WASM_TABLES: usize = 1; pub const MAX_WASM_MEMORIES: usize = 1; +pub const MAX_WASM_MODULES: usize = 1_000; +pub const MAX_WASM_INSTANCES: usize = 1_000; diff --git a/third_party/rust/wasmparser/src/module_resources.rs b/third_party/rust/wasmparser/src/module_resources.rs index d75eaae3389c..4ed1c4d58635 100644 --- a/third_party/rust/wasmparser/src/module_resources.rs +++ b/third_party/rust/wasmparser/src/module_resources.rs @@ -13,6 +13,8 @@ * limitations under the License. */ +use crate::{FuncType, TypeDef}; + /// Types that qualify as Wasm types for validation purposes. /// /// Must be comparable with `wasmparser` given Wasm types and @@ -26,6 +28,12 @@ pub trait WasmType: PartialEq + PartialEq + Eq { fn to_parser_type(&self) -> crate::Type; } +pub trait WasmTypeDef { + type FuncType: WasmFuncType; + + fn as_func(&self) -> Option<&Self::FuncType>; +} + /// Types that qualify as Wasm function types for validation purposes. pub trait WasmFuncType { /// A type that is comparable with Wasm types. @@ -277,7 +285,7 @@ pub trait WasmGlobalType { /// the need of an additional parsing or validation step or copying data around. pub trait WasmModuleResources { /// The function type used for validation. - type FuncType: WasmFuncType; + type TypeDef: WasmTypeDef; /// The table type used for validation. type TableType: WasmTableType; /// The memory type used for validation. @@ -286,7 +294,7 @@ pub trait WasmModuleResources { type GlobalType: WasmGlobalType; /// Returns the type at given index. - fn type_at(&self, at: u32) -> Option<&Self::FuncType>; + fn type_at(&self, at: u32) -> Option<&Self::TypeDef>; /// Returns the table at given index if any. fn table_at(&self, at: u32) -> Option<&Self::TableType>; /// Returns the linear memory at given index. @@ -294,7 +302,7 @@ pub trait WasmModuleResources { /// Returns the global variable at given index. fn global_at(&self, at: u32) -> Option<&Self::GlobalType>; /// Returns the function signature ID at given index. - fn func_type_id_at(&self, at: u32) -> Option; + fn func_type_at(&self, at: u32) -> Option<&::FuncType>; /// Returns the element type at the given index. fn element_type_at(&self, at: u32) -> Option; @@ -311,12 +319,12 @@ impl WasmModuleResources for &'_ T where T: ?Sized + WasmModuleResources, { - type FuncType = T::FuncType; + type TypeDef = T::TypeDef; type TableType = T::TableType; type MemoryType = T::MemoryType; type GlobalType = T::GlobalType; - fn type_at(&self, at: u32) -> Option<&Self::FuncType> { + fn type_at(&self, at: u32) -> Option<&Self::TypeDef> { T::type_at(self, at) } fn table_at(&self, at: u32) -> Option<&Self::TableType> { @@ -328,8 +336,8 @@ where fn global_at(&self, at: u32) -> Option<&Self::GlobalType> { T::global_at(self, at) } - fn func_type_id_at(&self, at: u32) -> Option { - T::func_type_id_at(self, at) + fn func_type_at(&self, at: u32) -> Option<&::FuncType> { + T::func_type_at(self, at) } fn element_type_at(&self, at: u32) -> Option { T::element_type_at(self, at) @@ -352,6 +360,17 @@ impl WasmType for crate::Type { } } +impl<'a> WasmTypeDef for TypeDef<'a> { + type FuncType = FuncType; + + fn as_func(&self) -> Option<&Self::FuncType> { + match self { + TypeDef::Func(f) => Some(f), + _ => None, + } + } +} + impl WasmFuncType for crate::FuncType { type Type = crate::Type; diff --git a/third_party/rust/wasmparser/src/operators_validator.rs b/third_party/rust/wasmparser/src/operators_validator.rs index 978c4215b2aa..b3a4c3206486 100644 --- a/third_party/rust/wasmparser/src/operators_validator.rs +++ b/third_party/rust/wasmparser/src/operators_validator.rs @@ -18,7 +18,7 @@ use std::cmp::min; use crate::primitives::{MemoryImmediate, Operator, SIMDLaneIndex, Type, TypeOrFuncType}; use crate::{ wasm_func_type_inputs, wasm_func_type_outputs, BinaryReaderError, WasmFuncType, WasmGlobalType, - WasmMemoryType, WasmModuleResources, WasmTableType, WasmType, + WasmModuleResources, WasmTableType, WasmType, WasmTypeDef, }; #[derive(Debug)] @@ -113,28 +113,17 @@ impl FuncState { } Ok(()) } - fn push_block( + fn push_block( &mut self, ty: TypeOrFuncType, block_type: BlockType, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, + resources: impl WasmModuleResources, ) -> OperatorValidatorResult<()> { let (start_types, return_types) = match ty { TypeOrFuncType::Type(Type::EmptyBlockType) => (vec![], vec![]), TypeOrFuncType::Type(ty) => (vec![], vec![ty]), TypeOrFuncType::FuncType(idx) => { - let ty = resources - .type_at(idx) - // Note: This was an out-of-bounds memory access before - // the change to return `Option` at `type_at`. So - // I assumed that invalid indices at this point are - // bugs. - .expect("function type index is out of bounds"); + let ty = func_type_at(&resources, idx)?; ( wasm_func_type_inputs(ty) .map(WasmType::to_parser_type) @@ -328,6 +317,7 @@ pub struct OperatorValidatorConfig { pub enable_bulk_memory: bool, pub enable_multi_value: bool, pub enable_tail_call: bool, + pub enable_module_linking: bool, #[cfg(feature = "deterministic")] pub deterministic_only: bool, @@ -341,6 +331,7 @@ pub(crate) const DEFAULT_OPERATOR_VALIDATOR_CONFIG: OperatorValidatorConfig = enable_bulk_memory: false, enable_multi_value: true, enable_tail_call: false, + enable_module_linking: false, #[cfg(feature = "deterministic")] deterministic_only: true, @@ -532,7 +523,7 @@ impl OperatorValidator { function_index: u32, resources: impl WasmModuleResources, ) -> OperatorValidatorResult<()> { - let type_index = match resources.func_type_id_at(function_index) { + let ty = match resources.func_type_at(function_index) { Some(i) => i, None => { bail_op_err!( @@ -541,9 +532,6 @@ impl OperatorValidator { ); } }; - let ty = resources - .type_at(type_index) - .expect("function type index is out of bounds"); self.check_operands(wasm_func_type_inputs(ty).map(WasmType::to_parser_type))?; self.func_state.change_frame_with_types( ty.len_inputs(), @@ -563,26 +551,18 @@ impl OperatorValidator { "unknown table: table index out of bounds", )); } - match resources.type_at(index) { - None => { - return Err(OperatorValidatorError::new( - "unknown type: type index out of bounds", - )) - } - Some(ty) => { - let types = { - let mut types = Vec::with_capacity(ty.len_inputs() + 1); - types.extend(wasm_func_type_inputs(ty).map(WasmType::to_parser_type)); - types.push(Type::I32); - types - }; - self.check_operands(types.into_iter())?; - self.func_state.change_frame_with_types( - ty.len_inputs() + 1, - wasm_func_type_outputs(ty).map(WasmType::to_parser_type), - )?; - } - } + let ty = func_type_at(&resources, index)?; + let types = { + let mut types = Vec::with_capacity(ty.len_inputs() + 1); + types.extend(wasm_func_type_inputs(ty).map(WasmType::to_parser_type)); + types.push(Type::I32); + types + }; + self.check_operands(types.into_iter())?; + self.func_state.change_frame_with_types( + ty.len_inputs() + 1, + wasm_func_type_outputs(ty).map(WasmType::to_parser_type), + )?; Ok(()) } @@ -674,20 +654,10 @@ impl OperatorValidator { Ok(()) } - fn check_memory_index< - F: WasmFuncType, - T: WasmTableType, - M: WasmMemoryType, - G: WasmGlobalType, - >( + fn check_memory_index( &self, memory_index: u32, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, + resources: impl WasmModuleResources, ) -> OperatorValidatorResult<()> { if resources.memory_at(memory_index).is_none() { bail_op_err!("unknown memory {}", memory_index); @@ -695,16 +665,11 @@ impl OperatorValidator { Ok(()) } - fn check_memarg( + fn check_memarg( &self, memarg: MemoryImmediate, max_align: u32, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, + resources: impl WasmModuleResources, ) -> OperatorValidatorResult<()> { self.check_memory_index(0, resources)?; let align = memarg.flags; @@ -766,20 +731,10 @@ impl OperatorValidator { Ok(()) } - fn check_shared_memarg_wo_align< - F: WasmFuncType, - T: WasmTableType, - M: WasmMemoryType, - G: WasmGlobalType, - >( + fn check_shared_memarg_wo_align( &self, _: MemoryImmediate, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, + resources: impl WasmModuleResources, ) -> OperatorValidatorResult<()> { self.check_memory_index(0, resources)?; Ok(()) @@ -792,15 +747,10 @@ impl OperatorValidator { Ok(()) } - fn check_block_type( + fn check_block_type( &self, ty: TypeOrFuncType, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, + resources: impl WasmModuleResources, ) -> OperatorValidatorResult<()> { match ty { TypeOrFuncType::Type(Type::EmptyBlockType) @@ -812,9 +762,9 @@ impl OperatorValidator { self.check_reference_types_enabled() } TypeOrFuncType::Type(Type::V128) => self.check_simd_enabled(), - TypeOrFuncType::FuncType(idx) => match resources.type_at(idx) { - None => Err(OperatorValidatorError::new("type index out of bounds")), - Some(ty) if !self.config.enable_multi_value => { + TypeOrFuncType::FuncType(idx) => { + let ty = func_type_at(&resources, idx)?; + if !self.config.enable_multi_value { if ty.len_outputs() > 1 { return Err(OperatorValidatorError::new( "blocks, loops, and ifs may only return at most one \ @@ -827,38 +777,21 @@ impl OperatorValidator { when multi-value is not enabled", )); } - Ok(()) } - Some(_) => Ok(()), - }, + Ok(()) + } _ => Err(OperatorValidatorError::new("invalid block return type")), } } - fn check_block_params< - F: WasmFuncType, - T: WasmTableType, - M: WasmMemoryType, - G: WasmGlobalType, - >( + fn check_block_params( &self, ty: TypeOrFuncType, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, + resources: impl WasmModuleResources, skip: usize, ) -> OperatorValidatorResult<()> { if let TypeOrFuncType::FuncType(idx) = ty { - let func_ty = resources - .type_at(idx) - // Note: This was an out-of-bounds memory access before - // the change to return `Option` at `type_at`. So - // I assumed that invalid indices at this point are - // bugs. - .expect("function type index is out of bounds"); + let func_ty = func_type_at(&resources, idx)?; let len = func_ty.len_inputs(); self.check_frame_size(len + skip)?; for (i, ty) in wasm_func_type_inputs(func_ty).enumerate() { @@ -908,20 +841,10 @@ impl OperatorValidator { Ok(Some(ty)) } - pub(crate) fn process_operator< - F: WasmFuncType, - T: WasmTableType, - M: WasmMemoryType, - G: WasmGlobalType, - >( + pub(crate) fn process_operator( &mut self, operator: &Operator, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, + resources: &impl WasmModuleResources, ) -> OperatorValidatorResult { if self.func_state.end_function { return Err(OperatorValidatorError::new("unexpected operator")); @@ -1633,7 +1556,7 @@ impl OperatorValidator { } Operator::RefFunc { function_index } => { self.check_reference_types_enabled()?; - if resources.func_type_id_at(function_index).is_none() { + if resources.func_type_at(function_index).is_none() { return Err(OperatorValidatorError::new( "unknown function: function index out of bounds", )); @@ -1903,10 +1826,13 @@ impl OperatorValidator { } Operator::I8x16AnyTrue | Operator::I8x16AllTrue + | Operator::I8x16Bitmask | Operator::I16x8AnyTrue | Operator::I16x8AllTrue + | Operator::I16x8Bitmask | Operator::I32x4AnyTrue - | Operator::I32x4AllTrue => { + | Operator::I32x4AllTrue + | Operator::I32x4Bitmask => { self.check_simd_enabled()?; self.check_operands_1(Type::V128)?; self.func_state.change_frame_with_type(1, Type::I32)?; @@ -2096,3 +2022,19 @@ impl OperatorValidator { Ok(()) } } + +fn func_type_at( + resources: &T, + at: u32, +) -> OperatorValidatorResult<&::FuncType> { + let ty = match resources.type_at(at) { + Some(ty) => ty, + None => { + return Err(OperatorValidatorError::new( + "unknown type: type index out of bounds", + )) + } + }; + ty.as_func() + .ok_or_else(|| OperatorValidatorError::new("type index not a function type")) +} diff --git a/third_party/rust/wasmparser/src/parser.rs b/third_party/rust/wasmparser/src/parser.rs index f8ef88d9a0dd..03b3f5ac8556 100644 --- a/third_party/rust/wasmparser/src/parser.rs +++ b/third_party/rust/wasmparser/src/parser.rs @@ -22,19 +22,12 @@ use crate::limits::{ }; use crate::primitives::{ - BinaryReaderError, CustomSectionKind, ExternalKind, FuncType, GlobalType, - ImportSectionEntryType, LinkingType, MemoryType, Naming, Operator, RelocType, Result, - SectionCode, TableType, Type, + BinaryReaderError, CustomSectionKind, ExternalKind, GlobalType, ImportSectionEntryType, + LinkingType, MemoryType, Naming, Operator, RelocType, Result, SectionCode, TableType, Type, + TypeDef, }; -use crate::readers::{ - CodeSectionReader, Data, DataKind, DataSectionReader, Element, ElementItem, ElementItems, - ElementKind, ElementSectionReader, Export, ExportSectionReader, FunctionBody, - FunctionSectionReader, Global, GlobalSectionReader, Import, ImportSectionReader, - LinkingSectionReader, MemorySectionReader, ModuleReader, Name, NameSectionReader, NamingReader, - OperatorsReader, Reloc, RelocSectionReader, Section, SectionReader, TableSectionReader, - TypeSectionReader, -}; +use crate::readers::*; use crate::binary_reader::{BinaryReader, Range}; @@ -85,10 +78,10 @@ pub enum ParserState<'a> { ReadingSectionRawData, SectionRawData(&'a [u8]), - TypeSectionEntry(FuncType), + TypeSectionEntry(TypeDef<'a>), ImportSectionEntry { module: &'a str, - field: &'a str, + field: Option<&'a str>, ty: ImportSectionEntryType, }, FunctionSectionEntry(u32), @@ -140,6 +133,19 @@ pub enum ParserState<'a> { LinkingSectionEntry(LinkingType), SourceMappingURL(&'a str), + + ModuleSectionEntry(u32), + AliasSectionEntry(Alias), + BeginInstantiate { + module: u32, + count: u32, + }, + InstantiateParameter { + kind: ExternalKind, + index: u32, + }, + EndInstantiate, + InlineModule(ModuleCode<'a>), } #[derive(Debug, Copy, Clone)] @@ -183,6 +189,10 @@ enum ParserSectionReader<'a> { NameSectionReader(NameSectionReader<'a>), LinkingSectionReader(LinkingSectionReader<'a>), RelocSectionReader(RelocSectionReader<'a>), + ModuleSectionReader(ModuleSectionReader<'a>), + AliasSectionReader(AliasSectionReader<'a>), + InstanceSectionReader(InstanceSectionReader<'a>), + ModuleCodeSectionReader(ModuleCodeSectionReader<'a>), } macro_rules! section_reader { @@ -221,6 +231,7 @@ pub struct Parser<'a> { current_data_segment: Option<&'a [u8]>, binary_reader: Option>, operators_reader: Option>, + instance_args: Option>, section_entries_left: u32, } @@ -247,6 +258,7 @@ impl<'a> Parser<'a> { current_data_segment: None, binary_reader: None, operators_reader: None, + instance_args: None, section_entries_left: 0, } } @@ -681,6 +693,63 @@ impl<'a> Parser<'a> { Ok(()) } + fn read_module_entry(&mut self) -> Result<()> { + if self.section_entries_left == 0 { + return self.check_section_end(); + } + let module_ty = section_reader!(self, ModuleSectionReader).read()?; + self.state = ParserState::ModuleSectionEntry(module_ty); + self.section_entries_left -= 1; + Ok(()) + } + + fn read_alias_entry(&mut self) -> Result<()> { + if self.section_entries_left == 0 { + return self.check_section_end(); + } + let alias_ty = section_reader!(self, AliasSectionReader).read()?; + self.state = ParserState::AliasSectionEntry(alias_ty); + self.section_entries_left -= 1; + Ok(()) + } + + fn read_instance_entry(&mut self) -> Result<()> { + if self.section_entries_left == 0 { + return self.check_section_end(); + } + let instance = section_reader!(self, InstanceSectionReader).read()?; + let args = instance.args()?; + self.state = ParserState::BeginInstantiate { + module: instance.module(), + count: args.get_count(), + }; + self.instance_args = Some(args); + self.section_entries_left -= 1; + Ok(()) + } + + fn read_instantiate_field(&mut self) -> Result<()> { + let instance = self.instance_args.as_mut().unwrap(); + if instance.eof() { + self.instance_args = None; + self.state = ParserState::EndInstantiate; + } else { + let (kind, index) = self.instance_args.as_mut().unwrap().read()?; + self.state = ParserState::InstantiateParameter { kind, index }; + } + Ok(()) + } + + fn read_module_code_entry(&mut self) -> Result<()> { + if self.section_entries_left == 0 { + return self.check_section_end(); + } + let module = section_reader!(self, ModuleCodeSectionReader).read()?; + self.state = ParserState::InlineModule(module); + self.section_entries_left -= 1; + Ok(()) + } + fn read_section_body(&mut self) -> Result<()> { match self.state { ParserState::BeginSection { @@ -775,6 +844,38 @@ impl<'a> Parser<'a> { .get_data_count_section_content()?; self.state = ParserState::DataCountSectionEntry(func_index); } + ParserState::BeginSection { + code: SectionCode::Module, + .. + } => { + start_section_reader!(self, ModuleSectionReader, get_module_section_reader); + self.read_module_entry()?; + } + ParserState::BeginSection { + code: SectionCode::Alias, + .. + } => { + start_section_reader!(self, AliasSectionReader, get_alias_section_reader); + self.read_alias_entry()?; + } + ParserState::BeginSection { + code: SectionCode::Instance, + .. + } => { + start_section_reader!(self, InstanceSectionReader, get_instance_section_reader); + self.read_instance_entry()?; + } + ParserState::BeginSection { + code: SectionCode::ModuleCode, + .. + } => { + start_section_reader!( + self, + ModuleCodeSectionReader, + get_module_code_section_reader + ); + self.read_module_code_entry()?; + } ParserState::BeginSection { code: SectionCode::Custom { .. }, .. @@ -849,6 +950,10 @@ impl<'a> Parser<'a> { ParserSectionReader::TypeSectionReader(ref reader) => reader.ensure_end()?, ParserSectionReader::LinkingSectionReader(ref reader) => reader.ensure_end()?, ParserSectionReader::RelocSectionReader(ref reader) => reader.ensure_end()?, + ParserSectionReader::ModuleSectionReader(ref reader) => reader.ensure_end()?, + ParserSectionReader::ModuleCodeSectionReader(ref reader) => reader.ensure_end()?, + ParserSectionReader::InstanceSectionReader(ref reader) => reader.ensure_end()?, + ParserSectionReader::AliasSectionReader(ref reader) => reader.ensure_end()?, _ => unreachable!(), } self.position_to_section_end() @@ -984,6 +1089,13 @@ impl<'a> Parser<'a> { ParserState::ReadingSectionRawData | ParserState::SectionRawData(_) => { self.read_section_body_bytes()? } + ParserState::ModuleSectionEntry(_) => self.read_module_entry()?, + ParserState::AliasSectionEntry(_) => self.read_alias_entry()?, + ParserState::BeginInstantiate { .. } | ParserState::InstantiateParameter { .. } => { + self.read_instantiate_field()? + } + ParserState::EndInstantiate => self.read_instance_entry()?, + ParserState::InlineModule(_) => self.read_module_code_entry()?, } Ok(()) } @@ -1085,7 +1197,7 @@ impl<'a> WasmDecoder<'a> for Parser<'a> { /// # 0x80, 0x80, 0x0, 0x0, 0xa, 0x91, 0x80, 0x80, 0x80, 0x0, /// # 0x2, 0x83, 0x80, 0x80, 0x80, 0x0, 0x0, 0x1, 0xb, 0x83, /// # 0x80, 0x80, 0x80, 0x0, 0x0, 0x0, 0xb]; - /// use wasmparser::{WasmDecoder, Parser, ParserState}; + /// use wasmparser::{WasmDecoder, Parser, ParserState, TypeDef}; /// let mut parser = Parser::new(data); /// let mut types = Vec::new(); /// let mut function_types = Vec::new(); @@ -1094,7 +1206,7 @@ impl<'a> WasmDecoder<'a> for Parser<'a> { /// match parser.read() { /// ParserState::Error(_) | /// ParserState::EndWasm => break, - /// ParserState::TypeSectionEntry(ty) => { + /// ParserState::TypeSectionEntry(TypeDef::Func(ty)) => { /// types.push(ty.clone()); /// } /// ParserState::FunctionSectionEntry(id) => { @@ -1182,3 +1294,20 @@ impl<'a> WasmDecoder<'a> for Parser<'a> { &self.state } } + +impl<'a> From> for Parser<'a> { + fn from(reader: ModuleReader<'a>) -> Parser<'a> { + let mut parser = Parser::default(); + parser.state = ParserState::BeginWasm { + version: reader.get_version(), + }; + parser.module_reader = Some(reader); + return parser; + } +} + +impl<'a> Default for Parser<'a> { + fn default() -> Parser<'a> { + Parser::new(&[]) + } +} diff --git a/third_party/rust/wasmparser/src/primitives.rs b/third_party/rust/wasmparser/src/primitives.rs index a49361cb43f9..08fa1fb7e0e6 100644 --- a/third_party/rust/wasmparser/src/primitives.rs +++ b/third_party/rust/wasmparser/src/primitives.rs @@ -83,18 +83,22 @@ pub enum SectionCode<'a> { name: &'a str, kind: CustomSectionKind, }, - Type, // Function signature declarations - Import, // Import declarations - Function, // Function declarations - Table, // Indirect function table and other tables - Memory, // Memory attributes - Global, // Global declarations - Export, // Exports - Start, // Start function declaration - Element, // Elements section - Code, // Function bodies (code) - Data, // Data segments - DataCount, // Count of passive data segments + Type, // Function signature declarations + Alias, // Aliased indices from nested/parent modules + Import, // Import declarations + Module, // Module declarations + Instance, // Instance definitions + Function, // Function declarations + Table, // Indirect function table and other tables + Memory, // Memory attributes + Global, // Global declarations + Export, // Exports + Start, // Start function declaration + Element, // Elements section + ModuleCode, // Module definitions + Code, // Function bodies (code) + Data, // Data segments + DataCount, // Count of passive data segments } /// Types as defined [here]. @@ -135,6 +139,16 @@ pub enum ExternalKind { Table, Memory, Global, + Type, + Module, + Instance, +} + +#[derive(Debug, Clone)] +pub enum TypeDef<'a> { + Func(FuncType), + Instance(InstanceType<'a>), + Module(ModuleType<'a>), } #[derive(Debug, Clone, Eq, PartialEq, Hash)] @@ -143,6 +157,23 @@ pub struct FuncType { pub returns: Box<[Type]>, } +#[derive(Debug, Clone)] +pub struct InstanceType<'a> { + pub exports: Box<[ExportType<'a>]>, +} + +#[derive(Debug, Clone)] +pub struct ModuleType<'a> { + pub imports: Box<[crate::Import<'a>]>, + pub exports: Box<[ExportType<'a>]>, +} + +#[derive(Debug, Clone)] +pub struct ExportType<'a> { + pub name: &'a str, + pub ty: ImportSectionEntryType, +} + #[derive(Debug, Copy, Clone)] pub struct ResizableLimits { pub initial: u32, @@ -161,7 +192,7 @@ pub struct MemoryType { pub shared: bool, } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct GlobalType { pub content_type: Type, pub mutable: bool, @@ -173,6 +204,8 @@ pub enum ImportSectionEntryType { Table(TableType), Memory(MemoryType), Global(GlobalType), + Module(u32), + Instance(u32), } #[derive(Debug, Copy, Clone)] @@ -617,6 +650,7 @@ pub enum Operator<'a> { I8x16Neg, I8x16AnyTrue, I8x16AllTrue, + I8x16Bitmask, I8x16Shl, I8x16ShrS, I8x16ShrU, @@ -634,6 +668,7 @@ pub enum Operator<'a> { I16x8Neg, I16x8AnyTrue, I16x8AllTrue, + I16x8Bitmask, I16x8Shl, I16x8ShrS, I16x8ShrU, @@ -652,6 +687,7 @@ pub enum Operator<'a> { I32x4Neg, I32x4AnyTrue, I32x4AllTrue, + I32x4Bitmask, I32x4Shl, I32x4ShrS, I32x4ShrU, diff --git a/third_party/rust/wasmparser/src/readers/alias_section.rs b/third_party/rust/wasmparser/src/readers/alias_section.rs new file mode 100644 index 000000000000..f98d2f1a05f6 --- /dev/null +++ b/third_party/rust/wasmparser/src/readers/alias_section.rs @@ -0,0 +1,84 @@ +use crate::{ + BinaryReader, BinaryReaderError, ExternalKind, Result, SectionIteratorLimited, SectionReader, + SectionWithLimitedItems, +}; + +pub struct AliasSectionReader<'a> { + reader: BinaryReader<'a>, + count: u32, +} + +#[derive(Debug)] +pub struct Alias { + pub instance: AliasedInstance, + pub kind: ExternalKind, + pub index: u32, +} + +#[derive(Debug)] +pub enum AliasedInstance { + Parent, + Child(u32), +} + +impl<'a> AliasSectionReader<'a> { + pub fn new(data: &'a [u8], offset: usize) -> Result> { + let mut reader = BinaryReader::new_with_offset(data, offset); + let count = reader.read_var_u32()?; + Ok(AliasSectionReader { reader, count }) + } + + pub fn original_position(&self) -> usize { + self.reader.original_position() + } + + pub fn get_count(&self) -> u32 { + self.count + } + + pub fn read(&mut self) -> Result { + Ok(Alias { + instance: match self.reader.read_u8()? { + 0x00 => AliasedInstance::Child(self.reader.read_var_u32()?), + 0x01 => AliasedInstance::Parent, + _ => { + return Err(BinaryReaderError::new( + "invalid byte in alias", + self.original_position() - 1, + )) + } + }, + kind: self.reader.read_external_kind()?, + index: self.reader.read_var_u32()?, + }) + } +} + +impl<'a> SectionReader for AliasSectionReader<'a> { + type Item = Alias; + + fn read(&mut self) -> Result { + AliasSectionReader::read(self) + } + fn eof(&self) -> bool { + self.reader.eof() + } + fn original_position(&self) -> usize { + AliasSectionReader::original_position(self) + } +} + +impl<'a> SectionWithLimitedItems for AliasSectionReader<'a> { + fn get_count(&self) -> u32 { + AliasSectionReader::get_count(self) + } +} + +impl<'a> IntoIterator for AliasSectionReader<'a> { + type Item = Result; + type IntoIter = SectionIteratorLimited>; + + fn into_iter(self) -> Self::IntoIter { + SectionIteratorLimited::new(self) + } +} diff --git a/third_party/rust/wasmparser/src/readers/import_section.rs b/third_party/rust/wasmparser/src/readers/import_section.rs index 92fd5f5e99d7..68e1ff58c0cc 100644 --- a/third_party/rust/wasmparser/src/readers/import_section.rs +++ b/third_party/rust/wasmparser/src/readers/import_section.rs @@ -13,15 +13,15 @@ * limitations under the License. */ -use super::{ - BinaryReader, ExternalKind, ImportSectionEntryType, Result, SectionIteratorLimited, - SectionReader, SectionWithLimitedItems, +use crate::{ + BinaryReader, ImportSectionEntryType, Result, SectionIteratorLimited, SectionReader, + SectionWithLimitedItems, }; #[derive(Debug, Copy, Clone)] pub struct Import<'a> { pub module: &'a str, - pub field: &'a str, + pub field: Option<&'a str>, pub ty: ImportSectionEntryType, } @@ -67,16 +67,7 @@ impl<'a> ImportSectionReader<'a> { where 'a: 'b, { - let module = self.reader.read_string()?; - let field = self.reader.read_string()?; - let kind = self.reader.read_external_kind()?; - let ty = match kind { - ExternalKind::Function => ImportSectionEntryType::Function(self.reader.read_var_u32()?), - ExternalKind::Table => ImportSectionEntryType::Table(self.reader.read_table_type()?), - ExternalKind::Memory => ImportSectionEntryType::Memory(self.reader.read_memory_type()?), - ExternalKind::Global => ImportSectionEntryType::Global(self.reader.read_global_type()?), - }; - Ok(Import { module, field, ty }) + self.reader.read_import() } } diff --git a/third_party/rust/wasmparser/src/readers/instance_section.rs b/third_party/rust/wasmparser/src/readers/instance_section.rs new file mode 100644 index 000000000000..c7ac301c54e8 --- /dev/null +++ b/third_party/rust/wasmparser/src/readers/instance_section.rs @@ -0,0 +1,147 @@ +use crate::{ + BinaryReader, ExternalKind, Result, SectionIteratorLimited, SectionReader, + SectionWithLimitedItems, +}; + +pub struct InstanceSectionReader<'a> { + reader: BinaryReader<'a>, + count: u32, +} + +impl<'a> InstanceSectionReader<'a> { + pub fn new(data: &'a [u8], offset: usize) -> Result> { + let mut reader = BinaryReader::new_with_offset(data, offset); + let count = reader.read_var_u32()?; + Ok(InstanceSectionReader { reader, count }) + } + + pub fn original_position(&self) -> usize { + self.reader.original_position() + } + + pub fn get_count(&self) -> u32 { + self.count + } + + pub fn read(&mut self) -> Result> { + let instance = Instance::new( + &self.reader.buffer[self.reader.position..], + self.original_position(), + )?; + self.reader.skip_var_32()?; + let count = self.reader.read_var_u32()?; + for _ in 0..count { + self.reader.skip_bytes(1)?; + self.reader.skip_var_32()?; + } + Ok(instance) + } +} + +impl<'a> SectionReader for InstanceSectionReader<'a> { + type Item = Instance<'a>; + + fn read(&mut self) -> Result { + InstanceSectionReader::read(self) + } + fn eof(&self) -> bool { + self.reader.eof() + } + fn original_position(&self) -> usize { + InstanceSectionReader::original_position(self) + } +} + +impl<'a> SectionWithLimitedItems for InstanceSectionReader<'a> { + fn get_count(&self) -> u32 { + InstanceSectionReader::get_count(self) + } +} + +impl<'a> IntoIterator for InstanceSectionReader<'a> { + type Item = Result>; + type IntoIter = SectionIteratorLimited>; + + fn into_iter(self) -> Self::IntoIter { + SectionIteratorLimited::new(self) + } +} + +pub struct Instance<'a> { + reader: BinaryReader<'a>, + module: u32, +} + +impl<'a> Instance<'a> { + pub fn new(data: &'a [u8], offset: usize) -> Result> { + let mut reader = BinaryReader::new_with_offset(data, offset); + let module = reader.read_var_u32()?; + Ok(Instance { module, reader }) + } + + pub fn original_position(&self) -> usize { + self.reader.original_position() + } + + pub fn module(&self) -> u32 { + self.module + } + + pub fn args(&self) -> Result> { + let mut reader = self.reader.clone(); + let count = reader.read_var_u32()?; + Ok(InstanceArgsReader { + count, + remaining: count, + reader, + }) + } +} + +pub struct InstanceArgsReader<'a> { + reader: BinaryReader<'a>, + count: u32, + remaining: u32, +} + +impl<'a> InstanceArgsReader<'a> { + pub fn original_position(&self) -> usize { + self.reader.original_position() + } + + pub fn read(&mut self) -> Result<(ExternalKind, u32)> { + let kind = self.reader.read_external_kind()?; + let index = self.reader.read_var_u32()?; + self.remaining -= 1; + Ok((kind, index)) + } +} + +impl<'a> SectionReader for InstanceArgsReader<'a> { + type Item = (ExternalKind, u32); + + fn read(&mut self) -> Result { + InstanceArgsReader::read(self) + } + fn eof(&self) -> bool { + self.remaining == 0 + } + fn original_position(&self) -> usize { + InstanceArgsReader::original_position(self) + } +} + +impl<'a> SectionWithLimitedItems for InstanceArgsReader<'a> { + fn get_count(&self) -> u32 { + self.count + } +} + +impl<'a> IntoIterator for InstanceArgsReader<'a> { + type Item = Result<(ExternalKind, u32)>; + type IntoIter = SectionIteratorLimited>; + + fn into_iter(self) -> Self::IntoIter { + SectionIteratorLimited::new(self) + } +} diff --git a/third_party/rust/wasmparser/src/readers/mod.rs b/third_party/rust/wasmparser/src/readers/mod.rs index 6eca50d9ff53..78d9e0c5beb7 100644 --- a/third_party/rust/wasmparser/src/readers/mod.rs +++ b/third_party/rust/wasmparser/src/readers/mod.rs @@ -14,9 +14,8 @@ */ use super::{ - BinaryReader, BinaryReaderError, CustomSectionKind, ExternalKind, FuncType, GlobalType, - ImportSectionEntryType, LinkingType, MemoryType, NameType, Naming, Operator, Range, RelocType, - Result, SectionCode, TableType, Type, + BinaryReader, BinaryReaderError, CustomSectionKind, ExternalKind, GlobalType, LinkingType, + MemoryType, NameType, Naming, Operator, Range, RelocType, Result, SectionCode, TableType, Type, }; use super::SectionHeader; @@ -56,6 +55,7 @@ pub use self::section_reader::SectionIteratorLimited; pub use self::section_reader::SectionReader; pub use self::section_reader::SectionWithLimitedItems; +pub use self::name_section::FunctionLocalReader; pub use self::name_section::FunctionName; pub use self::name_section::LocalName; pub use self::name_section::ModuleName; @@ -77,6 +77,12 @@ use self::sourcemappingurl_section::read_sourcemappingurl_section_content; pub use self::operators::OperatorsReader; +pub use self::alias_section::*; +pub use self::instance_section::*; +pub use self::module_code_section::*; +pub use self::module_section::*; + +mod alias_section; mod code_section; mod data_count_section; mod data_section; @@ -86,9 +92,12 @@ mod function_section; mod global_section; mod import_section; mod init_expr; +mod instance_section; mod linking_section; mod memory_section; mod module; +mod module_code_section; +mod module_section; mod name_section; mod operators; mod producers_section; diff --git a/third_party/rust/wasmparser/src/readers/module.rs b/third_party/rust/wasmparser/src/readers/module.rs index dd6499a443d0..6e8418a7c34f 100644 --- a/third_party/rust/wasmparser/src/readers/module.rs +++ b/third_party/rust/wasmparser/src/readers/module.rs @@ -13,15 +13,18 @@ * limitations under the License. */ +use std::fmt; + use super::{ BinaryReader, BinaryReaderError, CustomSectionKind, Range, Result, SectionCode, SectionHeader, }; use super::{ read_data_count_section_content, read_sourcemappingurl_section_content, - read_start_section_content, CodeSectionReader, DataSectionReader, ElementSectionReader, - ExportSectionReader, FunctionSectionReader, GlobalSectionReader, ImportSectionReader, - LinkingSectionReader, MemorySectionReader, NameSectionReader, ProducersSectionReader, + read_start_section_content, AliasSectionReader, CodeSectionReader, DataSectionReader, + ElementSectionReader, ExportSectionReader, FunctionSectionReader, GlobalSectionReader, + ImportSectionReader, InstanceSectionReader, LinkingSectionReader, MemorySectionReader, + ModuleCodeSectionReader, ModuleSectionReader, NameSectionReader, ProducersSectionReader, RelocSectionReader, TableSectionReader, TypeSectionReader, }; @@ -232,6 +235,46 @@ impl<'a> Section<'a> { } } + pub fn get_module_section_reader<'b>(&self) -> Result> + where + 'a: 'b, + { + match self.code { + SectionCode::Module => ModuleSectionReader::new(self.data, self.offset), + _ => panic!("Invalid state for get_module_section_reader"), + } + } + + pub fn get_alias_section_reader<'b>(&self) -> Result> + where + 'a: 'b, + { + match self.code { + SectionCode::Alias => AliasSectionReader::new(self.data, self.offset), + _ => panic!("Invalid state for get_alias_section_reader"), + } + } + + pub fn get_instance_section_reader<'b>(&self) -> Result> + where + 'a: 'b, + { + match self.code { + SectionCode::Instance => InstanceSectionReader::new(self.data, self.offset), + _ => panic!("Invalid state for get_instance_section_reader"), + } + } + + pub fn get_module_code_section_reader<'b>(&self) -> Result> + where + 'a: 'b, + { + match self.code { + SectionCode::ModuleCode => ModuleCodeSectionReader::new(self.data, self.offset), + _ => panic!("Invalid state for get_module_code_section_reader"), + } + } + pub fn get_binary_reader<'b>(&self) -> BinaryReader<'b> where 'a: 'b, @@ -239,6 +282,10 @@ impl<'a> Section<'a> { BinaryReader::new_with_offset(self.data, self.offset) } + pub fn content_raw(&self) -> &'a [u8] { + self.data + } + pub fn range(&self) -> Range { Range { start: self.offset, @@ -262,6 +309,12 @@ impl<'a> Section<'a> { SectionCode::Table => SectionContent::Table(self.get_table_section_reader()?), SectionCode::Element => SectionContent::Element(self.get_element_section_reader()?), SectionCode::Start => SectionContent::Start(self.get_start_section_content()?), + SectionCode::Module => SectionContent::Module(self.get_module_section_reader()?), + SectionCode::Alias => SectionContent::Alias(self.get_alias_section_reader()?), + SectionCode::Instance => SectionContent::Instance(self.get_instance_section_reader()?), + SectionCode::ModuleCode => { + SectionContent::ModuleCode(self.get_module_code_section_reader()?) + } SectionCode::DataCount => { SectionContent::DataCount(self.get_data_count_section_content()?) } @@ -323,6 +376,10 @@ pub enum SectionContent<'a> { binary: BinaryReader<'a>, content: Option>, }, + Module(ModuleSectionReader<'a>), + Alias(AliasSectionReader<'a>), + Instance(InstanceSectionReader<'a>), + ModuleCode(ModuleCodeSectionReader<'a>), } pub enum CustomSectionContent<'a> { @@ -334,6 +391,7 @@ pub enum CustomSectionContent<'a> { } /// Reads top-level WebAssembly file structure: header and sections. +#[derive(Clone)] pub struct ModuleReader<'a> { reader: BinaryReader<'a>, version: u32, @@ -342,7 +400,11 @@ pub struct ModuleReader<'a> { impl<'a> ModuleReader<'a> { pub fn new(data: &[u8]) -> Result { - let mut reader = BinaryReader::new(data); + ModuleReader::new_with_offset(data, 0) + } + + pub(crate) fn new_with_offset(data: &[u8], offset: usize) -> Result { + let mut reader = BinaryReader::new_with_offset(data, offset); let version = reader.read_file_header()?; Ok(ModuleReader { reader, @@ -362,6 +424,10 @@ impl<'a> ModuleReader<'a> { } } + pub fn original_position(&self) -> usize { + self.reader.original_position() + } + pub fn eof(&self) -> bool { self.read_ahead.is_none() && self.reader.eof() } @@ -412,11 +478,12 @@ impl<'a> ModuleReader<'a> { }; let payload_end = payload_start + payload_len; self.verify_section_end(payload_end)?; + let offset = self.reader.original_position(); let body_start = self.reader.position; self.reader.skip_to(payload_end); Ok(Section { code, - offset: body_start, + offset, data: &self.reader.buffer[body_start..payload_end], }) } @@ -481,6 +548,14 @@ impl<'a> IntoIterator for ModuleReader<'a> { } } +impl<'a> fmt::Debug for ModuleReader<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ModuleReader") + .field("version", &self.version) + .finish() + } +} + pub struct ModuleIterator<'a> { reader: ModuleReader<'a>, err: bool, diff --git a/third_party/rust/wasmparser/src/readers/module_code_section.rs b/third_party/rust/wasmparser/src/readers/module_code_section.rs new file mode 100644 index 000000000000..cb13b6b959c5 --- /dev/null +++ b/third_party/rust/wasmparser/src/readers/module_code_section.rs @@ -0,0 +1,93 @@ +use crate::{ + BinaryReader, BinaryReaderError, ModuleReader, Result, SectionIteratorLimited, SectionReader, + SectionWithLimitedItems, +}; + +pub struct ModuleCodeSectionReader<'a> { + reader: BinaryReader<'a>, + count: u32, +} + +#[derive(Debug)] +pub struct ModuleCode<'a> { + reader: BinaryReader<'a>, +} + +impl<'a> ModuleCodeSectionReader<'a> { + pub fn new(data: &'a [u8], offset: usize) -> Result> { + let mut reader = BinaryReader::new_with_offset(data, offset); + let count = reader.read_var_u32()?; + Ok(ModuleCodeSectionReader { reader, count }) + } + + pub fn original_position(&self) -> usize { + self.reader.original_position() + } + + pub fn get_count(&self) -> u32 { + self.count + } + + fn verify_module_end(&self, end: usize) -> Result<()> { + if self.reader.buffer.len() < end { + return Err(BinaryReaderError::new( + "module body extends past end of the module code section", + self.reader.original_offset + self.reader.buffer.len(), + )); + } + Ok(()) + } + + pub fn read(&mut self) -> Result> { + let size = self.reader.read_var_u32()? as usize; + let module_start = self.reader.position; + let module_end = module_start + size; + self.verify_module_end(module_end)?; + self.reader.skip_to(module_end); + Ok(ModuleCode { + reader: BinaryReader::new_with_offset( + &self.reader.buffer[module_start..module_end], + self.reader.original_offset + module_start, + ), + }) + } +} + +impl<'a> SectionReader for ModuleCodeSectionReader<'a> { + type Item = ModuleCode<'a>; + + fn read(&mut self) -> Result { + ModuleCodeSectionReader::read(self) + } + fn eof(&self) -> bool { + self.reader.eof() + } + fn original_position(&self) -> usize { + ModuleCodeSectionReader::original_position(self) + } +} + +impl<'a> SectionWithLimitedItems for ModuleCodeSectionReader<'a> { + fn get_count(&self) -> u32 { + ModuleCodeSectionReader::get_count(self) + } +} + +impl<'a> IntoIterator for ModuleCodeSectionReader<'a> { + type Item = Result>; + type IntoIter = SectionIteratorLimited>; + + fn into_iter(self) -> Self::IntoIter { + SectionIteratorLimited::new(self) + } +} + +impl<'a> ModuleCode<'a> { + pub fn module(&self) -> Result> { + ModuleReader::new_with_offset(self.reader.buffer, self.reader.original_position()) + } + + pub fn original_position(&self) -> usize { + self.reader.original_position() + } +} diff --git a/third_party/rust/wasmparser/src/readers/module_section.rs b/third_party/rust/wasmparser/src/readers/module_section.rs new file mode 100644 index 000000000000..e17f30f4dd9f --- /dev/null +++ b/third_party/rust/wasmparser/src/readers/module_section.rs @@ -0,0 +1,55 @@ +use super::{BinaryReader, Result, SectionIteratorLimited, SectionReader, SectionWithLimitedItems}; + +pub struct ModuleSectionReader<'a> { + reader: BinaryReader<'a>, + count: u32, +} + +impl<'a> ModuleSectionReader<'a> { + pub fn new(data: &'a [u8], offset: usize) -> Result> { + let mut reader = BinaryReader::new_with_offset(data, offset); + let count = reader.read_var_u32()?; + Ok(ModuleSectionReader { reader, count }) + } + + pub fn original_position(&self) -> usize { + self.reader.original_position() + } + + pub fn get_count(&self) -> u32 { + self.count + } + + pub fn read(&mut self) -> Result { + self.reader.read_var_u32() + } +} + +impl<'a> SectionReader for ModuleSectionReader<'a> { + type Item = u32; + + fn read(&mut self) -> Result { + ModuleSectionReader::read(self) + } + fn eof(&self) -> bool { + self.reader.eof() + } + fn original_position(&self) -> usize { + ModuleSectionReader::original_position(self) + } +} + +impl<'a> SectionWithLimitedItems for ModuleSectionReader<'a> { + fn get_count(&self) -> u32 { + ModuleSectionReader::get_count(self) + } +} + +impl<'a> IntoIterator for ModuleSectionReader<'a> { + type Item = Result; + type IntoIter = SectionIteratorLimited>; + + fn into_iter(self) -> Self::IntoIter { + SectionIteratorLimited::new(self) + } +} diff --git a/third_party/rust/wasmparser/src/readers/name_section.rs b/third_party/rust/wasmparser/src/readers/name_section.rs index 9ecaff99309d..9f7a6fc5360c 100644 --- a/third_party/rust/wasmparser/src/readers/name_section.rs +++ b/third_party/rust/wasmparser/src/readers/name_section.rs @@ -31,6 +31,10 @@ impl<'a> ModuleName<'a> { let mut reader = BinaryReader::new_with_offset(self.data, self.offset); reader.read_string() } + + pub fn original_position(&self) -> usize { + self.offset + } } pub struct NamingReader<'a> { @@ -85,6 +89,10 @@ impl<'a> FunctionName<'a> { { NamingReader::new(self.data, self.offset) } + + pub fn original_position(&self) -> usize { + self.offset + } } #[derive(Debug, Copy, Clone)] @@ -101,6 +109,10 @@ impl<'a> FunctionLocalName<'a> { { NamingReader::new(self.data, self.offset) } + + pub fn original_position(&self) -> usize { + self.offset + } } pub struct FunctionLocalReader<'a> { @@ -152,6 +164,10 @@ impl<'a> LocalName<'a> { { FunctionLocalReader::new(self.data, self.offset) } + + pub fn original_position(&self) -> usize { + self.offset + } } #[derive(Debug, Copy, Clone)] diff --git a/third_party/rust/wasmparser/src/readers/type_section.rs b/third_party/rust/wasmparser/src/readers/type_section.rs index 51fd6eea36e6..1fa288b21bbc 100644 --- a/third_party/rust/wasmparser/src/readers/type_section.rs +++ b/third_party/rust/wasmparser/src/readers/type_section.rs @@ -13,8 +13,9 @@ * limitations under the License. */ -use super::{ - BinaryReader, FuncType, Result, SectionIteratorLimited, SectionReader, SectionWithLimitedItems, +use crate::{ + BinaryReader, BinaryReaderError, Result, SectionIteratorLimited, SectionReader, + SectionWithLimitedItems, TypeDef, }; pub struct TypeSectionReader<'a> { @@ -53,13 +54,23 @@ impl<'a> TypeSectionReader<'a> { /// println!("Type {:?}", ty); /// } /// ``` - pub fn read(&mut self) -> Result { - self.reader.read_func_type() + pub fn read(&mut self) -> Result> { + Ok(match self.reader.read_u8()? { + 0x60 => TypeDef::Func(self.reader.read_func_type()?), + 0x61 => TypeDef::Module(self.reader.read_module_type()?), + 0x62 => TypeDef::Instance(self.reader.read_instance_type()?), + _ => { + return Err(BinaryReaderError::new( + "invalid leading byte in type definition", + self.original_position() - 1, + )) + } + }) } } impl<'a> SectionReader for TypeSectionReader<'a> { - type Item = FuncType; + type Item = TypeDef<'a>; fn read(&mut self) -> Result { TypeSectionReader::read(self) } @@ -78,7 +89,7 @@ impl<'a> SectionWithLimitedItems for TypeSectionReader<'a> { } impl<'a> IntoIterator for TypeSectionReader<'a> { - type Item = Result; + type Item = Result>; type IntoIter = SectionIteratorLimited>; /// Implements iterator over the type section. diff --git a/third_party/rust/wasmparser/src/validator.rs b/third_party/rust/wasmparser/src/validator.rs index 5c525d7f2c7e..d8fbe2b3175d 100644 --- a/third_party/rust/wasmparser/src/validator.rs +++ b/third_party/rust/wasmparser/src/validator.rs @@ -13,20 +13,19 @@ * limitations under the License. */ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; +use std::mem; use std::result; use std::str; -use crate::limits::{ - MAX_WASM_FUNCTIONS, MAX_WASM_FUNCTION_LOCALS, MAX_WASM_GLOBALS, MAX_WASM_MEMORIES, - MAX_WASM_MEMORY_PAGES, MAX_WASM_TABLES, MAX_WASM_TYPES, -}; +use crate::limits::*; use crate::binary_reader::BinaryReader; use crate::primitives::{ - BinaryReaderError, ExternalKind, FuncType, GlobalType, ImportSectionEntryType, MemoryType, - Operator, ResizableLimits, Result, SectionCode, TableType, Type, + BinaryReaderError, ExportType, ExternalKind, FuncType, GlobalType, ImportSectionEntryType, + InstanceType, MemoryType, ModuleType, Operator, ResizableLimits, Result, SectionCode, + TableType, Type, TypeDef, }; use crate::operators_validator::{ @@ -34,8 +33,8 @@ use crate::operators_validator::{ OperatorValidatorError, DEFAULT_OPERATOR_VALIDATOR_CONFIG, }; use crate::parser::{Parser, ParserInput, ParserState, WasmDecoder}; +use crate::{AliasedInstance, WasmModuleResources}; use crate::{ElemSectionEntryTable, ElementItem}; -use crate::{WasmFuncType, WasmGlobalType, WasmMemoryType, WasmModuleResources, WasmTableType}; use crate::readers::FunctionBody; @@ -48,11 +47,12 @@ struct InitExpressionState { validated: bool, } -#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq)] +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug)] enum SectionOrderState { Initial, Type, Import, + ModuleLinkingHeader, Function, Table, Memory, @@ -61,13 +61,22 @@ enum SectionOrderState { Start, Element, DataCount, + ModuleCode, Code, Data, } impl SectionOrderState { - pub fn from_section_code(code: &SectionCode) -> Option { + pub fn from_section_code( + code: &SectionCode, + config: &ValidatingParserConfig, + ) -> Option { match *code { + SectionCode::Type | SectionCode::Import + if config.operator_config.enable_module_linking => + { + Some(SectionOrderState::ModuleLinkingHeader) + } SectionCode::Type => Some(SectionOrderState::Type), SectionCode::Import => Some(SectionOrderState::Import), SectionCode::Function => Some(SectionOrderState::Function), @@ -80,11 +89,21 @@ impl SectionOrderState { SectionCode::Code => Some(SectionOrderState::Code), SectionCode::Data => Some(SectionOrderState::Data), SectionCode::DataCount => Some(SectionOrderState::DataCount), - _ => None, + SectionCode::Alias => Some(SectionOrderState::ModuleLinkingHeader), + SectionCode::Module => Some(SectionOrderState::ModuleLinkingHeader), + SectionCode::Instance => Some(SectionOrderState::ModuleLinkingHeader), + SectionCode::ModuleCode => Some(SectionOrderState::ModuleCode), + SectionCode::Custom { .. } => None, } } } +impl Default for SectionOrderState { + fn default() -> SectionOrderState { + SectionOrderState::Initial + } +} + #[derive(Copy, Clone)] pub struct ValidatingParserConfig { pub operator_config: OperatorValidatorConfig, @@ -94,73 +113,76 @@ const DEFAULT_VALIDATING_PARSER_CONFIG: ValidatingParserConfig = ValidatingParse operator_config: DEFAULT_OPERATOR_VALIDATOR_CONFIG, }; -struct ValidatingParserResources { - types: Vec, - tables: Vec, +#[derive(Default)] +struct ValidatingParserResources<'a> { + types: Vec>, + tables: Vec>, memories: Vec, - globals: Vec, + globals: Vec>, element_types: Vec, data_count: Option, - func_type_indices: Vec, + func_type_indices: Vec>, + module_type_indices: Vec>, + instance_type_indices: Vec>, function_references: HashSet, } -impl<'a> WasmModuleResources for ValidatingParserResources { - type FuncType = crate::FuncType; - type TableType = crate::TableType; - type MemoryType = crate::MemoryType; - type GlobalType = crate::GlobalType; +#[derive(Copy, Clone)] +struct Def { + module: usize, + item: T, +} - fn type_at(&self, at: u32) -> Option<&Self::FuncType> { - self.types.get(at as usize) +enum ValidatedType<'a> { + Def(TypeDef<'a>), + Alias(Def), +} + +impl Def { + fn as_ref(&self) -> Def<&T> { + Def { + module: self.module, + item: &self.item, + } } - fn table_at(&self, at: u32) -> Option<&Self::TableType> { - self.tables.get(at as usize) + fn map(self, map: impl FnOnce(T) -> U) -> Def { + Def { + module: self.module, + item: map(self.item), + } } +} - fn memory_at(&self, at: u32) -> Option<&Self::MemoryType> { - self.memories.get(at as usize) - } - - fn global_at(&self, at: u32) -> Option<&Self::GlobalType> { - self.globals.get(at as usize) - } - - fn func_type_id_at(&self, at: u32) -> Option { - self.func_type_indices.get(at as usize).copied() - } - - fn element_type_at(&self, at: u32) -> Option { - self.element_types.get(at as usize).cloned() - } - - fn element_count(&self) -> u32 { - self.element_types.len() as u32 - } - - fn data_count(&self) -> u32 { - self.data_count.unwrap_or(0) - } - - fn is_function_referenced(&self, idx: u32) -> bool { - self.function_references.contains(&idx) - } +enum InstanceDef { + Imported { type_idx: u32 }, + Instantiated { module_idx: u32 }, } pub struct ValidatingParser<'a> { parser: Parser<'a>, validation_error: Option>, read_position: Option, - section_order_state: SectionOrderState, - resources: ValidatingParserResources, - current_func_index: u32, - func_imports_count: u32, init_expression_state: Option, + current_operator_validator: Option, + /// Once we see a `BeginInstantiate` this tracks the type index of the + /// module type as well as which import index we're currently matching + /// against. + module_instantiation: Option<(Def, usize)>, + config: ValidatingParserConfig, + modules: Vec>, + total_nested_modules: usize, +} + +#[derive(Default)] +struct Module<'a> { + parser: Parser<'a>, + section_order_state: SectionOrderState, + resources: ValidatingParserResources<'a>, + current_func_index: u32, + func_nonlocal_count: u32, data_found: u32, exported_names: HashSet, - current_operator_validator: Option, - config: ValidatingParserConfig, } impl<'a> ValidatingParser<'a> { @@ -169,38 +191,15 @@ impl<'a> ValidatingParser<'a> { parser: Parser::new(bytes), validation_error: None, read_position: None, - section_order_state: SectionOrderState::Initial, - resources: ValidatingParserResources { - types: Vec::new(), - tables: Vec::new(), - memories: Vec::new(), - globals: Vec::new(), - element_types: Vec::new(), - data_count: None, - func_type_indices: Vec::new(), - function_references: HashSet::new(), - }, - current_func_index: 0, - func_imports_count: 0, current_operator_validator: None, init_expression_state: None, - data_found: 0, - exported_names: HashSet::new(), + module_instantiation: None, config: config.unwrap_or(DEFAULT_VALIDATING_PARSER_CONFIG), + modules: vec![Module::default()], + total_nested_modules: 0, } } - pub fn get_resources( - &self, - ) -> &dyn WasmModuleResources< - FuncType = crate::FuncType, - TableType = crate::TableType, - MemoryType = crate::MemoryType, - GlobalType = crate::GlobalType, - > { - &self.resources - } - fn set_validation_error(&mut self, message: impl Into) { self.validation_error = Some(ParserState::Error(BinaryReaderError::new( message, @@ -251,6 +250,37 @@ impl<'a> ValidatingParser<'a> { } } + fn check_module_type(&self, ty: &ModuleType<'a>) -> ValidatorResult<'a, ()> { + if !self.config.operator_config.enable_module_linking { + return self.create_error("module linking proposal not enabled"); + } + for i in ty.imports.iter() { + self.check_import_entry(&i.ty)?; + } + let mut names = HashSet::new(); + for e in ty.exports.iter() { + if !names.insert(e.name) { + return self.create_error("duplicate export name"); + } + self.check_import_entry(&e.ty)?; + } + Ok(()) + } + + fn check_instance_type(&self, ty: &InstanceType<'a>) -> ValidatorResult<'a, ()> { + if !self.config.operator_config.enable_module_linking { + return self.create_error("module linking proposal not enabled"); + } + let mut names = HashSet::new(); + for e in ty.exports.iter() { + if !names.insert(e.name) { + return self.create_error("duplicate export name"); + } + self.check_import_entry(&e.ty)?; + } + Ok(()) + } + fn check_table_type(&self, table_type: &TableType) -> ValidatorResult<'a, ()> { match table_type.element_type { Type::FuncRef => {} @@ -288,37 +318,60 @@ impl<'a> ValidatingParser<'a> { self.check_value_type(global_type.content_type) } + fn cur_module(&self) -> &Module<'a> { + self.modules.last().unwrap() + } + + fn cur_module_mut(&mut self) -> &mut Module<'a> { + self.modules.last_mut().unwrap() + } + fn check_import_entry(&self, import_type: &ImportSectionEntryType) -> ValidatorResult<'a, ()> { match *import_type { ImportSectionEntryType::Function(type_index) => { - if self.resources.func_type_indices.len() >= MAX_WASM_FUNCTIONS { + if self.cur_module().resources.func_type_indices.len() >= MAX_WASM_FUNCTIONS { return self.create_error("functions count out of bounds"); } - if type_index as usize >= self.resources.types.len() { - return self.create_error("unknown type: type index out of bounds"); - } + self.func_type_at(self.def(type_index))?; Ok(()) } ImportSectionEntryType::Table(ref table_type) => { if !self.config.operator_config.enable_reference_types - && self.resources.tables.len() >= MAX_WASM_TABLES + && !self.config.operator_config.enable_module_linking + && self.cur_module().resources.tables.len() >= MAX_WASM_TABLES { return self.create_error("multiple tables: tables count must be at most 1"); } self.check_table_type(table_type) } ImportSectionEntryType::Memory(ref memory_type) => { - if self.resources.memories.len() >= MAX_WASM_MEMORIES { + if !self.config.operator_config.enable_module_linking + && self.cur_module().resources.memories.len() >= MAX_WASM_MEMORIES + { return self.create_error("multiple memories: memory count must be at most 1"); } self.check_memory_type(memory_type) } ImportSectionEntryType::Global(global_type) => { - if self.resources.globals.len() >= MAX_WASM_GLOBALS { - return self.create_error("functions count out of bounds"); + if self.cur_module().resources.globals.len() >= MAX_WASM_GLOBALS { + return self.create_error("globals count out of bounds"); } self.check_global_type(global_type) } + ImportSectionEntryType::Module(type_index) => { + if self.cur_module().resources.module_type_indices.len() >= MAX_WASM_MODULES { + return self.create_error("modules count out of bounds"); + } + self.module_type_at(self.def(type_index))?; + Ok(()) + } + ImportSectionEntryType::Instance(type_index) => { + if self.cur_module().resources.instance_type_indices.len() >= MAX_WASM_INSTANCES { + return self.create_error("instance count out of bounds"); + } + self.instance_type_at(self.def(type_index))?; + Ok(()) + } } } @@ -329,6 +382,7 @@ impl<'a> ValidatingParser<'a> { "constant expression required: type mismatch: only one init_expr operator is expected", ); } + let expected_ty = state.ty; let ty = match operator { Operator::I32Const { .. } => Type::I32, Operator::I64Const { .. } => Type::I64, @@ -351,7 +405,7 @@ impl<'a> ValidatingParser<'a> { return self .create_error("unknown global: init_expr global index out of bounds"); } - self.resources.globals[global_index as usize].content_type + self.get_global(self.def(global_index))?.item.content_type } Operator::RefFunc { function_index } => { if function_index as usize >= state.function_count { @@ -360,7 +414,10 @@ impl<'a> ValidatingParser<'a> { function_index )); } - self.resources.function_references.insert(function_index); + self.cur_module_mut() + .resources + .function_references + .insert(function_index); Type::FuncRef } _ => { @@ -368,7 +425,7 @@ impl<'a> ValidatingParser<'a> { .create_error("constant expression required: invalid init_expr operator") } }; - if ty != state.ty { + if ty != expected_ty { return self.create_error("type mismatch: invalid init_expr type"); } Ok(()) @@ -380,67 +437,198 @@ impl<'a> ValidatingParser<'a> { kind: ExternalKind, index: u32, ) -> ValidatorResult<'a, ()> { - if self.exported_names.contains(field) { + if self.cur_module().exported_names.contains(field) { return self.create_error("duplicate export name"); } - match kind { - ExternalKind::Function => { - if index as usize >= self.resources.func_type_indices.len() { - return self - .create_error("unknown function: exported function index out of bounds"); - } - self.resources.function_references.insert(index); - } - ExternalKind::Table => { - if index as usize >= self.resources.tables.len() { - return self.create_error("unknown table: exported table index out of bounds"); - } - } - ExternalKind::Memory => { - if index as usize >= self.resources.memories.len() { - return self - .create_error("unknown memory: exported memory index out of bounds"); - } - } - ExternalKind::Global => { - if index as usize >= self.resources.globals.len() { - return self - .create_error("unknown global: exported global index out of bounds"); - } - } - }; + if let ExternalKind::Type = kind { + return self.create_error("cannot export types"); + } + self.check_external_kind("exported", kind, index)?; Ok(()) } - fn check_start(&self, func_index: u32) -> ValidatorResult<'a, ()> { - if func_index as usize >= self.resources.func_type_indices.len() { - return self.create_error("unknown function: start function index out of bounds"); + fn check_external_kind( + &mut self, + desc: &str, + kind: ExternalKind, + index: u32, + ) -> ValidatorResult<'a, ()> { + let module = self.cur_module_mut(); + let (ty, total) = match kind { + ExternalKind::Function => ("function", module.resources.func_type_indices.len()), + ExternalKind::Table => ("table", module.resources.tables.len()), + ExternalKind::Memory => ("memory", module.resources.memories.len()), + ExternalKind::Global => ("global", module.resources.globals.len()), + ExternalKind::Module => ("module", module.resources.module_type_indices.len()), + ExternalKind::Instance => ("instance", module.resources.instance_type_indices.len()), + ExternalKind::Type => return self.create_error("cannot export types"), + }; + if index as usize >= total { + return self.create_error(&format!( + "unknown {0}: {1} {0} index out of bounds", + ty, desc + )); } - let type_index = self.resources.func_type_indices[func_index as usize]; - let ty = &self.resources.types[type_index as usize]; - if !ty.params.is_empty() || !ty.returns.is_empty() { + if let ExternalKind::Function = kind { + module.resources.function_references.insert(index); + } + Ok(()) + } + + fn def(&self, item: T) -> Def { + Def { + module: self.modules.len() - 1, + item, + } + } + + fn current_func_index(&self) -> Def { + let module = &self.cur_module(); + self.def(module.current_func_index + module.func_nonlocal_count) + } + + fn get<'me, T>( + &'me self, + idx: Def, + desc: &str, + get_list: impl FnOnce(&'me ValidatingParserResources<'a>) -> &'me [T], + ) -> ValidatorResult<'a, &'me T> { + match get_list(&self.modules[idx.module].resources).get(idx.item as usize) { + Some(ty) => Ok(ty), + None => self.create_error(&format!("unknown {0}: {0} index out of bounds", desc)), + } + } + + fn get_type<'me>(&'me self, mut idx: Def) -> ValidatorResult<'a, Def<&'me TypeDef<'a>>> { + loop { + let def = self.get(idx, "type", |v| &v.types)?; + match def { + ValidatedType::Def(item) => { + break Ok(Def { + module: idx.module, + item, + }) + } + ValidatedType::Alias(other) => idx = *other, + } + } + } + + fn get_table<'me>(&'me self, idx: Def) -> ValidatorResult<'a, &'me Def> { + self.get(idx, "table", |v| &v.tables) + } + + fn get_memory<'me>(&'me self, idx: Def) -> ValidatorResult<'a, &'me MemoryType> { + self.get(idx, "memory", |v| &v.memories) + } + + fn get_global<'me>(&'me self, idx: Def) -> ValidatorResult<'a, &'me Def> { + self.get(idx, "global", |v| &v.globals) + } + + fn get_func_type_index<'me>(&'me self, idx: Def) -> ValidatorResult<'a, Def> { + Ok(*self.get(idx, "func", |v| &v.func_type_indices)?) + } + + fn get_module_type_index<'me>(&'me self, idx: Def) -> ValidatorResult<'a, Def> { + Ok(*self.get(idx, "module", |v| &v.module_type_indices)?) + } + + fn get_instance_def<'me>( + &'me self, + idx: Def, + ) -> ValidatorResult<'a, &'me Def> { + self.get(idx, "module", |v| &v.instance_type_indices) + } + + fn func_type_at<'me>( + &'me self, + type_index: Def, + ) -> ValidatorResult<'a, Def<&'me FuncType>> { + let def = self.get_type(type_index)?; + match &def.item { + TypeDef::Func(item) => Ok(Def { + module: def.module, + item, + }), + _ => self.create_error("type index is not a function"), + } + } + + fn module_type_at<'me>( + &'me self, + type_index: Def, + ) -> ValidatorResult<'a, Def<&'me ModuleType<'a>>> { + if !self.config.operator_config.enable_module_linking { + return self.create_error("module linking proposal not enabled"); + } + let ty = self.get_type(type_index)?; + match &ty.item { + TypeDef::Module(item) => Ok(Def { + module: ty.module, + item, + }), + _ => self.create_error("type index is not a module"), + } + } + + fn instance_type_at<'me>( + &'me self, + type_index: Def, + ) -> ValidatorResult<'a, Def<&'me InstanceType<'a>>> { + if !self.config.operator_config.enable_module_linking { + return self.create_error("module linking proposal not enabled"); + } + let def = self.get_type(type_index)?; + match &def.item { + TypeDef::Instance(item) => Ok(Def { + module: def.module, + item, + }), + _ => self.create_error("type index is not an instance"), + } + } + + fn check_start(&self, func_index: u32) -> ValidatorResult<'a, ()> { + let ty = match self.get_func_type_index(self.def(func_index)) { + Ok(ty) => self.func_type_at(ty)?, + Err(_) => { + return self.create_error("unknown function: start function index out of bounds") + } + }; + if !ty.item.params.is_empty() || !ty.item.returns.is_empty() { return self.create_error("invlid start function type"); } Ok(()) } fn process_begin_section(&self, code: &SectionCode) -> ValidatorResult<'a, SectionOrderState> { - let order_state = SectionOrderState::from_section_code(code); - Ok(match self.section_order_state { - SectionOrderState::Initial => match order_state { - Some(section) => section, - _ => SectionOrderState::Initial, - }, - previous => { - if let Some(order_state_unwraped) = order_state { - if previous >= order_state_unwraped { - return self.create_error("section out of order"); - } - order_state_unwraped - } else { - previous - } + use SectionOrderState::*; + + let state = SectionOrderState::from_section_code(code, &self.config); + let state = match state { + Some(state) => state, + None => return Ok(self.cur_module().section_order_state), + }; + Ok(match self.cur_module().section_order_state { + // Did we just start? In that case move to our newly-found state. + Initial => state, + + // If our previous state comes before our current state, nothing to + // worry about, just advance ourselves. + previous if previous < state => state, + + // In the module linking proposal we can see this state multiple + // times in a row. + ModuleLinkingHeader + if state == ModuleLinkingHeader + && self.config.operator_config.enable_module_linking => + { + ModuleLinkingHeader } + + // otherwise the sections are out of order + _ => return self.create_error("section out of order"), }) } @@ -456,17 +644,22 @@ impl<'a> ValidatingParser<'a> { if check.is_err() { self.validation_error = check.err(); } else { - self.section_order_state = check.ok().unwrap(); + self.cur_module_mut().section_order_state = check.ok().unwrap(); } } - ParserState::TypeSectionEntry(ref func_type) => { - let check = self.check_func_type(func_type); + ParserState::TypeSectionEntry(ref def) => { + let check = match def { + TypeDef::Func(ty) => self.check_func_type(ty), + TypeDef::Instance(ty) => self.check_instance_type(ty), + TypeDef::Module(ty) => self.check_module_type(ty), + }; if check.is_err() { self.validation_error = check.err(); - } else if self.resources.types.len() > MAX_WASM_TYPES { + } else if self.cur_module().resources.types.len() > MAX_WASM_TYPES { self.set_validation_error("types count is out of bounds"); } else { - self.resources.types.push(func_type.clone()); + let def = ValidatedType::Def(def.clone()); + self.cur_module_mut().resources.types.push(def); } } ParserState::ImportSectionEntry { ref ty, .. } => { @@ -476,62 +669,88 @@ impl<'a> ValidatingParser<'a> { } else { match *ty { ImportSectionEntryType::Function(type_index) => { - self.func_imports_count += 1; - self.resources.func_type_indices.push(type_index); + let def = self.def(type_index); + self.cur_module_mut().resources.func_type_indices.push(def); + self.cur_module_mut().func_nonlocal_count += 1; } ImportSectionEntryType::Table(ref table_type) => { - self.resources.tables.push(table_type.clone()); + let def = self.def(table_type.clone()); + self.cur_module_mut().resources.tables.push(def); } ImportSectionEntryType::Memory(ref memory_type) => { - self.resources.memories.push(memory_type.clone()); + let ty = memory_type.clone(); + self.cur_module_mut().resources.memories.push(ty); } ImportSectionEntryType::Global(ref global_type) => { - self.resources.globals.push(global_type.clone()); + let def = self.def(global_type.clone()); + self.cur_module_mut().resources.globals.push(def); + } + + ImportSectionEntryType::Instance(type_index) => { + let def = self.def(InstanceDef::Imported { + type_idx: type_index, + }); + self.cur_module_mut() + .resources + .instance_type_indices + .push(def); + } + ImportSectionEntryType::Module(type_index) => { + let def = self.def(type_index); + self.cur_module_mut() + .resources + .module_type_indices + .push(def); } } } } ParserState::FunctionSectionEntry(type_index) => { - if type_index as usize >= self.resources.types.len() { - self.set_validation_error("unknown type: func type index out of bounds"); - } else if self.resources.func_type_indices.len() >= MAX_WASM_FUNCTIONS { + let type_index = self.def(type_index); + if self.cur_module().resources.func_type_indices.len() >= MAX_WASM_FUNCTIONS { self.set_validation_error("functions count out of bounds"); + } else if let Err(err) = self.func_type_at(type_index) { + self.validation_error = Some(err); } else { - self.resources.func_type_indices.push(type_index); + self.cur_module_mut() + .resources + .func_type_indices + .push(type_index); } } ParserState::TableSectionEntry(ref table_type) => { if !self.config.operator_config.enable_reference_types - && self.resources.tables.len() >= MAX_WASM_TABLES + && !self.config.operator_config.enable_module_linking + && self.cur_module().resources.tables.len() >= MAX_WASM_TABLES { self.set_validation_error("multiple tables: tables count must be at most 1"); } else { self.validation_error = self.check_table_type(table_type).err(); - self.resources.tables.push(table_type.clone()); + let def = self.def(table_type.clone()); + self.cur_module_mut().resources.tables.push(def); } } ParserState::MemorySectionEntry(ref memory_type) => { - if self.resources.memories.len() >= MAX_WASM_MEMORIES { + if !self.config.operator_config.enable_module_linking + && self.cur_module().resources.memories.len() >= MAX_WASM_MEMORIES + { self.set_validation_error( "multiple memories: memories count must be at most 1", ); } else { self.validation_error = self.check_memory_type(memory_type).err(); - self.resources.memories.push(memory_type.clone()); + let ty = memory_type.clone(); + self.cur_module_mut().resources.memories.push(ty); } } ParserState::BeginGlobalSectionEntry(global_type) => { - if self.resources.globals.len() >= MAX_WASM_GLOBALS { + if self.cur_module().resources.globals.len() >= MAX_WASM_GLOBALS { self.set_validation_error("globals count out of bounds"); } else { self.validation_error = self.check_global_type(global_type).err(); - self.init_expression_state = Some(InitExpressionState { - ty: global_type.content_type, - global_count: self.resources.globals.len(), - function_count: self.resources.func_type_indices.len(), - validated: false, - }); - self.resources.globals.push(global_type); + self.set_init_expression_state(global_type.content_type); + let def = self.def(global_type); + self.cur_module_mut().resources.globals.push(def); } } ParserState::BeginInitExpressionBody => { @@ -550,37 +769,34 @@ impl<'a> ValidatingParser<'a> { } ParserState::ExportSectionEntry { field, kind, index } => { self.validation_error = self.check_export_entry(field, kind, index).err(); - self.exported_names.insert(String::from(field)); + self.cur_module_mut() + .exported_names + .insert(String::from(field)); } ParserState::StartSectionEntry(func_index) => { self.validation_error = self.check_start(func_index).err(); } ParserState::DataCountSectionEntry(count) => { - self.resources.data_count = Some(count); + self.cur_module_mut().resources.data_count = Some(count); } ParserState::BeginElementSectionEntry { table, ty } => { - self.resources.element_types.push(ty); + self.cur_module_mut().resources.element_types.push(ty); match table { ElemSectionEntryTable::Active(table_index) => { - let table = match self.resources.tables.get(table_index as usize) { - Some(t) => t, - None => { + let table = match self.get_table(self.def(table_index)) { + Ok(table) => table, + Err(_) => { self.set_validation_error( "unknown table: element section table index out of bounds", ); return; } }; - if ty != table.element_type { + if ty != table.item.element_type { self.set_validation_error("element_type != table type"); return; } - self.init_expression_state = Some(InitExpressionState { - ty: Type::I32, - global_count: self.resources.globals.len(), - function_count: self.resources.func_type_indices.len(), - validated: false, - }); + self.set_init_expression_state(Type::I32); } ElemSectionEntryTable::Passive | ElemSectionEntryTable::Declared => { if !self.config.operator_config.enable_bulk_memory { @@ -605,30 +821,35 @@ impl<'a> ValidatingParser<'a> { } } ParserState::ElementSectionEntryBody(ref indices) => { + let mut references = Vec::with_capacity(indices.len()); for item in &**indices { - if let ElementItem::Func(func_index) = item { - if *func_index as usize >= self.resources.func_type_indices.len() { + if let ElementItem::Func(func_index) = *item { + if self.get_func_type_index(self.def(func_index)).is_err() { self.set_validation_error( "unknown function: element func index out of bounds", ); break; } - self.resources.function_references.insert(*func_index); + references.push(func_index); } } + self.cur_module_mut() + .resources + .function_references + .extend(references); } ParserState::BeginFunctionBody { .. } => { - let index = (self.current_func_index + self.func_imports_count) as usize; - if index as usize >= self.resources.func_type_indices.len() { + let index = self.current_func_index(); + if self.get_func_type_index(index).is_err() { self.set_validation_error("func type is not defined"); } } ParserState::FunctionBodyLocals { ref locals } => { - let index = (self.current_func_index + self.func_imports_count) as usize; - let func_type = - &self.resources.types[self.resources.func_type_indices[index] as usize]; + let index = self.current_func_index(); + let func_type = self.get_func_type_index(index).unwrap(); + let func_type = self.func_type_at(func_type).unwrap(); let operator_config = self.config.operator_config; - match OperatorValidator::new(func_type, locals, operator_config) { + match OperatorValidator::new(func_type.item, locals, operator_config) { Ok(validator) => self.current_operator_validator = Some(validator), Err(err) => { self.validation_error = Some(ParserState::Error( @@ -638,11 +859,9 @@ impl<'a> ValidatingParser<'a> { } } ParserState::CodeOperator(ref operator) => { - let check = self - .current_operator_validator - .as_mut() - .unwrap() - .process_operator(operator, &self.resources); + let mut validator = self.current_operator_validator.take().unwrap(); + let check = validator.process_operator(operator, self); + self.current_operator_validator = Some(validator); if let Err(err) = check { self.set_operator_validation_error(err); @@ -651,50 +870,164 @@ impl<'a> ValidatingParser<'a> { ParserState::EndFunctionBody => { let check = self .current_operator_validator - .as_ref() + .take() .unwrap() .process_end_function(); if let Err(err) = check { self.set_operator_validation_error(err); } - self.current_func_index += 1; - self.current_operator_validator = None; + self.cur_module_mut().current_func_index += 1; } ParserState::BeginDataSectionEntryBody(_) => { - self.data_found += 1; + self.cur_module_mut().data_found += 1; } ParserState::BeginActiveDataSectionEntry(memory_index) => { - if memory_index as usize >= self.resources.memories.len() { + if self.get_memory(self.def(memory_index)).is_err() { self.set_validation_error( "unknown memory: data section memory index out of bounds", ); } else { - self.init_expression_state = Some(InitExpressionState { - ty: Type::I32, - global_count: self.resources.globals.len(), - function_count: self.resources.func_type_indices.len(), - validated: false, - }); + self.set_init_expression_state(Type::I32); } } ParserState::EndWasm => { - if self.resources.func_type_indices.len() - != self.current_func_index as usize + self.func_imports_count as usize - { + let current_func = self.current_func_index(); + let module = &mut self.cur_module(); + if module.resources.func_type_indices.len() != current_func.item as usize { self.set_validation_error( "function and code section have inconsistent lengths", ); + return; } - if let Some(data_count) = self.resources.data_count { - if data_count != self.data_found { + if let Some(data_count) = module.resources.data_count { + if data_count != module.data_found { self.set_validation_error("data count section and passive data mismatch"); } + return; + } + if self.modules.len() > 1 { + // Pop our nested module from the stack since it's no longer + // needed + self.modules.pop(); + + // Restore the parser back to the previous state + mem::swap( + &mut self.parser, + &mut self.modules.last_mut().unwrap().parser, + ); + } + } + + ParserState::ModuleSectionEntry(type_index) => { + if !self.config.operator_config.enable_module_linking { + self.set_validation_error("module linking proposal not enabled"); + } else if self.cur_module().resources.module_type_indices.len() >= MAX_WASM_MODULES + { + self.set_validation_error("modules count out of bounds"); + } else { + let type_index = self.def(type_index); + match self.module_type_at(type_index) { + Ok(_) => self + .cur_module_mut() + .resources + .module_type_indices + .push(type_index), + Err(e) => self.validation_error = Some(e), + } } } + ParserState::BeginInstantiate { module, count } => { + if !self.config.operator_config.enable_module_linking { + self.set_validation_error("module linking proposal not enabled"); + } else if self.cur_module().resources.instance_type_indices.len() + >= MAX_WASM_INSTANCES + { + self.set_validation_error("instance count out of bounds"); + } else { + let def = self.def(InstanceDef::Instantiated { module_idx: module }); + self.cur_module_mut() + .resources + .instance_type_indices + .push(def); + let module_ty = match self.get_module_type_index(self.def(module)) { + Ok(ty) => ty, + Err(e) => { + self.validation_error = Some(e); + return; + } + }; + let ty = self.module_type_at(module_ty).unwrap(); + if count as usize != ty.item.imports.len() { + self.set_validation_error("wrong number of imports provided"); + } else { + self.module_instantiation = Some((module_ty, 0)); + } + } + } + ParserState::InstantiateParameter { kind, index } => { + let (module_ty_idx, import_idx) = self.module_instantiation.take().unwrap(); + let module_ty = self.module_type_at(module_ty_idx).unwrap(); + let ty = module_ty.item.imports[import_idx].ty.clone(); + let ty = module_ty.map(|_| &ty); + match self.check_instantiate_field(ty, kind, index) { + Ok(()) => { + self.module_instantiation = Some((module_ty_idx, import_idx + 1)); + } + Err(e) => self.validation_error = Some(e), + } + } + ParserState::EndInstantiate => { + let (module_ty, import_idx) = self.module_instantiation.take().unwrap(); + let module_ty = self.module_type_at(module_ty).unwrap(); + if import_idx != module_ty.item.imports.len() { + self.set_validation_error("not enough imports provided"); + } + } + ParserState::AliasSectionEntry(ref alias) => { + let instance_idx = match alias.instance { + AliasedInstance::Parent => None, + AliasedInstance::Child(instance_idx) => Some(instance_idx), + }; + let (kind, index) = (alias.kind, alias.index); + match self.check_alias_entry(instance_idx, kind, index) { + Ok(()) => {} + Err(e) => self.validation_error = Some(e), + } + } + ParserState::InlineModule(ref module) => { + let parser = match module.module() { + Ok(m) => m, + Err(e) => { + self.validation_error = Some(ParserState::Error(e)); + return; + } + }; + self.total_nested_modules += 1; + if self.total_nested_modules > MAX_WASM_MODULES { + self.set_validation_error("too many nested modules"); + } + + // Save the state of our parser in our module + let old_parser = mem::replace(&mut self.parser, parser.into()); + self.cur_module_mut().parser = old_parser; + + // Then allocate a child module and push it onto our stack of + // modules we're validating. + self.modules.push(Module::default()); + } _ => (), }; } + fn set_init_expression_state(&mut self, ty: Type) { + self.init_expression_state = Some(InitExpressionState { + ty, + global_count: self.cur_module().resources.globals.len(), + function_count: self.cur_module().resources.func_type_indices.len(), + validated: false, + }); + } + pub fn create_validating_operator_parser<'b>( &mut self, ) -> ValidatorResult> @@ -708,11 +1041,10 @@ impl<'a> ValidatingParser<'a> { self.read(); let operator_validator = match *self.last_state() { ParserState::FunctionBodyLocals { ref locals } => { - let index = (self.current_func_index + self.func_imports_count) as usize; - let func_type = - &self.resources.types[self.resources.func_type_indices[index] as usize]; + let index = self.current_func_index(); + let func_type = self.func_type_at(self.get_func_type_index(index)?).unwrap(); let operator_config = self.config.operator_config; - OperatorValidator::new(func_type, locals, operator_config) + OperatorValidator::new(func_type.item, locals, operator_config) .map_err(|e| ParserState::Error(e.set_offset(self.read_position.unwrap())))? } _ => panic!("Invalid reader state"), @@ -728,6 +1060,306 @@ impl<'a> ValidatingParser<'a> { pub fn current_position(&self) -> usize { self.parser.current_position() } + + fn check_instantiate_field( + &mut self, + expected: Def<&ImportSectionEntryType>, + kind: ExternalKind, + index: u32, + ) -> ValidatorResult<'a, ()> { + let index = self.def(index); + let actual = match kind { + ExternalKind::Function => self + .get_func_type_index(index)? + .map(ImportSectionEntryType::Function), + ExternalKind::Table => self.get_table(index)?.map(ImportSectionEntryType::Table), + ExternalKind::Memory => { + self.def(ImportSectionEntryType::Memory(*self.get_memory(index)?)) + } + ExternalKind::Global => self.get_global(index)?.map(ImportSectionEntryType::Global), + ExternalKind::Module => self + .get_module_type_index(index)? + .map(ImportSectionEntryType::Module), + ExternalKind::Instance => { + let def = self.get_instance_def(index)?; + match def.item { + InstanceDef::Imported { type_idx } => def + .as_ref() + .map(|_| ImportSectionEntryType::Instance(type_idx)), + InstanceDef::Instantiated { module_idx } => { + let expected = match expected.item { + ImportSectionEntryType::Instance(idx) => expected.map(|_| *idx), + _ => { + return self.create_error("wrong kind of item used for instantiate") + } + }; + let expected = self.instance_type_at(expected)?; + let module_idx = def.as_ref().map(|_| module_idx); + let actual = self.get_module_type_index(module_idx)?; + let actual = self.module_type_at(actual)?; + return self.check_export_sets_match( + expected.map(|m| &*m.exports), + actual.map(|m| &*m.exports), + ); + } + } + } + ExternalKind::Type => return self.create_error("cannot export types"), + }; + let item = actual.item; + self.check_imports_match(expected, actual.map(|_| &item)) + } + + // Note that this function is basically implementing + // https://webassembly.github.io/spec/core/exec/modules.html#import-matching + fn check_imports_match( + &self, + expected: Def<&ImportSectionEntryType>, + actual: Def<&ImportSectionEntryType>, + ) -> ValidatorResult<'a, ()> { + let limits_match = |expected: &ResizableLimits, actual: &ResizableLimits| { + actual.initial >= expected.initial + && match expected.maximum { + Some(expected_max) => match actual.maximum { + Some(actual_max) => actual_max <= expected_max, + None => false, + }, + None => true, + } + }; + match (expected.item, actual.item) { + ( + ImportSectionEntryType::Function(expected_idx), + ImportSectionEntryType::Function(actual_idx), + ) => { + let expected = self.func_type_at(expected.map(|_| *expected_idx))?; + let actual = self.func_type_at(actual.map(|_| *actual_idx))?; + if actual.item == expected.item { + return Ok(()); + } + self.create_error("function provided for instantiation has wrong type") + } + (ImportSectionEntryType::Table(expected), ImportSectionEntryType::Table(actual)) => { + if expected.element_type == actual.element_type + && limits_match(&expected.limits, &actual.limits) + { + return Ok(()); + } + self.create_error("table provided for instantiation has wrong type") + } + (ImportSectionEntryType::Memory(expected), ImportSectionEntryType::Memory(actual)) => { + if limits_match(&expected.limits, &actual.limits) + && expected.shared == actual.shared + { + return Ok(()); + } + self.create_error("memory provided for instantiation has wrong type") + } + (ImportSectionEntryType::Global(expected), ImportSectionEntryType::Global(actual)) => { + if expected == actual { + return Ok(()); + } + self.create_error("global provided for instantiation has wrong type") + } + ( + ImportSectionEntryType::Instance(expected_idx), + ImportSectionEntryType::Instance(actual_idx), + ) => { + let expected = self.instance_type_at(expected.map(|_| *expected_idx))?; + let actual = self.instance_type_at(actual.map(|_| *actual_idx))?; + self.check_export_sets_match( + expected.map(|i| &*i.exports), + actual.map(|i| &*i.exports), + )?; + Ok(()) + } + ( + ImportSectionEntryType::Module(expected_idx), + ImportSectionEntryType::Module(actual_idx), + ) => { + let expected = self.module_type_at(expected.map(|_| *expected_idx))?; + let actual = self.module_type_at(actual.map(|_| *actual_idx))?; + if expected.item.imports.len() != actual.item.imports.len() { + return self.create_error("mismatched number of module imports"); + } + for (a, b) in expected.item.imports.iter().zip(actual.item.imports.iter()) { + self.check_imports_match(expected.map(|_| &a.ty), actual.map(|_| &b.ty))?; + } + self.check_export_sets_match( + expected.map(|i| &*i.exports), + actual.map(|i| &*i.exports), + )?; + Ok(()) + } + _ => self.create_error("wrong kind of item used for instantiate"), + } + } + + fn check_export_sets_match( + &self, + expected: Def<&[ExportType<'_>]>, + actual: Def<&[ExportType<'_>]>, + ) -> ValidatorResult<'a, ()> { + let name_to_idx = actual + .item + .iter() + .enumerate() + .map(|(i, e)| (e.name, i)) + .collect::>(); + for expected_export in expected.item { + let idx = match name_to_idx.get(expected_export.name) { + Some(i) => *i, + None => { + return self + .create_error(&format!("no export named `{}`", expected_export.name)) + } + }; + self.check_imports_match( + expected.map(|_| &expected_export.ty), + actual.map(|_| &actual.item[idx].ty), + )?; + } + Ok(()) + } + + fn check_alias_entry( + &mut self, + instance_idx: Option, + kind: ExternalKind, + idx: u32, + ) -> ValidatorResult<'a, ()> { + match instance_idx { + Some(instance_idx) => { + let ty = self.get_instance_def(self.def(instance_idx))?; + let exports = match ty.item { + InstanceDef::Imported { type_idx } => { + let ty = self.instance_type_at(ty.as_ref().map(|_| type_idx))?; + ty.map(|t| &t.exports) + } + InstanceDef::Instantiated { module_idx } => { + let ty = self.get_module_type_index(ty.as_ref().map(|_| module_idx))?; + let ty = self.module_type_at(ty)?; + ty.map(|t| &t.exports) + } + }; + let export = match exports.item.get(idx as usize) { + Some(e) => e, + None => { + return self.create_error("aliased export index out of bounds"); + } + }; + match (export.ty, kind) { + (ImportSectionEntryType::Function(ty), ExternalKind::Function) => { + let def = exports.map(|_| ty); + self.cur_module_mut().resources.func_type_indices.push(def); + self.cur_module_mut().func_nonlocal_count += 1; + } + (ImportSectionEntryType::Table(ty), ExternalKind::Table) => { + let def = exports.map(|_| ty); + self.cur_module_mut().resources.tables.push(def); + } + (ImportSectionEntryType::Memory(ty), ExternalKind::Memory) => { + self.cur_module_mut().resources.memories.push(ty); + } + (ImportSectionEntryType::Global(ty), ExternalKind::Global) => { + let def = exports.map(|_| ty); + self.cur_module_mut().resources.globals.push(def); + } + (ImportSectionEntryType::Instance(ty), ExternalKind::Instance) => { + let def = exports.map(|_| InstanceDef::Imported { type_idx: ty }); + self.cur_module_mut() + .resources + .instance_type_indices + .push(def); + } + (ImportSectionEntryType::Module(ty), ExternalKind::Module) => { + let def = exports.map(|_| ty); + self.cur_module_mut() + .resources + .module_type_indices + .push(def); + } + _ => return self.create_error("alias kind mismatch with export kind"), + } + } + None => { + let idx = match self.modules.len().checked_sub(2) { + None => return self.create_error("no parent module to alias from"), + Some(module) => Def { module, item: idx }, + }; + match kind { + ExternalKind::Module => { + let ty = self.get_module_type_index(idx)?; + self.cur_module_mut().resources.module_type_indices.push(ty); + } + ExternalKind::Type => { + // make sure this type actually exists, then push it as + // ourselve aliasing that type. + self.get_type(idx)?; + self.cur_module_mut() + .resources + .types + .push(ValidatedType::Alias(idx)); + } + _ => return self.create_error("only parent types/modules can be aliased"), + } + } + } + + Ok(()) + } +} + +impl<'a> WasmModuleResources for ValidatingParser<'a> { + type TypeDef = crate::TypeDef<'a>; + type TableType = crate::TableType; + type MemoryType = crate::MemoryType; + type GlobalType = crate::GlobalType; + + fn type_at(&self, at: u32) -> Option<&Self::TypeDef> { + self.get_type(self.def(at)).ok().map(|t| t.item) + } + + fn table_at(&self, at: u32) -> Option<&Self::TableType> { + self.get_table(self.def(at)).ok().map(|t| &t.item) + } + + fn memory_at(&self, at: u32) -> Option<&Self::MemoryType> { + self.get_memory(self.def(at)).ok() + } + + fn global_at(&self, at: u32) -> Option<&Self::GlobalType> { + self.get_global(self.def(at)).ok().map(|t| &t.item) + } + + fn func_type_at(&self, at: u32) -> Option<&FuncType> { + let ty = self.get_func_type_index(self.def(at)).ok()?; + let ty = self.func_type_at(ty).ok()?; + Some(ty.item) + } + + fn element_type_at(&self, at: u32) -> Option { + self.cur_module() + .resources + .element_types + .get(at as usize) + .cloned() + } + + fn element_count(&self) -> u32 { + self.cur_module().resources.element_types.len() as u32 + } + + fn data_count(&self) -> u32 { + self.cur_module().resources.data_count.unwrap_or(0) + } + + fn is_function_referenced(&self, idx: u32) -> bool { + self.cur_module() + .resources + .function_references + .contains(&idx) + } } impl<'a> WasmDecoder<'a> for ValidatingParser<'a> { @@ -746,7 +1378,7 @@ impl<'a> WasmDecoder<'a> for ValidatingParser<'a> { ParserInput::SkipSection => panic!("Not supported"), ParserInput::ReadSectionRawData => panic!("Not supported"), ParserInput::SkipFunctionBody => { - self.current_func_index += 1; + self.cur_module_mut().current_func_index += 1; self.parser.push_input(input); } _ => self.parser.push_input(input), @@ -828,13 +1460,11 @@ impl<'b> ValidatingOperatorParser<'b> { /// let mut parser = ValidatingParser::new(data, None); /// let mut i = 0; /// loop { - /// { - /// match *parser.read() { - /// ParserState::Error(_) | - /// ParserState::EndWasm => break, - /// ParserState::BeginFunctionBody {..} => (), - /// _ => continue - /// } + /// match parser.read() { + /// ParserState::Error(_) | + /// ParserState::EndWasm => break, + /// ParserState::BeginFunctionBody {..} => (), + /// _ => continue /// } /// let mut reader = parser /// .create_validating_operator_parser() @@ -842,7 +1472,7 @@ impl<'b> ValidatingOperatorParser<'b> { /// println!("Function {}", i); /// i += 1; /// while !reader.eof() { - /// let read = reader.next(parser.get_resources()); + /// let read = reader.next(&parser); /// if let Ok(ref op) = read { /// println!(" {:?}", op); /// } else { @@ -851,20 +1481,12 @@ impl<'b> ValidatingOperatorParser<'b> { /// } /// } /// ``` - pub fn next<'c, F: WasmFuncType, T: WasmTableType, M: WasmMemoryType, G: WasmGlobalType>( - &mut self, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, - ) -> Result> + pub fn next<'c>(&mut self, resources: impl WasmModuleResources) -> Result> where 'b: 'c, { let op = self.reader.read_operator()?; - match self.operator_validator.process_operator(&op, resources) { + match self.operator_validator.process_operator(&op, &resources) { Err(err) => { let offset = self.func_body_offset + self.reader.current_position(); return Err(err.set_offset(offset)); @@ -886,21 +1508,11 @@ impl<'b> ValidatingOperatorParser<'b> { /// Test whether the given buffer contains a valid WebAssembly function. /// The resources parameter contains all needed data to validate the operators. -pub fn validate_function_body< - F: WasmFuncType, - T: WasmTableType, - M: WasmMemoryType, - G: WasmGlobalType, ->( +pub fn validate_function_body( bytes: &[u8], offset: usize, func_index: u32, - resources: &dyn WasmModuleResources< - FuncType = F, - TableType = T, - MemoryType = M, - GlobalType = G, - >, + resources: impl WasmModuleResources, operator_config: Option, ) -> Result<()> { let operator_config = operator_config.unwrap_or(DEFAULT_OPERATOR_VALIDATOR_CONFIG); @@ -929,18 +1541,12 @@ pub fn validate_function_body< locals.push((count, ty)); } let operators_reader = function_body.get_operators_reader()?; - let func_type_index = resources - .func_type_id_at(func_index) + let func_type = resources + .func_type_at(func_index) // Note: This was an out-of-bounds access before the change to return `Option` // so I assumed it is considered a bug to access a non-existing function // id here and went with panicking instead of returning a proper error. .expect("the function index of the validated function itself is out of bounds"); - let func_type = resources - .type_at(func_type_index) - // Note: This was an out-of-bounds access before the change to return `Option` - // so I assumed it is considered a bug to access a non-existing function - // id here and went with panicking instead of returning a proper error. - .expect("the function type indexof the validated function itself is out of bounds"); let mut operator_validator = OperatorValidator::new(func_type, &locals, operator_config) .map_err(|e| e.set_offset(offset))?; let mut eof_found = false; @@ -948,7 +1554,7 @@ pub fn validate_function_body< for item in operators_reader.into_iter_with_offsets() { let (ref op, offset) = item?; match operator_validator - .process_operator(op, resources) + .process_operator(op, &resources) .map_err(|e| e.set_offset(offset))? { FunctionEnd::Yes => { @@ -978,8 +1584,10 @@ pub fn validate(bytes: &[u8], config: Option) -> Result< ParserState::EndWasm => break, ParserState::Error(ref e) => return Err(e.clone()), ParserState::BeginFunctionBody { range } => { - parser_input = Some(ParserInput::SkipFunctionBody); - func_ranges.push(range); + if parser.modules.len() == 1 { + parser_input = Some(ParserInput::SkipFunctionBody); + func_ranges.push(range); + } } _ => (), } @@ -987,12 +1595,12 @@ pub fn validate(bytes: &[u8], config: Option) -> Result< let operator_config = config.map(|c| c.operator_config); for (i, range) in func_ranges.into_iter().enumerate() { let function_body = range.slice(bytes); - let function_index = i as u32 + parser.func_imports_count; + let function_index = i as u32 + parser.modules[0].func_nonlocal_count; validate_function_body( function_body, range.start, function_index, - &parser.resources, + &parser, operator_config, )?; }