diff --git a/codegen/luajit/runtime/runtime.lua b/codegen/luajit/runtime/runtime.lua index 3489a6b..87fdd04 100644 --- a/codegen/luajit/runtime/runtime.lua +++ b/codegen/luajit/runtime/runtime.lua @@ -754,6 +754,19 @@ do ffi.copy(start, data, len or #data) end + function store.copy(memory_1, addr_1, memory_2, addr_2, len) + local start_1 = by_offset(memory_1.data, addr_1) + local start_2 = by_offset(memory_2.data, addr_2) + + ffi.copy(start_1, start_2, len) + end + + function store.fill(memory, addr, len, value) + local start = by_offset(memory.data, addr) + + ffi.fill(start, len, value) + end + local WASM_PAGE_SIZE = 65536 local function finalizer(memory) @@ -795,19 +808,6 @@ do return old end end - - function allocator.copy(memory, destination, source, length) - local destination_addr = by_offset(memory.data, destination) - local source_addr = by_offset(memory.data, source) - - ffi.copy(destination_addr, source_addr, length) - end - - function allocator.fill(memory, address, value, length) - local start = by_offset(memory.data, address) - - ffi.fill(start, length, value) - end module.load = load module.store = store diff --git a/codegen/luajit/src/backend/statement.rs b/codegen/luajit/src/backend/statement.rs index 38144cc..86e3328 100644 --- a/codegen/luajit/src/backend/statement.rs +++ b/codegen/luajit/src/backend/statement.rs @@ -288,10 +288,14 @@ impl DriverNoContext for MemoryGrow { impl DriverNoContext for MemoryCopy { fn write(&self, w: &mut dyn Write) -> Result<()> { - let dst = self.dst(); - let src = self.src(); + let memory_1 = self.destination().memory(); + let memory_2 = self.source().memory(); - write!(w, "rt.allocator.copy(memory_at_0, {dst}, {src}, ")?; + write!(w, "rt.store.copy(memory_at_{memory_1}, ")?; + self.destination().pointer().write(w)?; + write!(w, ", memory_at_{memory_2}, ")?; + self.source().pointer().write(w)?; + write!(w, ", ")?; self.size().write(w)?; write!(w, ")") } @@ -299,14 +303,14 @@ impl DriverNoContext for MemoryCopy { impl DriverNoContext for MemoryFill { fn write(&self, w: &mut dyn Write) -> Result<()> { - let mem = self.mem(); - let value = self.value(); - let n = self.n(); + let memory = self.destination().memory(); - write!(w, "rt.allocator.fill(memory_at_0, {mem}, ")?; - value.write(w)?; + write!(w, "rt.store.fill(memory_at_{memory}, ")?; + self.destination().pointer().write(w)?; write!(w, ", ")?; - n.write(w)?; + self.size().write(w)?; + write!(w, ", ")?; + self.value().write(w)?; write!(w, ")") } } diff --git a/codegen/luau/runtime/runtime.lua b/codegen/luau/runtime/runtime.lua index ef14f12..dd2c993 100644 --- a/codegen/luau/runtime/runtime.lua +++ b/codegen/luau/runtime/runtime.lua @@ -849,8 +849,6 @@ do store_byte(memory.data, addr, value) end - local store_i8 = store.i32_n8 - function store.i32_n16(memory, addr, value) store_byte(memory.data, addr, value) store_byte(memory.data, addr + 1, bit_rshift(value, 8)) @@ -911,7 +909,7 @@ do end function store.string(memory, addr, data, len) - len = len or #data + len = if len then len else #data local rem = len % 4 @@ -924,7 +922,35 @@ do for i = len - rem + 1, len do local v = string_byte(data, i) - store_i8(memory, addr + i - 1, v) + store_i32_n8(memory, addr + i - 1, v) + end + end + + -- FIXME: `store.copy` and `store.fill` should be ideally using the same batched store as `store.string` + function store.copy(memory_1, addr_1, memory_2, addr_2, len) + local data_1 = memory_1.data + local data_2 = memory_2.data + + if addr_1 <= addr_2 then + for i = 1, len do + local v = load_byte(data_2, addr_2 + i - 1) + + store_byte(data_1, addr_1 + i - 1, v) + end + else + for i = len, 1, -1 do + local v = load_byte(data_2, addr_2 + i - 1) + + store_byte(data_1, addr_1 + i - 1, v) + end + end + end + + function store.fill(memory, addr, len, value) + local data = memory.data + + for i = 1, len do + store_byte(data, addr + i - 1, value) end end @@ -944,20 +970,6 @@ do return old end end - - function allocator.copy(memory, destination, source, length) - for i = 1, length do - local v = load_byte(memory, source + i - 1) - - store_byte(memory, destination + i - 1, v) - end - end - - function allocator.fill(memory, destination, value, length) - for i = 1, length do - store_byte(memory, destination + i - 1, value) - end - end module.load = load module.store = store diff --git a/codegen/luau/src/backend/statement.rs b/codegen/luau/src/backend/statement.rs index 87fd465..2b463e6 100644 --- a/codegen/luau/src/backend/statement.rs +++ b/codegen/luau/src/backend/statement.rs @@ -307,26 +307,29 @@ impl DriverNoContext for MemoryGrow { impl DriverNoContext for MemoryCopy { fn write(&self, w: &mut dyn Write) -> Result<()> { - let dst = self.dst(); - let src = self.src(); - let size = self.size(); + let memory_1 = self.destination().memory(); + let memory_2 = self.source().memory(); - write!(w, "rt.store.copy(memory_at_0, {dst}, {src}, ")?; - size.write(w)?; + write!(w, "rt.store.copy(memory_at_{memory_1}, ")?; + self.destination().pointer().write(w)?; + write!(w, ", memory_at_{memory_2}, ")?; + self.source().pointer().write(w)?; + write!(w, ", ")?; + self.size().write(w)?; write!(w, ")") } } impl DriverNoContext for MemoryFill { fn write(&self, w: &mut dyn Write) -> Result<()> { - let mem = self.mem(); - let value = self.value(); - let n = self.n(); + let memory = self.destination().memory(); - write!(w, "rt.store.copy(memory_at_0, {mem}, ")?; - value.write(w)?; + write!(w, "rt.store.fill(memory_at_{memory}, ")?; + self.destination().pointer().write(w)?; write!(w, ", ")?; - n.write(w)?; + self.size().write(w)?; + write!(w, ", ")?; + self.value().write(w)?; write!(w, ")") } } diff --git a/wasm-ast/src/factory.rs b/wasm-ast/src/factory.rs index dbc93b5..6b3be54 100644 --- a/wasm-ast/src/factory.rs +++ b/wasm-ast/src/factory.rs @@ -4,9 +4,9 @@ use crate::{ module::{read_checked, TypeInfo}, node::{ BinOp, BinOpType, Block, Br, BrIf, BrTable, Call, CallIndirect, CmpOp, CmpOpType, - Expression, FuncData, GetGlobal, GetLocal, If, LabelType, LoadAt, LoadType, MemoryCopy, - MemoryFill, MemoryGrow, MemorySize, Select, SetGlobal, SetLocal, Statement, StoreAt, - StoreType, Terminator, UnOp, UnOpType, Value, + Expression, FuncData, GetGlobal, GetLocal, If, LabelType, LoadAt, LoadType, MemoryArgument, + MemoryCopy, MemoryFill, MemoryGrow, MemorySize, Select, SetGlobal, SetLocal, Statement, + StoreAt, StoreType, Terminator, UnOp, UnOpType, Value, }, stack::{ReadType, Stack}, }; @@ -611,19 +611,43 @@ impl<'a> Factory<'a> { Operator::MemoryCopy { dst_mem, src_mem } => { let size = self.target.stack.pop().into(); + let source = MemoryArgument { + memory: src_mem.try_into().unwrap(), + pointer: self.target.stack.pop().into(), + }; + + let destination = MemoryArgument { + memory: dst_mem.try_into().unwrap(), + pointer: self.target.stack.pop().into(), + }; + + self.target.leak_memory_write(source.memory); + self.target.leak_memory_write(destination.memory); + let data = Statement::MemoryCopy(MemoryCopy { - dst: dst_mem, - src: src_mem, + destination, + source, size, }); self.target.code.push(data); } Operator::MemoryFill { mem } => { - let n = self.target.stack.pop().into(); + let size = self.target.stack.pop().into(); let value = self.target.stack.pop().into(); - let data = Statement::MemoryFill(MemoryFill { mem, value, n }); + let destination = MemoryArgument { + memory: mem.try_into().unwrap(), + pointer: self.target.stack.pop().into(), + }; + + self.target.leak_memory_write(destination.memory); + + let data = Statement::MemoryFill(MemoryFill { + destination, + size, + value, + }); self.target.code.push(data); } diff --git a/wasm-ast/src/node.rs b/wasm-ast/src/node.rs index db228b6..7c2bc8e 100644 --- a/wasm-ast/src/node.rs +++ b/wasm-ast/src/node.rs @@ -1108,21 +1108,40 @@ impl MemoryGrow { } } +pub struct MemoryArgument { + pub(crate) memory: usize, + pub(crate) pointer: Box, +} + +impl MemoryArgument { + #[must_use] + pub const fn memory(&self) -> usize { + self.memory + } + + #[must_use] + pub fn pointer(&self) -> &Expression { + &self.pointer + } +} + pub struct MemoryCopy { - pub(crate) dst: u32, - pub(crate) src: u32, + pub(crate) destination: MemoryArgument, + pub(crate) source: MemoryArgument, pub(crate) size: Box, } impl MemoryCopy { #[must_use] - pub const fn dst(&self) -> u32 { - self.dst + pub const fn destination(&self) -> &MemoryArgument { + &self.destination } + #[must_use] - pub const fn src(&self) -> u32 { - self.src + pub const fn source(&self) -> &MemoryArgument { + &self.source } + #[must_use] pub fn size(&self) -> &Expression { &self.size @@ -1130,24 +1149,26 @@ impl MemoryCopy { } pub struct MemoryFill { - pub(crate) mem: u32, + pub(crate) destination: MemoryArgument, + pub(crate) size: Box, pub(crate) value: Box, - pub(crate) n: Box, } impl MemoryFill { #[must_use] - pub const fn mem(&self) -> u32 { - self.mem + pub const fn destination(&self) -> &MemoryArgument { + &self.destination } + + #[must_use] + pub fn size(&self) -> &Expression { + &self.size + } + #[must_use] pub fn value(&self) -> &Expression { &self.value } - #[must_use] - pub fn n(&self) -> &Expression { - &self.n - } } pub enum Statement { diff --git a/wasm-ast/src/visit.rs b/wasm-ast/src/visit.rs index acb1f7e..b8b9605 100644 --- a/wasm-ast/src/visit.rs +++ b/wasm-ast/src/visit.rs @@ -110,6 +110,8 @@ impl Driver for MemorySize { impl Driver for MemoryCopy { fn accept(&self, visitor: &mut T) { + self.destination().pointer().accept(visitor); + self.source().pointer().accept(visitor); self.size().accept(visitor); visitor.visit_memory_copy(self); @@ -118,8 +120,9 @@ impl Driver for MemoryCopy { impl Driver for MemoryFill { fn accept(&self, visitor: &mut T) { + self.destination().pointer().accept(visitor); + self.size().accept(visitor); self.value().accept(visitor); - self.n().accept(visitor); visitor.visit_memory_fill(self); }