From 1227d50e9b72b45b6b27b431b7811054bd41d7a0 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Fri, 1 May 2020 15:46:45 -0700 Subject: [PATCH 1/3] Initial commit for support of custom sections. --- lib/compiler-llvm/src/compiler.rs | 58 +++++++++++++++++++++--- lib/compiler-llvm/src/translator/code.rs | 41 +++++++++++------ 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 857d03e19..afc831405 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -9,14 +9,16 @@ use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use wasm_common::entity::{EntityRef, PrimaryMap, SecondaryMap}; use wasm_common::Features; use wasm_common::{FuncIndex, FuncType, LocalFuncIndex, MemoryIndex, TableIndex}; -use wasmer_compiler::FunctionBodyData; -use wasmer_compiler::TrapInformation; -use wasmer_compiler::{Compilation, CompileError, CompiledFunction, Compiler}; -use wasmer_compiler::{CompilerConfig, ModuleTranslationState, Target}; +use wasmer_compiler::{ + Compilation, CompileError, CompiledFunction, Compiler, CompilerConfig, CustomSection, + CustomSectionProtection, FunctionBodyData, ModuleTranslationState, Relocation, + RelocationTarget, SectionIndex, Target, TrapInformation, +}; use wasmer_runtime::{MemoryPlan, Module, TablePlan, TrapCode}; use inkwell::targets::{InitializationConfig, Target as InkwellTarget}; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; // TODO: remove /// A compiler that compiles a WebAssembly module with LLVM, translating the Wasm to LLVM IR, @@ -62,7 +64,6 @@ impl Compiler for LLVMCompiler { ) -> Result { //let data = Arc::new(Mutex::new(0)); let mut func_names = SecondaryMap::new(); - let custom_sections = PrimaryMap::new(); for (func_index, _) in &module.functions { func_names[func_index] = module @@ -71,7 +72,7 @@ impl Compiler for LLVMCompiler { .cloned() .unwrap_or_else(|| format!("fn{}", func_index.index())); } - let functions = function_body_inputs + let mut outputs = function_body_inputs .into_iter() .collect::)>>() .par_iter() @@ -92,6 +93,51 @@ impl Compiler for LLVMCompiler { .into_iter() .collect::>(); + // We're going to "link" the sections by simply appending all compatible + // sections, then building the new relocations. + // TODO: merge constants. + let mut readonly_section = CustomSection { + protection: CustomSectionProtection::Read, + bytes: Vec::new(), + }; + /* + let mut readexecute_section = CustomSection { + protection: CustomSectionProtection::Read, + bytes: Vec::new(), + }; + */ + for (ref mut function, ref local_relocations, ref custom_sections) in outputs.values_mut() { + for (local_idx, custom_section) in custom_sections.iter().enumerate() { + let local_idx = local_idx as u32; + // TODO: these section numbers are potentially wrong, if there's + // no Read and only a ReadExecute then ReadExecute is 0. + let (ref mut section, section_num) = match &custom_section.protection { + Read => (&mut readonly_section, SectionIndex::from_u32(0)), + //ReadExecute => (&mut readexecute_section, 1), + }; + let offset = section.bytes.len() as i64; + section.bytes.extend(&custom_section.bytes); + // TODO: we're needlessly rescanning the whole list. + for local_relocation in local_relocations { + if local_relocation.local_section_index == local_idx { + function.relocations.push(Relocation { + kind: local_relocation.kind, + reloc_target: RelocationTarget::CustomSection(section_num), + offset: local_relocation.offset, + addend: local_relocation.addend + offset, + }); + } + } + } + } + + let functions = outputs + .iter_mut() + .map(|(i, (f, _, _))| *f) + .collect::>(); + + let mut custom_sections = PrimaryMap::new(); + custom_sections.push(readonly_section); Ok(Compilation::new(functions, custom_sections)) } diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index b1a43ce98..5f6d63320 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -39,14 +39,13 @@ use wasm_common::{ }; use wasmer_compiler::wasmparser::{self, BinaryReader, MemoryImmediate, Operator}; use wasmer_compiler::{ - to_wasm_error, wasm_unsupported, CompileError, CompiledFunction, WasmResult, -}; -use wasmer_compiler::{ - CompiledFunctionUnwindInfo, FunctionAddressMap, FunctionBodyData, Relocation, RelocationKind, - RelocationTarget, SourceLoc, + to_wasm_error, wasm_unsupported, Addend, CodeOffset, CompileError, CompiledFunction, + CompiledFunctionUnwindInfo, CustomSection, FunctionAddressMap, FunctionBodyData, Relocation, + RelocationKind, RelocationTarget, SourceLoc, WasmResult, }; +use wasmer_runtime::libcalls::LibCall; use wasmer_runtime::Module as WasmerCompilerModule; -use wasmer_runtime::{libcalls::LibCall, MemoryPlan, MemoryStyle, TablePlan}; +use wasmer_runtime::{MemoryPlan, MemoryStyle, TablePlan}; // TODO: debugging use std::fs; @@ -84,6 +83,14 @@ fn const_zero<'ctx>(ty: BasicTypeEnum<'ctx>) -> BasicValueEnum<'ctx> { } } +// Relocation against a per-function section. +pub struct LocalRelocation { + pub kind: RelocationKind, + pub local_section_index: u32, + pub offset: CodeOffset, + pub addend: Addend, +} + impl FuncTranslator { pub fn new() -> Self { Self { @@ -100,7 +107,7 @@ impl FuncTranslator { memory_plans: &PrimaryMap, table_plans: &PrimaryMap, func_names: &SecondaryMap, - ) -> Result { + ) -> Result<(CompiledFunction, Vec, Vec), CompileError> { let func_index = wasm_module.func_index(*func_index); let func_name = func_names.get(func_index).unwrap(); let module_name = match wasm_module.name.as_ref() { @@ -363,14 +370,18 @@ impl FuncTranslator { body_len: 0, // TODO }; - Ok(CompiledFunction { - address_map, - body: bytes, - jt_offsets: SecondaryMap::new(), - unwind_info: CompiledFunctionUnwindInfo::None, - relocations, - traps: vec![], - }) + Ok(( + CompiledFunction { + address_map, + body: bytes, + jt_offsets: SecondaryMap::new(), + unwind_info: CompiledFunctionUnwindInfo::None, + relocations, + traps: vec![], + }, + vec![], + vec![], + )) } } From cf70a297d8c9aa1582615d242829f03213dd7134 Mon Sep 17 00:00:00 2001 From: Syrus Date: Fri, 1 May 2020 15:57:18 -0700 Subject: [PATCH 2/3] Fixed compilation --- lib/compiler-llvm/src/compiler.rs | 77 ++++++++++++++----------------- lib/compiler/src/section.rs | 4 +- 2 files changed, 37 insertions(+), 44 deletions(-) diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index afc831405..6ec08c518 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -65,6 +65,14 @@ impl Compiler for LLVMCompiler { //let data = Arc::new(Mutex::new(0)); let mut func_names = SecondaryMap::new(); + // We're going to "link" the sections by simply appending all compatible + // sections, then building the new relocations. + // TODO: merge constants. + let mut readonly_section = CustomSection { + protection: CustomSectionProtection::Read, + bytes: Vec::new(), + }; + for (func_index, _) in &module.functions { func_names[func_index] = module .func_names @@ -72,7 +80,7 @@ impl Compiler for LLVMCompiler { .cloned() .unwrap_or_else(|| format!("fn{}", func_index.index())); } - let mut outputs = function_body_inputs + let mut functions = function_body_inputs .into_iter() .collect::)>>() .par_iter() @@ -91,49 +99,34 @@ impl Compiler for LLVMCompiler { }) .collect::, CompileError>>()? .into_iter() - .collect::>(); - - // We're going to "link" the sections by simply appending all compatible - // sections, then building the new relocations. - // TODO: merge constants. - let mut readonly_section = CustomSection { - protection: CustomSectionProtection::Read, - bytes: Vec::new(), - }; - /* - let mut readexecute_section = CustomSection { - protection: CustomSectionProtection::Read, - bytes: Vec::new(), - }; - */ - for (ref mut function, ref local_relocations, ref custom_sections) in outputs.values_mut() { - for (local_idx, custom_section) in custom_sections.iter().enumerate() { - let local_idx = local_idx as u32; - // TODO: these section numbers are potentially wrong, if there's - // no Read and only a ReadExecute then ReadExecute is 0. - let (ref mut section, section_num) = match &custom_section.protection { - Read => (&mut readonly_section, SectionIndex::from_u32(0)), - //ReadExecute => (&mut readexecute_section, 1), - }; - let offset = section.bytes.len() as i64; - section.bytes.extend(&custom_section.bytes); - // TODO: we're needlessly rescanning the whole list. - for local_relocation in local_relocations { - if local_relocation.local_section_index == local_idx { - function.relocations.push(Relocation { - kind: local_relocation.kind, - reloc_target: RelocationTarget::CustomSection(section_num), - offset: local_relocation.offset, - addend: local_relocation.addend + offset, - }); + .map(|(mut function, local_relocations, custom_sections)| { + /// We collect the sections data + for (local_idx, custom_section) in custom_sections.iter().enumerate() { + let local_idx = local_idx as u32; + // TODO: these section numbers are potentially wrong, if there's + // no Read and only a ReadExecute then ReadExecute is 0. + let (ref mut section, section_num) = match &custom_section.protection { + CustomSectionProtection::Read => (&mut readonly_section, SectionIndex::from_u32(0)), + //ReadExecute => (&mut readexecute_section, 1), + }; + let offset = section.bytes.len() as i64; + section.bytes.extend(&custom_section.bytes); + // TODO: we're needlessly rescanning the whole list. + for local_relocation in &local_relocations { + if local_relocation.local_section_index == local_idx { + function.relocations.push(Relocation { + kind: local_relocation.kind, + reloc_target: RelocationTarget::CustomSection(section_num), + offset: local_relocation.offset, + addend: local_relocation.addend + offset, + }); + } } } - } - } - - let functions = outputs - .iter_mut() - .map(|(i, (f, _, _))| *f) + Ok(function) + }) + .collect::, CompileError>>()? + .into_iter() .collect::>(); let mut custom_sections = PrimaryMap::new(); diff --git a/lib/compiler/src/section.rs b/lib/compiler/src/section.rs index af0fdbeff..0908f1b90 100644 --- a/lib/compiler/src/section.rs +++ b/lib/compiler/src/section.rs @@ -23,8 +23,8 @@ pub enum CustomSectionProtection { Read, // We don't include `ReadWrite` here because it would complicate freeze // and resumption of executing Modules. - /// A compiled section that is also executable. - ReadExecute, + // We also currently don't include `ReadExecute` as we don't have a way + // to represent relocations for this kind of section. } /// A Section for a `Compilation`. From 4827434eefd5138b5944965364c5004571e84492 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Fri, 1 May 2020 16:31:26 -0700 Subject: [PATCH 3/3] Build up list of per-function custom sections and build local relocations into them. --- lib/compiler-llvm/src/compiler.rs | 11 ++-- lib/compiler-llvm/src/translator/code.rs | 68 ++++++++++++++++++------ 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 6ec08c518..71c93338a 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -68,6 +68,7 @@ impl Compiler for LLVMCompiler { // We're going to "link" the sections by simply appending all compatible // sections, then building the new relocations. // TODO: merge constants. + let mut used_readonly_section = false; let mut readonly_section = CustomSection { protection: CustomSectionProtection::Read, bytes: Vec::new(), @@ -106,14 +107,16 @@ impl Compiler for LLVMCompiler { // TODO: these section numbers are potentially wrong, if there's // no Read and only a ReadExecute then ReadExecute is 0. let (ref mut section, section_num) = match &custom_section.protection { - CustomSectionProtection::Read => (&mut readonly_section, SectionIndex::from_u32(0)), - //ReadExecute => (&mut readexecute_section, 1), + CustomSectionProtection::Read => { + (&mut readonly_section, SectionIndex::from_u32(0)) + } }; let offset = section.bytes.len() as i64; section.bytes.extend(&custom_section.bytes); // TODO: we're needlessly rescanning the whole list. for local_relocation in &local_relocations { if local_relocation.local_section_index == local_idx { + used_readonly_section = true; function.relocations.push(Relocation { kind: local_relocation.kind, reloc_target: RelocationTarget::CustomSection(section_num), @@ -130,7 +133,9 @@ impl Compiler for LLVMCompiler { .collect::>(); let mut custom_sections = PrimaryMap::new(); - custom_sections.push(readonly_section); + if used_readonly_section { + custom_sections.push(readonly_section); + } Ok(Compilation::new(functions, custom_sections)) } diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index 5f6d63320..22f1b167d 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -40,8 +40,8 @@ use wasm_common::{ use wasmer_compiler::wasmparser::{self, BinaryReader, MemoryImmediate, Operator}; use wasmer_compiler::{ to_wasm_error, wasm_unsupported, Addend, CodeOffset, CompileError, CompiledFunction, - CompiledFunctionUnwindInfo, CustomSection, FunctionAddressMap, FunctionBodyData, Relocation, - RelocationKind, RelocationTarget, SourceLoc, WasmResult, + CompiledFunctionUnwindInfo, CustomSection, CustomSectionProtection, FunctionAddressMap, + FunctionBodyData, Relocation, RelocationKind, RelocationTarget, SourceLoc, WasmResult, }; use wasmer_runtime::libcalls::LibCall; use wasmer_runtime::Module as WasmerCompilerModule; @@ -84,6 +84,7 @@ fn const_zero<'ctx>(ty: BasicTypeEnum<'ctx>) -> BasicValueEnum<'ctx> { } // Relocation against a per-function section. +#[derive(Debug)] pub struct LocalRelocation { pub kind: RelocationKind, pub local_section_index: u32, @@ -308,6 +309,8 @@ impl FuncTranslator { let mut bytes = vec![]; let mut relocations = vec![]; + let mut local_relocations = vec![]; + let mut required_custom_sections = vec![]; for section in object.get_sections() { match section.get_name().map(std::ffi::CStr::to_bytes) { Some(b"wasmer_function") => { @@ -342,26 +345,57 @@ impl FuncTranslator { } } } - if reloc_target.is_none() { - unimplemented!("reference to non-user function {:?}", &target); - } - let reloc_target = reloc_target.unwrap(); + if reloc_target.is_some() { + let reloc_target = reloc_target.unwrap(); - let relocation = Relocation { - kind, - reloc_target, - offset: rel.get_offset() as u32, - // TODO: it appears the LLVM C API has no way to - // retrieve the addend. - addend: 0, - }; - relocations.push(relocation); + relocations.push(Relocation { + kind, + reloc_target, + offset: rel.get_offset() as u32, + // TODO: it appears the LLVM C API has no way to + // retrieve the addend. + addend: 0, + }); + } else { + if let Some(ref target) = &target { + let local_section_index = required_custom_sections.len() as u32; + required_custom_sections.push(target.clone()); + local_relocations.push(LocalRelocation { + kind, + local_section_index, + offset: rel.get_offset() as u32, + // TODO: it appears the LLVM C API has no way to + // retrieve the addend. + addend: 0, + }); + } + } } } _ => {} }; } + // TODO: verify that we see all of them + let mut custom_sections = vec![]; + custom_sections.resize( + required_custom_sections.len(), + CustomSection { + protection: CustomSectionProtection::Read, + bytes: vec![], + }, + ); + for section in object.get_sections() { + if let Some(name) = section.get_name().map(std::ffi::CStr::to_bytes) { + if let Some(index) = required_custom_sections + .iter() + .position(|n| n.as_bytes() == name) + { + custom_sections[index].bytes.extend(section.get_contents()); + } + } + } + let address_map = FunctionAddressMap { instructions: vec![], start_srcloc: SourceLoc::default(), @@ -379,8 +413,8 @@ impl FuncTranslator { relocations, traps: vec![], }, - vec![], - vec![], + local_relocations, + custom_sections, )) } }