Translating RISC-V programs directly from ELF files. (#1453)

There are many advantages in using standard assemblers and linkers, like: - maturity; - more complete support of assembly language, which in turn allows for support to more high level languages and compilers; - link-time optimizations; - no need to deal with language edge cases (our assembler and linker deals with a number of rust stuff, but it would need even more to support `std`). But it comes with a cost, as we need to lift references to text data back into labels, (because powdr operates on a higher abstraction level), and to do that, we need the ELF file to either be a PIE (Position Independent Executable), or to still have the linkage relocation tables (option `--emit-relocs` of GNU and LLVM linkers). --------- Co-authored-by: Leo Alt <[email protected]> Co-authored-by: Leo <[email protected]>
powdr-labs · Jul 8, 2024 · b06fa28 · b06fa28
1 parent 01c1a5c
commit b06fa28
Show file tree

Hide file tree

Showing 62 changed files with 2,240 additions and 330 deletions.
diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml
@@ -155,6 +155,8 @@ jobs:
         key: ${{ runner.os }}-pilcom-node-modules
     - name: Install Rust toolchain 1.77 (with clippy and rustfmt)
       run: rustup toolchain install 1.77-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.77-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.77-x86_64-unknown-linux-gnu
+    - name: Install test dependencies
+      run: sudo apt-get install -y binutils-riscv64-unknown-elf lld
     - name: Install nightly
       run: rustup toolchain install nightly-2024-02-01-x86_64-unknown-linux-gnu
     - name: Install riscv target

diff --git a/asm-utils/src/utils.rs b/asm-utils/src/utils.rs
@@ -41,11 +41,9 @@ pub fn escape_label(l: &str) -> String {
     l.replace('.', "_dot_").replace('/', "_slash_")
 }
 
-pub fn argument_to_escaped_symbol<R: Register, F: FunctionOpKind>(
-    x: &Argument<R, F>,
-) -> Option<String> {
+pub fn argument_to_symbol<R: Register, F: FunctionOpKind>(x: &Argument<R, F>) -> Option<&str> {
     if let Argument::Expression(Expression::Symbol(symbol)) = x {
-        Some(escape_label(symbol))
+        Some(symbol)
     } else {
         None
     }

diff --git a/cli-rs/src/main.rs b/cli-rs/src/main.rs
@@ -12,8 +12,8 @@ use powdr_pipeline::Pipeline;
 use powdr_riscv_executor::ProfilerOptions;
 
 use std::ffi::OsStr;
-use std::io;
 use std::{borrow::Cow, io::Write, path::Path};
+use std::{fs, io};
 use strum::{Display, EnumString, EnumVariantNames};
 
 #[derive(Clone, EnumString, EnumVariantNames, Display)]
@@ -62,13 +62,17 @@ enum Commands {
         #[arg(long)]
         coprocessors: Option<String>,
 
+        /// Convert from the executable ELF file instead of the assembly.
+        #[arg(short, long)]
+        #[arg(default_value_t = false)]
+        elf: bool,
+
         /// Run a long execution in chunks (Experimental and not sound!)
         #[arg(short, long)]
         #[arg(default_value_t = false)]
         continuations: bool,
     },
-    /// Compiles riscv assembly to powdr assembly and then to PIL
-    /// and generates fixed and witness columns.
+    /// Compiles riscv assembly to powdr assembly.
     RiscvAsm {
         /// Input files
         #[arg(required = true)]
@@ -94,6 +98,32 @@ enum Commands {
         #[arg(default_value_t = false)]
         continuations: bool,
     },
+    /// Translates a RISC-V statically linked executable to powdr assembly.
+    RiscvElf {
+        /// Input file
+        #[arg(required = true)]
+        file: String,
+
+        /// The field to use
+        #[arg(long)]
+        #[arg(default_value_t = FieldArgument::Gl)]
+        #[arg(value_parser = clap_enum_variants!(FieldArgument))]
+        field: FieldArgument,
+
+        /// Directory for output files.
+        #[arg(short, long)]
+        #[arg(default_value_t = String::from("."))]
+        output_directory: String,
+
+        /// Comma-separated list of coprocessors.
+        #[arg(long)]
+        coprocessors: Option<String>,
+
+        /// Run a long execution in chunks (Experimental and not sound!)
+        #[arg(short, long)]
+        #[arg(default_value_t = false)]
+        continuations: bool,
+    },
     /// Executes a powdr-asm file with given inputs.
     Execute {
         /// input powdr-asm code compiled from Rust/RISCV
@@ -194,12 +224,14 @@ fn run_command(command: Commands) {
             field,
             output_directory,
             coprocessors,
+            elf,
             continuations,
         } => {
             call_with_field!(compile_rust::<field>(
                 &file,
                 Path::new(&output_directory),
                 coprocessors,
+                elf,
                 continuations
             ))
         }
@@ -225,6 +257,20 @@ fn run_command(command: Commands) {
                 continuations
             ))
         }
+        Commands::RiscvElf {
+            file,
+            field,
+            output_directory,
+            coprocessors,
+            continuations,
+        } => {
+            call_with_field!(compile_riscv_elf::<field>(
+                &file,
+                Path::new(&output_directory),
+                coprocessors,
+                continuations
+            ))
+        }
         Commands::Execute {
             file,
             field,
@@ -271,6 +317,7 @@ fn compile_rust<F: FieldElement>(
     file_name: &str,
     output_dir: &Path,
     coprocessors: Option<String>,
+    via_elf: bool,
     continuations: bool,
 ) -> Result<(), Vec<String>> {
     let mut runtime = match coprocessors {
@@ -284,8 +331,15 @@ fn compile_rust<F: FieldElement>(
         runtime = runtime.with_poseidon();
     }
 
-    powdr_riscv::compile_rust::<F>(file_name, output_dir, true, &runtime, continuations)
-        .ok_or_else(|| vec!["could not compile rust".to_string()])?;
+    powdr_riscv::compile_rust::<F>(
+        file_name,
+        output_dir,
+        true,
+        &runtime,
+        via_elf,
+        continuations,
+    )
+    .ok_or_else(|| vec!["could not compile rust".to_string()])?;
 
     Ok(())
 }
@@ -305,9 +359,14 @@ fn compile_riscv_asm<F: FieldElement>(
         None => powdr_riscv::Runtime::base(),
     };
 
-    powdr_riscv::compile_riscv_asm::<F>(
+    powdr_riscv::compile_riscv_asm_bundle::<F>(
         original_file_name,
-        file_names,
+        file_names
+            .map(|name| {
+                let contents = fs::read_to_string(&name).unwrap();
+                (name, contents)
+            })
+            .collect(),
         output_dir,
         true,
         &runtime,
@@ -318,6 +377,32 @@ fn compile_riscv_asm<F: FieldElement>(
     Ok(())
 }
 
+fn compile_riscv_elf<F: FieldElement>(
+    input_file: &str,
+    output_dir: &Path,
+    coprocessors: Option<String>,
+    continuations: bool,
+) -> Result<(), Vec<String>> {
+    let runtime = match coprocessors {
+        Some(list) => {
+            powdr_riscv::Runtime::try_from(list.split(',').collect::<Vec<_>>().as_ref()).unwrap()
+        }
+        None => powdr_riscv::Runtime::base(),
+    };
+
+    powdr_riscv::compile_riscv_elf::<F>(
+        input_file,
+        Path::new(input_file),
+        output_dir,
+        true,
+        &runtime,
+        continuations,
+    )
+    .ok_or_else(|| vec!["could not translate RISC-V executable".to_string()])?;
+
+    Ok(())
+}
+
 #[allow(clippy::too_many_arguments)]
 fn execute<F: FieldElement>(
     file_name: &Path,

diff --git a/riscv-runtime/src/lib.rs b/riscv-runtime/src/lib.rs
@@ -42,23 +42,21 @@ unsafe fn panic(panic: &PanicInfo<'_>) -> ! {
 // 3. Tail call the main function (in powdr, the return address register is already
 //    set, so that returning from the entry point function will cause the execution
 //    to succeed).
+// TODO: support Position Independent Executables (PIE) by using lla.
 global_asm!(
     r"
 .global __runtime_start
+.type __runtime_start, @function
 __runtime_start:
     .option push
     .option norelax
+    #lla gp, __global_pointer$
     lui gp, %hi(__global_pointer$)
     addi gp, gp, %lo(__global_pointer$)
     .option pop
+    #lla sp, __powdr_stack_start
     lui sp, %hi(__powdr_stack_start)
     addi sp, sp, %lo(__powdr_stack_start)
     tail main
 "
 );
-
-// TODO: ideally, the above code would use `la` instead of `lui` + `addi`, but
-// for some reason rustc automatically expands it to `auipc %pcrel_hi(...)`
-// + `addi %pcrel_lo(...)`, which our asm converter doesn't support on multiple
-// levels. We can't use `li` either, because rustc doesn't like `li` with
-// symbols.
diff --git a/riscv/Cargo.toml b/riscv/Cargo.toml
@@ -24,12 +24,16 @@ powdr-pipeline.workspace = true
 powdr-riscv-executor.workspace = true
 powdr-riscv-syscalls.workspace = true
 
+goblin = { version = "0.8" }
 lazy_static = "1.4.0"
 itertools = "0.13"
 lalrpop-util = { version = "^0.19", features = ["lexer"] }
 log = "0.4.17"
 mktemp = "0.5.0"
 num-traits = "0.2.15"
+# Use the patched version of raki until the fix is merged.
+# Fixes the name of "mulhsu" instruction.
+raki = { git = "https://github.com/powdr-labs/raki.git", branch = "patch-1" }
 serde_json = "1.0"
 # This is only here to work around https://github.com/lalrpop/lalrpop/issues/750
 # It should be removed once that workaround is no longer needed.

diff --git a/riscv/build.rs b/riscv/build.rs
@@ -36,17 +36,16 @@ fn build_instruction_tests() {
             .to_string()
             .strip_suffix(".S")
         {
-            println!("cargo:rerun-if-changed={generated_path}/{file_name}.S");
             write!(
                 test_file,
-                r#"
+                r##"
 #[test]
 #[ignore = "Too slow"]
 fn {file_name}() {{
-    run_instruction_test(include_str!("{test_file}"), "{file_name}");
+    run_instruction_test(Path::new(r#"{file}"#), r#"{file_name}"#);
 }}
-"#,
-                test_file = file.path().canonicalize().unwrap().display(),
+"##,
+                file = file.path().to_str().unwrap(),
             )
             .unwrap();
         }