Rust backend (#2787)

* Implements code generation through Rust.
* CLI: adds two `dev` compilation targets: 
  1. `rust` for generating Rust code
  2. `native-rust` for generating a native executable via Rust
* Adds end-to-end tests for compilation from Juvix to native executable
via Rust.
* A target for RISC0 needs to be added in a separate PR building on this
one.

app/Commands/Dev/DevCompile.hs

@@ -4,8 +4,10 @@ import Commands.Base
 import Commands.Dev.DevCompile.Asm qualified as Asm
 import Commands.Dev.DevCompile.Casm qualified as Casm
 import Commands.Dev.DevCompile.Core qualified as Core
+import Commands.Dev.DevCompile.NativeRust qualified as NativeRust
 import Commands.Dev.DevCompile.Options
 import Commands.Dev.DevCompile.Reg qualified as Reg
+import Commands.Dev.DevCompile.Rust qualified as Rust
 import Commands.Dev.DevCompile.Tree qualified as Tree
 
 runCommand :: (Members '[App, EmbedIO, TaggedLock] r) => DevCompileCommand -> Sem r ()
@@ -15,3 +17,5 @@ runCommand = \case
   Asm opts -> Asm.runCommand opts
   Tree opts -> Tree.runCommand opts
   Casm opts -> Casm.runCommand opts
+  Rust opts -> Rust.runCommand opts
+  NativeRust opts -> NativeRust.runCommand opts

app/Commands/Dev/DevCompile/NativeRust.hs

@@ -0,0 +1,75 @@
+module Commands.Dev.DevCompile.NativeRust where
+
+import Commands.Base
+import Commands.Dev.DevCompile.NativeRust.Options
+import Commands.Extra.Rust
+import Data.ByteString qualified as BS
+import Data.FileEmbed qualified as FE
+import Juvix.Compiler.Backend.Rust.Data.Result
+
+runCommand ::
+  (Members '[App, EmbedIO, TaggedLock] r) =>
+  NativeRustOptions 'InputMain ->
+  Sem r ()
+runCommand opts = do
+  let opts' = opts ^. nativeRustCompileCommonOptions
+      inputFile = opts' ^. compileInputFile
+      moutputFile = opts' ^. compileOutputFile
+  mainFile <- getMainFile inputFile
+  Result {..} <- runPipeline opts inputFile upToRust
+  rustFile <- inputRustFile mainFile
+  writeFileEnsureLn rustFile _resultRustCode
+  buildDir <- askBuildDir
+  ensureDir buildDir
+  prepareRuntime
+  outputFile <- nativeOutputFile mainFile moutputFile
+  let args =
+        RustArgs
+          { _rustDebug = opts' ^. compileDebug,
+            _rustInputFile = rustFile,
+            _rustOutputFile = outputFile,
+            _rustOptimizationLevel = fmap (min 3 . (+ 1)) (opts' ^. compileOptimizationLevel)
+          }
+  rustCompile args
+  where
+    prepareRuntime ::
+      forall s.
+      (Members '[App, EmbedIO] s) =>
+      Sem s ()
+    prepareRuntime = writeRuntime runtime
+      where
+        runtime :: BS.ByteString
+        runtime
+          | opts ^. nativeRustCompileCommonOptions . compileDebug = rustDebugRuntime
+          | otherwise = rustReleaseRuntime
+          where
+            rustReleaseRuntime :: BS.ByteString
+            rustReleaseRuntime = $(FE.makeRelativeToProject "runtime/rust/target/release/libjuvix.rlib" >>= FE.embedFile)
+
+            rustDebugRuntime :: BS.ByteString
+            rustDebugRuntime = $(FE.makeRelativeToProject "runtime/rust/target/debug/libjuvix.rlib" >>= FE.embedFile)
+
+inputRustFile :: (Members '[App, EmbedIO] r) => Path Abs File -> Sem r (Path Abs File)
+inputRustFile inputFileCompile = do
+  buildDir <- askBuildDir
+  ensureDir buildDir
+  return (buildDir <//> replaceExtension' ".rs" (filename inputFileCompile))
+
+nativeOutputFile :: forall r. (Member App r) => Path Abs File -> Maybe (AppPath File) -> Sem r (Path Abs File)
+nativeOutputFile inputFile moutputFile =
+  case moutputFile of
+    Just f -> fromAppFile f
+    Nothing -> do
+      invokeDir <- askInvokeDir
+      let baseOutputFile = invokeDir <//> filename inputFile
+      return $ removeExtension' baseOutputFile
+
+writeRuntime ::
+  forall r.
+  (Members '[App, EmbedIO] r) =>
+  BS.ByteString ->
+  Sem r ()
+writeRuntime runtime = do
+  buildDir <- askBuildDir
+  liftIO $
+    BS.writeFile (toFilePath (buildDir <//> $(mkRelFile "libjuvix.rlib"))) runtime

app/Commands/Dev/DevCompile/NativeRust/Options.hs

@@ -0,0 +1,28 @@
+{-# LANGUAGE UndecidableInstances #-}
+
+module Commands.Dev.DevCompile.NativeRust.Options
+  ( module Commands.Dev.DevCompile.NativeRust.Options,
+    module Commands.Compile.CommonOptions,
+  )
+where
+
+import Commands.Compile.CommonOptions
+import CommonOptions
+
+data NativeRustOptions (k :: InputKind) = NativeRustOptions
+  { _nativeRustCompileCommonOptions :: CompileCommonOptions k
+  }
+
+deriving stock instance (Typeable k, Data (InputFileType k)) => Data (NativeRustOptions k)
+
+makeLenses ''NativeRustOptions
+
+parseNativeRust :: (SingI k) => Parser (NativeRustOptions k)
+parseNativeRust = do
+  _nativeRustCompileCommonOptions <- parseCompileCommonOptions
+  pure NativeRustOptions {..}
+
+instance EntryPointOptions (NativeRustOptions k) where
+  applyOptions opts =
+    set entryPointTarget (Just TargetRust)
+      . applyOptions (opts ^. nativeRustCompileCommonOptions)

app/Commands/Dev/DevCompile/Options.hs

@@ -3,7 +3,9 @@ module Commands.Dev.DevCompile.Options where
 import Commands.Dev.DevCompile.Asm.Options
 import Commands.Dev.DevCompile.Casm.Options
 import Commands.Dev.DevCompile.Core.Options
+import Commands.Dev.DevCompile.NativeRust.Options
 import Commands.Dev.DevCompile.Reg.Options
+import Commands.Dev.DevCompile.Rust.Options
 import Commands.Dev.DevCompile.Tree.Options
 import CommonOptions
 
@@ -13,6 +15,8 @@ data DevCompileCommand
   | Reg (RegOptions 'InputMain)
   | Tree (TreeOptions 'InputMain)
   | Casm (CasmOptions 'InputMain)
+  | Rust (RustOptions 'InputMain)
+  | NativeRust (NativeRustOptions 'InputMain)
   deriving stock (Data)
 
 parseDevCompileCommand :: Parser DevCompileCommand
@@ -23,7 +27,9 @@ parseDevCompileCommand =
           commandReg,
           commandTree,
           commandCasm,
-          commandAsm
+          commandAsm,
+          commandRust,
+          commandNativeRust
         ]
     )
 
@@ -61,3 +67,17 @@ commandAsm =
     info
       (Asm <$> parseAsm)
       (progDesc "Compile to Juvix ASM")
+
+commandRust :: Mod CommandFields DevCompileCommand
+commandRust =
+  command "rust" $
+    info
+      (Rust <$> parseRust)
+      (progDesc "Compile to Rust")
+
+commandNativeRust :: Mod CommandFields DevCompileCommand
+commandNativeRust =
+  command "native-rust" $
+    info
+      (NativeRust <$> parseNativeRust)
+      (progDesc "Compile to native executable through Rust")

app/Commands/Dev/DevCompile/Rust.hs

@@ -0,0 +1,17 @@
+module Commands.Dev.DevCompile.Rust where
+
+import Commands.Base
+import Commands.Dev.DevCompile.Rust.Options
+import Commands.Extra.NewCompile
+import Juvix.Compiler.Backend.Rust.Data.Result
+
+runCommand ::
+  (Members '[App, EmbedIO, TaggedLock] r) =>
+  RustOptions 'InputMain ->
+  Sem r ()
+runCommand opts = do
+  let inputFile = opts ^. rustCompileCommonOptions . compileInputFile
+      moutputFile = opts ^. rustCompileCommonOptions . compileOutputFile
+  outFile :: Path Abs File <- getOutputFile FileExtRust inputFile moutputFile
+  Result {..} <- runPipeline opts inputFile upToRust
+  writeFileEnsureLn outFile _resultRustCode

app/Commands/Dev/DevCompile/Rust/Options.hs

@@ -0,0 +1,28 @@
+{-# LANGUAGE UndecidableInstances #-}
+
+module Commands.Dev.DevCompile.Rust.Options
+  ( module Commands.Dev.DevCompile.Rust.Options,
+    module Commands.Compile.CommonOptions,
+  )
+where
+
+import Commands.Compile.CommonOptions
+import CommonOptions
+
+data RustOptions (k :: InputKind) = RustOptions
+  { _rustCompileCommonOptions :: CompileCommonOptions k
+  }
+
+deriving stock instance (Typeable k, Data (InputFileType k)) => Data (RustOptions k)
+
+makeLenses ''RustOptions
+
+parseRust :: (SingI k) => Parser (RustOptions k)
+parseRust = do
+  _rustCompileCommonOptions <- parseCompileCommonOptions
+  pure RustOptions {..}
+
+instance EntryPointOptions (RustOptions k) where
+  applyOptions opts =
+    set entryPointTarget (Just TargetRust)
+      . applyOptions (opts ^. rustCompileCommonOptions)

app/Commands/Extra/Rust.hs

@@ -0,0 +1,87 @@
+module Commands.Extra.Rust where
+
+import Commands.Base
+import System.Process qualified as P
+
+data RustArgs = RustArgs
+  { _rustDebug :: Bool,
+    _rustInputFile :: Path Abs File,
+    _rustOutputFile :: Path Abs File,
+    _rustOptimizationLevel :: Maybe Int
+  }
+
+makeLenses ''RustArgs
+
+rustCompile ::
+  forall r.
+  (Members '[App, EmbedIO] r) =>
+  RustArgs ->
+  Sem r ()
+rustCompile args = do
+  getRustcCliArgs args >>= runRustc
+
+getRustcCliArgs :: (Members '[App, EmbedIO] r) => RustArgs -> Sem r [String]
+getRustcCliArgs args = do
+  buildDir <- askBuildDir
+  return (nativeArgs buildDir args)
+
+nativeArgs :: Path Abs Dir -> RustArgs -> [String]
+nativeArgs buildDir args@RustArgs {..} =
+  commonArgs buildDir args ++ extraArgs
+  where
+    extraArgs :: [String]
+    extraArgs = run . execAccumList $ do
+      addOptimizationOption args
+      addArg (toFilePath _rustInputFile)
+
+addOptimizationOption :: (Member (Accum String) r) => RustArgs -> Sem r ()
+addOptimizationOption args = do
+  addArg "-C"
+  addArg $ "opt-level=" <> show (maybe defaultOptLevel (max 1) (args ^. rustOptimizationLevel))
+  where
+    defaultOptLevel :: Int
+    defaultOptLevel
+      | args ^. rustDebug = debugRustOptimizationLevel
+      | otherwise = defaultRustOptimizationLevel
+
+debugRustOptimizationLevel :: Int
+debugRustOptimizationLevel = 1
+
+defaultRustOptimizationLevel :: Int
+defaultRustOptimizationLevel = 3
+
+addArg :: (Member (Accum String) r) => String -> Sem r ()
+addArg = accum
+
+commonArgs :: Path Abs Dir -> RustArgs -> [String]
+commonArgs buildDir RustArgs {..} = run . execAccumList $ do
+  when _rustDebug $ do
+    addArg "-g"
+    addArg "-C"
+    addArg "debug-assertions=true"
+    addArg "-C"
+    addArg "overflow-checks=true"
+  addArg "-o"
+  addArg (toFilePath _rustOutputFile)
+  addArg ("-L")
+  addArg (toFilePath buildDir)
+
+runRustc ::
+  forall r.
+  (Members '[App, EmbedIO] r) =>
+  [String] ->
+  Sem r ()
+runRustc args = do
+  cp <- rustcBinPath
+  (exitCode, _, err) <- liftIO (P.readProcessWithExitCode cp args "")
+  case exitCode of
+    ExitSuccess -> return ()
+    _ -> exitFailMsg (pack err)
+  where
+    rustcBinPath :: Sem r String
+    rustcBinPath = do
+      p <- findExecutable $(mkRelFile "rustc")
+      maybe (exitFailMsg rustcNotFoundErr) (return . toFilePath) p
+
+    rustcNotFoundErr :: Text
+    rustcNotFoundErr = "Error: The rustc executable was not found. Please install the Rust compiler"

cntlines.sh

@@ -17,6 +17,7 @@ RUNTIME_CASM=$(count_ext '*.casm' runtime/casm)
 RUNTIME=$((RUNTIME_C+RUNTIME_RUST+RUNTIME_VAMPIR+RUNTIME_JVT+RUNTIME_CASM))
 
 BACKENDC=$(count src/Juvix/Compiler/Backend/C/)
+BACKENDRUST=$(count src/Juvix/Compiler/Backend/Rust/)
 CAIRO=$(count src/Juvix/Compiler/Backend/Cairo/)
 GEB=$(count src/Juvix/Compiler/Backend/Geb/)
 VAMPIR=$(count src/Juvix/Compiler/Backend/VampIR/)
@@ -40,7 +41,7 @@ PRELUDE=$(count src/Juvix/Prelude/)
 STORE=$(count src/Juvix/Compiler/Store/)
 
 FRONT=$((CONCRETE + INTERNAL + BUILTINS + PIPELINE))
-BACK=$((BACKENDC + GEB + VAMPIR + NOCK + REG + ASM + TREE + CORE + CASM + CAIRO))
+BACK=$((BACKENDC + BACKENDRUST + GEB + VAMPIR + NOCK + REG + ASM + TREE + CORE + CASM + CAIRO))
 OTHER=$((APP + STORE + HTML + EXTRA + DATA + PRELUDE))
 TESTS=$(count test/)
 
@@ -55,6 +56,7 @@ echo "Middle and back end: $BACK LOC"
 echo "   VampIR backend: $VAMPIR LOC"
 echo "   GEB backend: $GEB LOC"
 echo "   C backend: $BACKENDC LOC"
+echo "   Rust backend: $BACKENDRUST LOC"
 echo "   Cairo backend: $((CASM + CAIRO)) LOC"
 echo "   Nockma backend: $NOCK LOC"
 echo "   JuvixReg: $REG LOC"

package.yaml

@@ -38,6 +38,7 @@ extra-source-files:
   - include/package-base/**/*.juvix
   - runtime/c/include/**/*.h
   - runtime/c/**/*.a
+  - runtime/rust/target/**/*.rlib
   - runtime/tree/*.jvt
   - runtime/vampir/*.pir
   - runtime/casm/*.casm

runtime/Makefile

@@ -10,7 +10,7 @@ juvix_c:
 
 .PHONY: juvix_rust
 juvix_rust:
-	cd rust && cargo build --release
+	cd rust && cargo build && cargo build --release
 
 .PHONY: clean
 clean:

runtime/rust/src/apply.rs

@@ -29,11 +29,11 @@ impl Memory {
 
 #[macro_export]
 macro_rules! tapply {
-    ($lab:lifetime, $program:ident, $mem:ident, $fid:ident, $args:ident, $cl0:expr, $cargs0:expr) => {
+    ($lab:lifetime, $program:ident, $mem:ident, $fid:ident, $args:ident, $cl0:expr, $cargs0:expr) => {{
         let mut cl = $cl0;
         let mut cargs = $cargs0;
         loop {
-            match $mem.apply( cl, &cargs) {
+            match $mem.apply(cl, &cargs) {
                 apply::AppResult::Call(fid1, args1) => {
                     $fid = fid1;
                     $args = args1;
@@ -46,7 +46,7 @@ macro_rules! tapply {
                 }
             }
         }
-    };
+    }}
 }
 
 #[macro_export]

runtime/rust/src/closure.rs

@@ -30,8 +30,8 @@ impl Memory {
         p
     }
 
-    // Assigns stored closure args plus the provided closure args (`cargs`) to the
-    // argument space (`args`). Returns the function id to call.
+    // Returns the function id to call and the full arguments for closure call:
+    // stored closure args plus the provided closure args (`cargs`).
     pub fn call_closure(self: &Memory, cl: Word, cargs: &[Word]) -> (Word, Vec<Word>) {
         let mut args = Vec::from(self.get_closure_args(cl));
         args.extend(cargs);

runtime/rust/src/defs.rs

@@ -12,6 +12,11 @@ pub type SmallInt = i32;
 pub const INITIAL_STACK_CAPACITY: usize = 10 * 1024;
 pub const INITIAL_MEMORY_CAPACITY: usize = 10 * 1024;
 
+pub const BOOL_FALSE: Word = 0;
+pub const BOOL_TRUE: Word = 1;
+pub const OBJ_UNIT: Word = 0;
+pub const OBJ_VOID: Word = 0;
+
 pub fn word_to_usize(s: Word) -> usize {
     s as usize
 }