[Turbopack] add new backend

Cargo.toml

@@ -45,6 +45,7 @@ swc-ast-explorer = { path = "turbopack/crates/turbopack-swc-ast-explorer" }
 turbo-prehash = { path = "turbopack/crates/turbo-prehash" }
 turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false }
 turbo-tasks = { path = "turbopack/crates/turbo-tasks" }
+turbo-tasks-backend = { path = "turbopack/crates/turbo-tasks-backend" }
 turbo-tasks-build = { path = "turbopack/crates/turbo-tasks-build" }
 turbo-tasks-bytes = { path = "turbopack/crates/turbo-tasks-bytes" }
 turbo-tasks-env = { path = "turbopack/crates/turbo-tasks-env" }
@@ -143,6 +144,7 @@ dunce = "1.0.3"
 either = "1.9.0"
 futures = "0.3.26"
 futures-retry = "0.6.0"
+hashbrown = "0.14.5"
 httpmock = { version = "0.6.8", default-features = false }
 image = { version = "0.25.0", default-features = false }
 indexmap = "1.9.2"

crates/napi/Cargo.toml

@@ -37,6 +37,8 @@ __internal_dhat-heap = ["dhat"]
 # effectively does nothing.
 __internal_dhat-ad-hoc = ["dhat"]
 
+new-backend = ["dep:turbo-tasks-backend"]
+
 # Enable specific tls features per-target.
 [target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies]
 next-core = { workspace = true, features = ["native-tls"] }
@@ -105,6 +107,7 @@ lightningcss-napi = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 turbo-tasks = { workspace = true }
 turbo-tasks-memory = { workspace = true }
+turbo-tasks-backend = { workspace = true, optional = true }
 turbo-tasks-fs = { workspace = true }
 next-api = { workspace = true }
 next-build = { workspace = true }

crates/napi/src/next_api/project.rs

@@ -1,4 +1,4 @@
-use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
+use std::{path::PathBuf, sync::Arc, thread, time::Duration};
 
 use anyhow::{anyhow, bail, Context, Result};
 use napi::{
@@ -24,7 +24,6 @@
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
 use turbo_tasks::{Completion, RcStr, ReadRef, TransientInstance, TurboTasks, UpdateInfo, Vc};
 use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath};
-use turbo_tasks_memory::MemoryBackend;
 use turbopack_core::{
     diagnostics::PlainDiagnostic,
     error::PrettyPrintError,
@@ -44,8 +43,8 @@
 use super::{
     endpoint::ExternalEndpoint,
     utils::{
-        get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, RootTask,
-        TurbopackResult, VcArc,
+        create_turbo_tasks, get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue,
+        NextBackend, RootTask, TurbopackResult, VcArc,
     },
 };
 use crate::register;
@@ -99,7 +98,7 @@
 
     /// next.config's distDir. Project initialization occurs eariler than
     /// deserializing next.config, so passing it as separate option.
-    pub dist_dir: Option<String>,
+    pub dist_dir: String,
 
     /// Filesystem watcher options.
     pub watch: NapiWatchOptions,
@@ -273,7 +272,7 @@
 }
 
 pub struct ProjectInstance {
-    turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
+    turbo_tasks: Arc<TurboTasks<NextBackend>>,
     container: Vc<ProjectContainer>,
     exit_receiver: tokio::sync::Mutex<Option<ExitReceiver>>,
 }
@@ -309,10 +308,7 @@
         let subscriber = Registry::default();
 
         let subscriber = subscriber.with(EnvFilter::builder().parse(trace).unwrap());
-        let dist_dir = options
-            .dist_dir
-            .as_ref()
-            .map_or_else(|| ".next".to_string(), |d| d.to_string());
+        let dist_dir = options.dist_dir.clone();
 
         let internal_dir = PathBuf::from(&options.project_path).join(dist_dir);
         std::fs::create_dir_all(&internal_dir)
@@ -338,27 +334,30 @@
         subscriber.init();
     }
 
-    let turbo_tasks = TurboTasks::new(MemoryBackend::new(
-        turbo_engine_options
-            .memory_limit
-            .map(|m| m as usize)
-            .unwrap_or(usize::MAX),
-    ));
-    let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
-    if let Some(stats_path) = stats_path {
-        let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
-        exit.on_exit(async move {
-            tokio::task::spawn_blocking(move || {
-                let mut file = std::fs::File::create(&stats_path)
-                    .with_context(|| format!("failed to create or open {stats_path:?}"))?;
-                serde_json::to_writer(&file, &task_stats)
-                    .context("failed to serialize or write task statistics")?;
-                file.flush().context("failed to flush file")
-            })
-            .await
-            .unwrap()
-            .unwrap();
-        });
+    let memory_limit = turbo_engine_options
+        .memory_limit
+        .map(|m| m as usize)
+        .unwrap_or(usize::MAX);
+    let turbo_tasks = create_turbo_tasks(PathBuf::from(&options.dist_dir), memory_limit)?;
+    #[cfg(not(feature = "new-backend"))]
+    {
+        use std::io::Write;
+        let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
+        if let Some(stats_path) = stats_path {
+            let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
+            exit.on_exit(async move {
+                tokio::task::spawn_blocking(move || {
+                    let mut file = std::fs::File::create(&stats_path)
+                        .with_context(|| format!("failed to create or open {stats_path:?}"))?;
+                    serde_json::to_writer(&file, &task_stats)
+                        .context("failed to serialize or write task statistics")?;
+                    file.flush().context("failed to flush file")
+                })
+                .await
+                .unwrap()
+                .unwrap();
+            });
+        }
     }
     let options: ProjectOptions = options.into();
     let container = turbo_tasks
@@ -502,7 +501,7 @@
     fn from_route(
         pathname: String,
         value: Route,
-        turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
+        turbo_tasks: &Arc<TurboTasks<NextBackend>>,
     ) -> Self {
         let convert_endpoint = |endpoint: Vc<Box<dyn Endpoint>>| {
             Some(External::new(ExternalEndpoint(VcArc::new(
@@ -569,7 +568,7 @@
 impl NapiMiddleware {
     fn from_middleware(
         value: &Middleware,
-        turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
+        turbo_tasks: &Arc<TurboTasks<NextBackend>>,
     ) -> Result<Self> {
         Ok(NapiMiddleware {
             endpoint: External::new(ExternalEndpoint(VcArc::new(
@@ -589,7 +588,7 @@
 impl NapiInstrumentation {
     fn from_instrumentation(
         value: &Instrumentation,
-        turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
+        turbo_tasks: &Arc<TurboTasks<NextBackend>>,
     ) -> Result<Self> {
         Ok(NapiInstrumentation {
             node_js: External::new(ExternalEndpoint(VcArc::new(
@@ -931,7 +930,7 @@
     }
 }
 
 /// Subscribes to lifecycle events of the compilation.
 /// Emits an [UpdateMessage::Start] event when any computation starts.
 /// Emits an [UpdateMessage::End] event when there was no computation for the
 /// specified time (`aggregation_ms`). The [UpdateMessage::End] event contains

crates/napi/src/next_api/utils.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, future::Future, ops::Deref, sync::Arc};
+use std::{collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc};
 
 use anyhow::{anyhow, Context, Result};
 use napi::{
@@ -9,7 +9,6 @@ use napi::{
 use serde::Serialize;
 use turbo_tasks::{ReadRef, TaskId, TryJoinIterExt, TurboTasks, Vc};
 use turbo_tasks_fs::FileContent;
-use turbo_tasks_memory::MemoryBackend;
 use turbopack_core::{
     diagnostics::{Diagnostic, DiagnosticContextExt, PlainDiagnostic},
     error::PrettyPrintError,
@@ -19,22 +18,41 @@ use turbopack_core::{
 
 use crate::util::log_internal_error_and_inform;
 
+#[cfg(not(feature = "new-backend"))]
+pub type NextBackend = turbo_tasks_memory::MemoryBackend;
+#[cfg(feature = "new-backend")]
+pub type NextBackend = turbo_tasks_backend::TurboTasksBackend;
+
+#[allow(unused_variables, reason = "feature-gated")]
+pub fn create_turbo_tasks(
+    output_path: PathBuf,
+    memory_limit: usize,
+) -> Result<Arc<TurboTasks<NextBackend>>> {
+    #[cfg(not(feature = "new-backend"))]
+    let backend = TurboTasks::new(turbo_tasks_memory::MemoryBackend::new(memory_limit));
+    #[cfg(feature = "new-backend")]
+    let backend = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new(Arc::new(
+        turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?,
+    )));
+    Ok(backend)
+}
+
 /// A helper type to hold both a Vc operation and the TurboTasks root process.
 /// Without this, we'd need to pass both individually all over the place
 #[derive(Clone)]
 pub struct VcArc<T> {
-    turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
+    turbo_tasks: Arc<TurboTasks<NextBackend>>,
     /// The Vc. Must be resolved, otherwise you are referencing an inactive
     /// operation.
     vc: T,
 }
 
 impl<T> VcArc<T> {
-    pub fn new(turbo_tasks: Arc<TurboTasks<MemoryBackend>>, vc: T) -> Self {
+    pub fn new(turbo_tasks: Arc<TurboTasks<NextBackend>>, vc: T) -> Self {
         Self { turbo_tasks, vc }
     }
 
-    pub fn turbo_tasks(&self) -> &Arc<TurboTasks<MemoryBackend>> {
+    pub fn turbo_tasks(&self) -> &Arc<TurboTasks<NextBackend>> {
         &self.turbo_tasks
     }
 }
@@ -57,7 +75,7 @@ pub fn serde_enum_to_string<T: Serialize>(value: &T) -> Result<String> {
 /// The root of our turbopack computation.
 pub struct RootTask {
     #[allow(dead_code)]
-    turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
+    turbo_tasks: Arc<TurboTasks<NextBackend>>,
     #[allow(dead_code)]
     task_id: Option<TaskId>,
 }
@@ -301,7 +319,7 @@ impl<T: ToNapiValue> ToNapiValue for TurbopackResult<T> {
 }
 
 pub fn subscribe<T: 'static + Send + Sync, F: Future<Output = Result<T>> + Send, V: ToNapiValue>(
-    turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
+    turbo_tasks: Arc<TurboTasks<NextBackend>>,
     func: JsFunction,
     handler: impl 'static + Sync + Send + Clone + Fn() -> F,
     mapper: impl 'static + Sync + Send + FnMut(ThreadSafeCallContext<T>) -> napi::Result<Vec<V>>,

crates/napi/src/turbotrace.rs

@@ -1,35 +1,35 @@
-use std::sync::Arc;
+use std::{path::PathBuf, sync::Arc};
 
 use napi::bindgen_prelude::*;
 use node_file_trace::{start, Args};
 use turbo_tasks::TurboTasks;
-use turbo_tasks_memory::MemoryBackend;
 use turbopack::{
     module_options::{EcmascriptOptionsContext, ModuleOptionsContext},
     resolve_options_context::ResolveOptionsContext,
 };
 
+use crate::next_api::utils::{self, NextBackend};
+
 #[napi]
-pub fn create_turbo_tasks(memory_limit: Option<i64>) -> External<Arc<TurboTasks<MemoryBackend>>> {
-    let turbo_tasks = TurboTasks::new(MemoryBackend::new(
-        memory_limit.map(|m| m as usize).unwrap_or(usize::MAX),
-    ));
-    External::new_with_size_hint(
-        turbo_tasks,
-        memory_limit.map(|u| u as usize).unwrap_or(usize::MAX),
-    )
+pub fn create_turbo_tasks(
+    output_path: String,
+    memory_limit: Option<i64>,
+) -> External<Arc<TurboTasks<NextBackend>>> {
+    let limit = memory_limit.map(|u| u as usize).unwrap_or(usize::MAX);
+    let turbo_tasks = utils::create_turbo_tasks(PathBuf::from(&output_path), limit)
+        .expect("Failed to create TurboTasks");
+    External::new_with_size_hint(turbo_tasks, limit)
 }
 
 #[napi]
 pub async fn run_turbo_tracing(
     options: Buffer,
-    turbo_tasks: Option<External<Arc<TurboTasks<MemoryBackend>>>>,
+    turbo_tasks: External<Arc<TurboTasks<NextBackend>>>,
 ) -> napi::Result<Vec<String>> {
     let args: Args = serde_json::from_slice(options.as_ref())?;
-    let turbo_tasks = turbo_tasks.map(|t| t.clone());
     let files = start(
         Arc::new(args),
-        turbo_tasks.as_ref(),
+        turbo_tasks.clone(),
         Some(ModuleOptionsContext {
             ecmascript: EcmascriptOptionsContext {
                 enable_types: true,

packages/next/src/build/collect-build-traces.ts

@@ -119,6 +119,7 @@ export async function collectBuildTraces({
       let turbotraceOutputPath: string | undefined
       let turbotraceFiles: string[] | undefined
       turboTasksForTrace = bindings.turbo.createTurboTasks(
+        distDir,
         (config.experimental.turbotrace?.memoryLimit ??
           TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
           1024 *

packages/next/src/build/index.ts

@@ -1379,6 +1379,7 @@ export default async function build(
               config.experimental?.turbo?.root ||
               config.outputFileTracingRoot ||
               dir,
+            distDir,
             nextConfig: config,
             jsConfig: await getTurbopackJsConfig(dir, config),
             watch: {

packages/next/src/build/swc/generated-native.d.ts

@@ -103,7 +103,7 @@ export interface NapiProjectOptions {
    * next.config's distDir. Project initialization occurs eariler than
    * deserializing next.config, so passing it as separate option.
    */
-  distDir?: string
+  distDir: string
   /** Filesystem watcher options. */
   watch: NapiWatchOptions
   /** The contents of next.config.js, serialized to JSON. */
@@ -363,11 +363,12 @@ export interface NapiRewrite {
   missing?: Array<NapiRouteHas>
 }
 export function createTurboTasks(
+  outputPath: string,
   memoryLimit?: number | undefined | null
 ): ExternalObject<TurboTasks>
 export function runTurboTracing(
   options: Buffer,
-  turboTasks?: ExternalObject<TurboTasks> | undefined | null
+  turboTasks: ExternalObject<TurboTasks>
 ): Promise<Array<string>>
 export function getTargetTriple(): string
 export function initHeapProfiler(): ExternalObject<RefCell>

packages/next/src/build/swc/index.ts

@@ -1054,6 +1054,7 @@ async function loadWasm(importPath = '') {
             Log.error('Wasm binding does not support trace yet')
           },
           createTurboTasks: function (
+            _outputPath: string,
             _memoryLimit?: number | undefined
           ): ExternalObject<TurboTasks> {
             throw new Error(
@@ -1234,8 +1235,11 @@ function loadNative(importPath?: string) {
             turboTasks
           )
         },
-        createTurboTasks(memoryLimit?: number): ExternalObject<TurboTasks> {
-          return bindings.createTurboTasks(memoryLimit)
+        createTurboTasks(
+          outputPath: string,
+          memoryLimit?: number
+        ): ExternalObject<TurboTasks> {
+          return bindings.createTurboTasks(outputPath, memoryLimit)
         },
         createProject: bindingToApi(customBindings ?? bindings, false),
         startTurbopackTraceServer(traceFilePath) {