add new backend

turbopack/crates/turbo-tasks-backend/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "turbo-tasks-backend"
+version = "0.1.0"
+description = "TBD"
+license = "MPL-2.0"
+edition = "2021"
+autobenches = false
+
+[lib]
+bench = false
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+auto-hash-map = { workspace = true }
+dashmap = { workspace = true }
+once_cell = { workspace = true }
+parking_lot = { workspace = true }
+rustc-hash = { workspace = true }
+serde = { workspace = true }
+smallvec = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+turbo-prehash = { workspace = true }
+turbo-tasks = { workspace = true }
+turbo-tasks-hash = { workspace = true }
+turbo-tasks-malloc = { workspace = true, default-features = false }
+
+[build-dependencies]
+turbo-tasks-build = { workspace = true }

turbopack/crates/turbo-tasks-backend/build.rs

@@ -0,0 +1,5 @@
+use turbo_tasks_build::generate_register;
+
+fn main() {
+    generate_register();
+}

turbopack/crates/turbo-tasks-backend/src/backend/mod.rs

@@ -0,0 +1,37 @@
+mod operation;
+mod storage;
+
+use std::{collections::VecDeque, sync::Arc};
+
+use parking_lot::Mutex;
+use turbo_tasks::{backend::CachedTaskType, TaskId};
+
+use self::{operation::Operation, storage::Storage};
+use crate::{
+    data::{CachedDataItem, CachedDataUpdate},
+    utils::{bi_map::BiMap, chunked_vec::ChunkedVec},
+};
+
+pub struct TurboTasksBackend {
+    persisted_task_cache_log: Mutex<ChunkedVec<(Arc<CachedTaskType>, TaskId)>>,
+    task_cache: BiMap<Arc<CachedTaskType>, TaskId>,
+    persisted_storage_log: Mutex<ChunkedVec<CachedDataUpdate>>,
+    storage: Storage<TaskId, CachedDataItem>,
+    operations: Mutex<VecDeque<Box<dyn Operation>>>,
+}
+
+impl TurboTasksBackend {
+    pub fn new() -> Self {
+        Self {
+            persisted_task_cache_log: Mutex::new(ChunkedVec::new()),
+            task_cache: BiMap::new(),
+            persisted_storage_log: Mutex::new(ChunkedVec::new()),
+            storage: Storage::new(),
+            operations: Mutex::new(VecDeque::new()),
+        }
+    }
+
+    fn run_operation(&self, operation: Box<dyn Operation>) {
+        self.operations.lock().push_back(operation);
+    }
+}

turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs

@@ -0,0 +1 @@
+pub trait Operation {}

turbopack/crates/turbo-tasks-backend/src/backend/storage.rs

@@ -0,0 +1,34 @@
+use auto_hash_map::AutoMap;
+use dashmap::DashMap;
+use turbo_tasks::Keyed;
+
+enum PersistanceState {
+    /// We know that all state of the object is only in the cache and nothing is
+    /// stored in the persistent cache.
+    CacheOnly,
+    /// We know that some state of the object is stored in the persistent cache.
+    Persisted,
+    /// We have never checked the persistent cache for the state of the object.
+    Unknown,
+}
+
+struct InnerStorage<T: Keyed> {
+    map: AutoMap<T::Key, T::Value>,
+    persistance_state: PersistanceState,
+}
+
+pub struct Storage<K, T: Keyed> {
+    map: DashMap<K, InnerStorage<T>>,
+}
+
+impl<K, T: Keyed> Storage<K, T>
+where
+    K: Eq + std::hash::Hash + Clone,
+    T: Keyed,
+{
+    pub fn new() -> Self {
+        Self {
+            map: DashMap::new(),
+        }
+    }
+}

turbopack/crates/turbo-tasks-backend/src/data.rs

@@ -0,0 +1,184 @@
+use turbo_tasks::{util::SharedError, CellId, SharedReference, TaskId};
+
+#[derive(Debug, Copy, Clone)]
+pub struct CellRef {
+    task: TaskId,
+    cell: CellId,
+}
+
+#[derive(Debug, Copy, Clone)]
+pub enum OutputValue {
+    Cell(CellRef),
+    Output(TaskId),
+    Error,
+}
+impl OutputValue {
+    fn is_transient(&self) -> bool {
+        match self {
+            OutputValue::Cell(cell) => cell.task.is_transient(),
+            OutputValue::Output(task) => task.is_transient(),
+            OutputValue::Error => false,
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub enum RootType {
+    RootTask,
+    OnceTask,
+    ReadingStronglyConsistent,
+}
+
+#[derive(Debug, Copy, Clone)]
+pub enum InProgressState {
+    Scheduled { clean: bool },
+    InProgress { clean: bool, stale: bool },
+}
+
+#[turbo_tasks::with_key]
+#[derive(Debug, Clone)]
+pub enum CachedDataItem {
+    // Output
+    Output {
+        value: OutputValue,
+    },
+    Collectible {
+        collectible: CellRef,
+        value: (),
+    },
+
+    // State
+    Dirty {
+        value: (),
+    },
+    DirtyWhenPersisted {
+        value: (),
+    },
+
+    // Children
+    Child {
+        task: TaskId,
+        value: (),
+    },
+
+    // Cells
+    CellData {
+        cell: CellId,
+        value: SharedReference,
+    },
+
+    // Dependencies
+    OutputDependency {
+        target: TaskId,
+        value: (),
+    },
+    CellDependency {
+        target: CellRef,
+        value: (),
+    },
+
+    // Dependent
+    OutputDependent {
+        task: TaskId,
+        value: (),
+    },
+    CellDependent {
+        cell: CellId,
+        task: TaskId,
+        value: (),
+    },
+
+    // Aggregation Graph
+    AggregationNumber {
+        value: u32,
+    },
+    Follower {
+        task: TaskId,
+        value: (),
+    },
+    Upper {
+        task: TaskId,
+        value: (),
+    },
+
+    // Aggregated Data
+    AggregatedDirtyTask {
+        task: TaskId,
+        value: (),
+    },
+    AggregatedCollectible {
+        collectible: CellRef,
+        value: (),
+    },
+    AggregatedUnfinishedTasks {
+        value: u32,
+    },
+
+    // Transient Root Type
+    AggregateRootType {
+        value: RootType,
+    },
+
+    // Transient In Progress state
+    InProgress {
+        value: InProgressState,
+    },
+    OutdatedCollectible {
+        collectible: CellRef,
+        value: (),
+    },
+    OutdatedOutputDependency {
+        target: TaskId,
+        value: (),
+    },
+    OutdatedCellDependency {
+        target: CellRef,
+        value: (),
+    },
+
+    // Transient Error State
+    Error {
+        value: SharedError,
+    },
+}
+
+impl CachedDataItem {
+    pub fn is_persistent(&self) -> bool {
+        match self {
+            CachedDataItem::Output { value } => value.is_transient(),
+            CachedDataItem::Collectible { collectible, .. } => !collectible.task.is_transient(),
+            CachedDataItem::Dirty { .. } => true,
+            CachedDataItem::DirtyWhenPersisted { .. } => true,
+            CachedDataItem::Child { task, .. } => !task.is_transient(),
+            CachedDataItem::CellData { .. } => true,
+            CachedDataItem::OutputDependency { target, .. } => !target.is_transient(),
+            CachedDataItem::CellDependency { target, .. } => !target.task.is_transient(),
+            CachedDataItem::OutputDependent { task, .. } => !task.is_transient(),
+            CachedDataItem::CellDependent { task, .. } => !task.is_transient(),
+            CachedDataItem::AggregationNumber { .. } => true,
+            CachedDataItem::Follower { task, .. } => !task.is_transient(),
+            CachedDataItem::Upper { task, .. } => !task.is_transient(),
+            CachedDataItem::AggregatedDirtyTask { task, .. } => !task.is_transient(),
+            CachedDataItem::AggregatedCollectible { collectible, .. } => {
+                !collectible.task.is_transient()
+            }
+            CachedDataItem::AggregatedUnfinishedTasks { .. } => true,
+            CachedDataItem::AggregateRootType { .. } => false,
+            CachedDataItem::InProgress { .. } => false,
+            CachedDataItem::OutdatedCollectible { .. } => false,
+            CachedDataItem::OutdatedOutputDependency { .. } => false,
+            CachedDataItem::OutdatedCellDependency { .. } => false,
+            CachedDataItem::Error { .. } => false,
+        }
+    }
+}
+
+trait IsDefault {
+    fn is_default(&self) -> bool;
+}
+
+pub struct CachedDataUpdate {
+    pub task: TaskId,
+    pub key: CachedDataItemKey,
+    pub value: Option<CachedDataItemValue>,
+}

turbopack/crates/turbo-tasks-backend/src/lib.rs

@@ -0,0 +1,3 @@
+mod backend;
+mod data;
+mod utils;

turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs

@@ -0,0 +1,50 @@
+use std::{borrow::Borrow, hash::Hash};
+
+use dashmap::{mapref::entry::Entry, DashMap};
+
+pub struct BiMap<K, V> {
+    forward: DashMap<K, V>,
+    reverse: DashMap<V, K>,
+}
+
+impl<K, V> BiMap<K, V>
+where
+    K: Eq + Hash + Clone,
+    V: Eq + Hash + Clone,
+{
+    pub fn new() -> Self {
+        Self {
+            forward: DashMap::new(),
+            reverse: DashMap::new(),
+        }
+    }
+
+    pub fn lookup_forward<Q>(&self, key: &Q) -> Option<V>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq,
+    {
+        self.forward.get(key).map(|v| v.value().clone())
+    }
+
+    pub fn lookup_reverse<Q>(&self, key: &Q) -> Option<K>
+    where
+        V: Borrow<Q>,
+        Q: Hash + Eq,
+    {
+        self.reverse.get(key).map(|v| v.value().clone())
+    }
+
+    pub fn try_insert(&self, key: K, value: V) -> Result<(), V> {
+        match self.forward.entry(key) {
+            Entry::Occupied(e) => Err(e.get().clone()),
+            Entry::Vacant(e) => {
+                let e = e.insert_entry(value.clone());
+                let key = e.key().clone();
+                drop(e);
+                self.reverse.insert(value, key);
+                Ok(())
+            }
+        }
+    }
+}