From fcdf05f90f98852b157ef0c7b5561ef7ba630461 Mon Sep 17 00:00:00 2001 From: Fabian Murariu Date: Wed, 18 Dec 2024 17:52:15 +0000 Subject: [PATCH] remove Graph and PersistentGraph from Prop --- Cargo.lock | 161 +++++++++++------- Cargo.toml | 10 +- .../src/core/entities/properties/props.rs | 14 +- raphtory-api/src/core/mod.rs | 6 +- raphtory/src/algorithms/pathing/dijkstra.rs | 6 +- .../src/core/entities/properties/tprop.rs | 56 ++---- raphtory/src/core/mod.rs | 85 +++------ .../src/db/api/properties/temporal_props.rs | 14 +- raphtory/src/db/graph/graph.rs | 31 ---- raphtory/src/python/types/wrappers/prop.rs | 12 +- raphtory/src/serialise/graph.proto | 10 +- raphtory/src/serialise/proto_ext.rs | 126 ++++++++------ raphtory/src/serialise/serialise.rs | 8 - raphtory/src/vectors/template.rs | 3 +- 14 files changed, 233 insertions(+), 309 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd9484af9a..807cc82c56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,7 @@ dependencies = [ "const-random", "getrandom", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -769,18 +770,18 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.5.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" -version = "0.6.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" @@ -949,9 +950,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf" dependencies = [ "jobserver", "libc", @@ -1308,18 +1309,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1336,9 +1337,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" @@ -2531,11 +2532,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2620,9 +2621,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -2644,9 +2645,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -2671,7 +2672,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", @@ -2679,13 +2680,13 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "f6884a48c6826ec44f524c7456b163cebe9e55a18d7b5e307cb4f100371cc767" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "rustls 0.23.20", "rustls-native-certs 0.8.1", @@ -2702,7 +2703,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "pin-project-lite", "tokio", @@ -2720,7 +2721,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.1", + "hyper 1.5.2", "pin-project-lite", "socket2", "tokio", @@ -3404,9 +3405,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -4016,7 +4017,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.6", + "thiserror 2.0.8", "ucd-trie", ] @@ -4191,7 +4192,7 @@ dependencies = [ "headers", "http 1.2.0", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-util", "mime", "nix", @@ -4423,6 +4424,37 @@ dependencies = [ name = "pometry-storage" version = "0.14.0" +[[package]] +name = "pometry-storage-private" +version = "0.12.1" +dependencies = [ + "ahash", + "bincode", + "bytemuck", + "criterion", + "itertools 0.12.1", + "kdam", + "memmap2 0.9.5", + "once_cell", + "parking_lot", + "polars-arrow", + "polars-parquet", + "polars-utils", + "pretty_assertions", + "proptest", + "rand", + "raphtory-api", + "rayon", + "serde", + "serde_json", + "strum", + "tempfile", + "thiserror 1.0.69", + "tracing", + "tracing-subscriber", + "twox-hash 1.6.3", +] + [[package]] name = "portable-atomic" version = "1.10.0" @@ -4516,9 +4548,9 @@ dependencies = [ [[package]] name = "proptest" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", "bit-vec", @@ -4722,7 +4754,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls 0.23.20", "socket2", - "thiserror 2.0.6", + "thiserror 2.0.8", "tokio", "tracing", ] @@ -4741,7 +4773,7 @@ dependencies = [ "rustls 0.23.20", "rustls-pki-types", "slab", - "thiserror 2.0.6", + "thiserror 2.0.8", "tinyvec", "tracing", "web-time", @@ -4749,9 +4781,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -4859,7 +4891,7 @@ dependencies = [ "polars-core", "polars-io", "polars-parquet", - "pometry-storage", + "pometry-storage-private", "pretty_assertions", "proptest", "prost", @@ -4883,7 +4915,7 @@ dependencies = [ "streaming-stats", "tantivy", "tempfile", - "thiserror 2.0.6", + "thiserror 2.0.8", "tokio", "tracing", "zip", @@ -4908,7 +4940,7 @@ dependencies = [ "rayon", "rustc-hash 2.1.0", "serde", - "thiserror 2.0.6", + "thiserror 2.0.8", "tracing", "tracing-subscriber", "twox-hash 2.1.0", @@ -4922,7 +4954,7 @@ dependencies = [ "criterion", "csv", "flate2", - "pometry-storage", + "pometry-storage-private", "rand", "raphtory", "raphtory-api", @@ -4949,7 +4981,7 @@ dependencies = [ "pest", "pest_derive", "polars-arrow", - "pometry-storage", + "pometry-storage-private", "pretty_assertions", "proptest", "raphtory", @@ -4958,7 +4990,7 @@ dependencies = [ "serde_json", "sqlparser", "tempfile", - "thiserror 2.0.6", + "thiserror 2.0.8", "tokio", "tracing", "tracing-subscriber", @@ -4997,7 +5029,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror 2.0.6", + "thiserror 2.0.8", "tokio", "tracing", "tracing-opentelemetry", @@ -5036,7 +5068,7 @@ dependencies = [ "raphtory-graphql", "serde", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.8", "tokio", ] @@ -5152,7 +5184,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.31", + "hyper 0.14.32", "hyper-rustls 0.24.2", "ipnet", "js-sys", @@ -5193,8 +5225,8 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", - "hyper-rustls 0.27.3", + "hyper 1.5.2", + "hyper-rustls 0.27.4", "hyper-util", "ipnet", "js-sys", @@ -5274,9 +5306,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395b0c39c00f9296f3937624c1fa4e0ee44f8c0e4b2c49408179ef381c6c2e6e" +checksum = "41589aba99537475bf697f2118357cad1c31590c5a1b9f6d9fc4ad6d07503661" dependencies = [ "bytemuck", "byteorder", @@ -5414,7 +5446,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.0.1", + "security-framework 3.1.0", ] [[package]] @@ -5437,9 +5469,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -5554,9 +5586,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" dependencies = [ "bitflags 2.6.0", "core-foundation 0.10.0", @@ -5567,9 +5599,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" dependencies = [ "core-foundation-sys", "libc", @@ -6203,11 +6235,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "08f5383f3e0071702bf93ab5ee99b52d26936be9dedd9413067cbdcddcb6141a" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.8", ] [[package]] @@ -6223,9 +6255,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "f2f357fcec90b3caef6623a099691be676d033b40a058ac95d2a6ade6fa0c943" dependencies = [ "proc-macro2", "quote", @@ -6474,7 +6506,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.1", + "hyper 1.5.2", "hyper-timeout", "hyper-util", "percent-encoding", @@ -6651,6 +6683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", + "rand", "static_assertions", ] @@ -7380,9 +7413,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d52293fc86ea7cf13971b3bb81eb21683636e7ae24c729cdaf1b7c4157a352" +checksum = "ae9c1ea7b3a5e1f4b922ff856a129881167511563dc219869afe3787fc0c1a45" dependencies = [ "aes", "arbitrary", @@ -7400,7 +7433,7 @@ dependencies = [ "pbkdf2", "rand", "sha1", - "thiserror 2.0.6", + "thiserror 2.0.8", "time", "zeroize", "zopfli", diff --git a/Cargo.toml b/Cargo.toml index 13d7956660..bf3f405e63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,10 @@ members = [ "raphtory-graphql", "raphtory-api", ] -default-members = ["raphtory", "raphtory-graphql"] +default-members = [ + "raphtory", + "raphtory-graphql" +] resolver = "2" [workspace.package] @@ -40,9 +43,9 @@ debug = 0 [workspace.dependencies] #[public-storage] -pometry-storage = { version = ">=0.8.1", path = "pometry-storage" } +# pometry-storage = { version = ">=0.8.1", path = "pometry-storage" } #[private-storage] -# pometry-storage = { path = "pometry-storage-private", package = "pometry-storage-private" } +pometry-storage = { path = "pometry-storage-private", package = "pometry-storage-private" } async-graphql = { version = "7.0.12", features = [ "dynamic-schema", ] } @@ -136,6 +139,7 @@ memmap2 = { version = "0.9.4" } ahash = { version = "0.8.3", features = ["serde"] } strum = { version = "0.26.1", features = ["derive"] } bytemuck = { version = "1.18.0", features = ["derive"] } +bytes = {version = "1.9", features = ["serde"]} ouroboros = "0.18.3" url = "2.2" base64-compat = { package = "base64-compat", version = "1.0.0" } diff --git a/raphtory-api/src/core/entities/properties/props.rs b/raphtory-api/src/core/entities/properties/props.rs index 152c9ba392..31ac677aaf 100644 --- a/raphtory-api/src/core/entities/properties/props.rs +++ b/raphtory-api/src/core/entities/properties/props.rs @@ -259,13 +259,15 @@ impl PropMapper { } } - pub fn set_id_and_dtype(&self, key: impl Into, id: usize, dtype: PropType) { - let mut dtypes = self.dtypes.write(); - self.set_id(key, id); - if dtypes.len() <= id { - dtypes.resize(id + 1, PropType::Empty); + pub fn set_id_and_dtype(&self, key: impl Into, id: usize, dtype: Option) { + if let Some(dtype) = dtype { + let mut dtypes = self.dtypes.write(); + self.set_id(key, id); + if dtypes.len() <= id { + dtypes.resize(id + 1, PropType::Empty); + } + dtypes[id] = dtype; } - dtypes[id] = dtype; } pub fn get_dtype(&self, prop_id: usize) -> Option { diff --git a/raphtory-api/src/core/mod.rs b/raphtory-api/src/core/mod.rs index 1948024dc6..655902fca0 100644 --- a/raphtory-api/src/core/mod.rs +++ b/raphtory-api/src/core/mod.rs @@ -33,8 +33,7 @@ pub enum PropType { List, Map, NDTime, - Graph, - PersistentGraph, + Blob, Document, DTime, } @@ -56,8 +55,7 @@ impl Display for PropType { PropType::List => "List", PropType::Map => "Map", PropType::NDTime => "NDTime", - PropType::Graph => "Graph", - PropType::PersistentGraph => "PersistentGraph", + PropType::Blob => "Blob", PropType::Document => "Document", PropType::DTime => "DTime", }; diff --git a/raphtory/src/algorithms/pathing/dijkstra.rs b/raphtory/src/algorithms/pathing/dijkstra.rs index e61c3bc56d..08f84d520f 100644 --- a/raphtory/src/algorithms/pathing/dijkstra.rs +++ b/raphtory/src/algorithms/pathing/dijkstra.rs @@ -107,9 +107,8 @@ pub fn dijkstra_single_source_shortest_paths return Err("Weight type: List, not supported"), PropType::Map => return Err("Weight type: Map, not supported"), PropType::DTime => return Err("Weight type: DTime, not supported"), + PropType::Blob => return Err("Weight type: Blob, not supported"), PropType::NDTime => return Err("Weight type: NDTime, not supported"), - PropType::Graph => return Err("Weight type: Graph, not supported"), - PropType::PersistentGraph => return Err("Weight type: Persistent Graph, not supported"), PropType::Document => return Err("Weight type: Document, not supported"), }; let max_val = match weight_type.unwrap() { @@ -128,8 +127,7 @@ pub fn dijkstra_single_source_shortest_paths return Err("Weight type: Map, not supported"), PropType::DTime => return Err("Weight type: DTime, not supported"), PropType::NDTime => return Err("Weight type: NDTime, not supported"), - PropType::Graph => return Err("Weight type: Graph, not supported"), - PropType::PersistentGraph => return Err("Weight type: Persistent Graph, not supported"), + PropType::Blob => return Err("Weight type: Blob, not supported"), PropType::Document => return Err("Weight type: Document, not supported"), }; let mut heap = BinaryHeap::new(); diff --git a/raphtory/src/core/entities/properties/tprop.rs b/raphtory/src/core/entities/properties/tprop.rs index 59d4de19dd..399a7e41ac 100644 --- a/raphtory/src/core/entities/properties/tprop.rs +++ b/raphtory/src/core/entities/properties/tprop.rs @@ -3,10 +3,7 @@ use crate::{ entities::properties::tcell::TCell, storage::timeindex::TimeIndexEntry, utils::errors::GraphError, DocumentInput, Prop, PropType, }, - db::{ - api::storage::graph::tprop_storage_ops::TPropOps, - graph::{graph::Graph, views::deletion_graph::PersistentGraph}, - }, + db::api::storage::graph::tprop_storage_ops::TPropOps, }; use chrono::{DateTime, NaiveDateTime, Utc}; use raphtory_api::{core::storage::arc_str::ArcStr, iter::BoxedLIter}; @@ -29,9 +26,8 @@ pub enum TProp { F64(TCell), Bool(TCell), DTime(TCell>), + Blob(TCell>), NDTime(TCell), - Graph(TCell), - PersistentGraph(TCell), Document(TCell), List(TCell>>), Map(TCell>>), @@ -52,12 +48,11 @@ impl TProp { TProp::F64(_) => PropType::F64, TProp::Bool(_) => PropType::Bool, TProp::NDTime(_) => PropType::NDTime, - TProp::Graph(_) => PropType::Graph, - TProp::PersistentGraph(_) => PropType::PersistentGraph, TProp::Document(_) => PropType::Document, TProp::List(_) => PropType::List, TProp::Map(_) => PropType::Map, TProp::DTime(_) => PropType::DTime, + TProp::Blob(_) => PropType::Bool, } } @@ -75,8 +70,7 @@ impl TProp { Prop::Bool(value) => TProp::Bool(TCell::new(t, value)), Prop::DTime(value) => TProp::DTime(TCell::new(t, value)), Prop::NDTime(value) => TProp::NDTime(TCell::new(t, value)), - Prop::Graph(value) => TProp::Graph(TCell::new(t, value)), - Prop::PersistentGraph(value) => TProp::PersistentGraph(TCell::new(t, value)), + Prop::Blob(value) => TProp::Blob(TCell::new(t, value)), Prop::Document(value) => TProp::Document(TCell::new(t, value)), Prop::List(value) => TProp::List(TCell::new(t, value)), Prop::Map(value) => TProp::Map(TCell::new(t, value)), @@ -126,10 +120,7 @@ impl TProp { (TProp::NDTime(cell), Prop::NDTime(a)) => { cell.set(t, a); } - (TProp::Graph(cell), Prop::Graph(a)) => { - cell.set(t, a); - } - (TProp::PersistentGraph(cell), Prop::PersistentGraph(a)) => { + (TProp::Blob(cell), Prop::Blob(a)) => { cell.set(t, a); } (TProp::Document(cell), Prop::Document(a)) => { @@ -166,13 +157,9 @@ impl TProp { TProp::NDTime(cell) => { Box::new(cell.iter().map(|(t, value)| (*t, Prop::NDTime(*value)))) } - TProp::Graph(cell) => Box::new( + TProp::Blob(cell) => Box::new( cell.iter() - .map(|(t, value)| (*t, Prop::Graph(value.clone()))), - ), - TProp::PersistentGraph(cell) => Box::new( - cell.iter() - .map(|(t, value)| (*t, Prop::PersistentGraph(value.clone()))), + .map(|(t, value)| (*t, Prop::Blob(value.clone()))), ), TProp::Document(cell) => Box::new( cell.iter() @@ -210,13 +197,9 @@ impl TProp { TProp::NDTime(cell) => { Box::new(cell.iter_t().map(|(t, value)| (t, Prop::NDTime(*value)))) } - TProp::Graph(cell) => Box::new( + TProp::Blob(cell) => Box::new( cell.iter_t() - .map(|(t, value)| (t, Prop::Graph(value.clone()))), - ), - TProp::PersistentGraph(cell) => Box::new( - cell.iter_t() - .map(|(t, value)| (t, Prop::PersistentGraph(value.clone()))), + .map(|(t, value)| (t, Prop::Blob(value.clone()))), ), TProp::Document(cell) => Box::new( cell.iter_t() @@ -286,13 +269,9 @@ impl TProp { cell.iter_window(r) .map(|(t, value)| (*t, Prop::NDTime(*value))), ), - TProp::Graph(cell) => Box::new( - cell.iter_window(r) - .map(|(t, value)| (*t, Prop::Graph(value.clone()))), - ), - TProp::PersistentGraph(cell) => Box::new( + TProp::Blob(cell) => Box::new( cell.iter_window(r) - .map(|(t, value)| (*t, Prop::PersistentGraph(value.clone()))), + .map(|(t, value)| (*t, Prop::Blob(value.clone()))), ), TProp::Document(cell) => Box::new( cell.iter_window(r) @@ -326,12 +305,7 @@ impl<'a> TPropOps<'a> for &'a TProp { TProp::Bool(cell) => cell.last_before(t).map(|(t, v)| (t, Prop::Bool(*v))), TProp::DTime(cell) => cell.last_before(t).map(|(t, v)| (t, Prop::DTime(*v))), TProp::NDTime(cell) => cell.last_before(t).map(|(t, v)| (t, Prop::NDTime(*v))), - TProp::Graph(cell) => cell - .last_before(t) - .map(|(t, v)| (t, Prop::Graph(v.clone()))), - TProp::PersistentGraph(cell) => cell - .last_before(t) - .map(|(t, v)| (t, Prop::PersistentGraph(v.clone()))), + TProp::Blob(cell) => cell.last_before(t).map(|(t, v)| (t, Prop::Blob(v.clone()))), TProp::Document(cell) => cell .last_before(t) .map(|(t, v)| (t, Prop::Document(v.clone()))), @@ -366,8 +340,7 @@ impl<'a> TPropOps<'a> for &'a TProp { TProp::Bool(cell) => cell.at(ti).map(|v| Prop::Bool(*v)), TProp::DTime(cell) => cell.at(ti).map(|v| Prop::DTime(*v)), TProp::NDTime(cell) => cell.at(ti).map(|v| Prop::NDTime(*v)), - TProp::Graph(cell) => cell.at(ti).map(|v| Prop::Graph(v.clone())), - TProp::PersistentGraph(cell) => cell.at(ti).map(|v| Prop::PersistentGraph(v.clone())), + TProp::Blob(cell) => cell.at(ti).map(|v| Prop::Blob(v.clone())), TProp::Document(cell) => cell.at(ti).map(|v| Prop::Document(v.clone())), TProp::List(cell) => cell.at(ti).map(|v| Prop::List(v.clone())), TProp::Map(cell) => cell.at(ti).map(|v| Prop::Map(v.clone())), @@ -389,8 +362,7 @@ impl<'a> TPropOps<'a> for &'a TProp { TProp::Bool(v) => v.len(), TProp::DTime(v) => v.len(), TProp::NDTime(v) => v.len(), - TProp::Graph(v) => v.len(), - TProp::PersistentGraph(v) => v.len(), + TProp::Blob(v) => v.len(), TProp::Document(v) => v.len(), TProp::List(v) => v.len(), TProp::Map(v) => v.len(), diff --git a/raphtory/src/core/mod.rs b/raphtory/src/core/mod.rs index 55409dde72..da03e30081 100644 --- a/raphtory/src/core/mod.rs +++ b/raphtory/src/core/mod.rs @@ -24,10 +24,6 @@ //! * `macOS` //! -use crate::{ - db::graph::{graph::Graph, views::deletion_graph::PersistentGraph}, - prelude::GraphViewOps, -}; use chrono::{DateTime, NaiveDateTime, Utc}; use itertools::Itertools; use raphtory_api::core::storage::arc_str::ArcStr; @@ -90,8 +86,7 @@ pub enum Prop { Map(Arc>), NDTime(NaiveDateTime), DTime(DateTime), - Graph(Graph), - PersistentGraph(PersistentGraph), + Blob(Vec), Document(DocumentInput), } @@ -115,6 +110,7 @@ impl Hash for Prop { } Prop::Bool(b) => b.hash(state), Prop::NDTime(dt) => dt.hash(state), + Prop::Blob(b) => b.hash(state), Prop::DTime(dt) => dt.hash(state), Prop::List(v) => { for prop in v.iter() { @@ -127,22 +123,6 @@ impl Hash for Prop { prop.hash(state); } } - Prop::Graph(g) => { - for node in g.nodes() { - node.node.hash(state); - } - for edge in g.edges() { - edge.edge.pid().hash(state); - } - } - Prop::PersistentGraph(pg) => { - for node in pg.nodes() { - node.node.hash(state); - } - for edge in pg.edges() { - edge.edge.pid().hash(state); - } - } Prop::Document(d) => d.hash(state), } } @@ -186,8 +166,7 @@ impl Prop { Prop::List(_) => PropType::List, Prop::Map(_) => PropType::Map, Prop::NDTime(_) => PropType::NDTime, - Prop::Graph(_) => PropType::Graph, - Prop::PersistentGraph(_) => PropType::PersistentGraph, + Prop::Blob(_) => PropType::Blob, Prop::Document(_) => PropType::Document, Prop::DTime(_) => PropType::DTime, } @@ -323,18 +302,15 @@ pub trait PropUnwrap: Sized { self.into_ndtime().unwrap() } - fn into_graph(self) -> Option; - - fn into_persistent_graph(self) -> Option; - - fn unwrap_graph(self) -> Graph { - self.into_graph().unwrap() - } - fn into_document(self) -> Option; fn unwrap_document(self) -> DocumentInput { self.into_document().unwrap() } + + fn into_blob(self) -> Option>; + fn unwrap_blob(self) -> Vec { + self.into_blob().unwrap() + } } impl PropUnwrap for Option

{ @@ -390,17 +366,13 @@ impl PropUnwrap for Option

{ self.and_then(|p| p.into_ndtime()) } - fn into_graph(self) -> Option { - self.and_then(|p| p.into_graph()) - } - - fn into_persistent_graph(self) -> Option { - self.and_then(|p| p.into_persistent_graph()) - } - fn into_document(self) -> Option { self.and_then(|p| p.into_document()) } + + fn into_blob(self) -> Option> { + self.and_then(|p| p.into_blob()) + } } impl PropUnwrap for Prop { @@ -508,25 +480,17 @@ impl PropUnwrap for Prop { } } - fn into_graph(self) -> Option { - if let Prop::Graph(g) = self { - Some(g) - } else { - None - } - } - - fn into_persistent_graph(self) -> Option { - if let Prop::PersistentGraph(g) = self { - Some(g) + fn into_document(self) -> Option { + if let Prop::Document(d) = self { + Some(d) } else { None } } - fn into_document(self) -> Option { - if let Prop::Document(d) = self { - Some(d) + fn into_blob(self) -> Option> { + if let Prop::Blob(v) = self { + Some(v) } else { None } @@ -548,18 +512,7 @@ impl Display for Prop { Prop::Bool(value) => write!(f, "{}", value), Prop::DTime(value) => write!(f, "{}", value), Prop::NDTime(value) => write!(f, "{}", value), - Prop::Graph(value) => write!( - f, - "Graph(num_nodes={}, num_edges={})", - value.count_nodes(), - value.count_edges() - ), - Prop::PersistentGraph(value) => write!( - f, - "Graph(num_nodes={}, num_edges={})", - value.count_nodes(), - value.count_edges() - ), + Prop::Blob(value) => write!(f, "{:?}", value), Prop::List(value) => { write!( f, diff --git a/raphtory/src/db/api/properties/temporal_props.rs b/raphtory/src/db/api/properties/temporal_props.rs index 48fe62d35c..29598e46d8 100644 --- a/raphtory/src/db/api/properties/temporal_props.rs +++ b/raphtory/src/db/api/properties/temporal_props.rs @@ -1,10 +1,6 @@ use crate::{ core::{DocumentInput, Prop, PropType, PropUnwrap}, - db::{ - api::{properties::internal::PropertiesOps, view::BoxedLIter}, - graph::views::deletion_graph::PersistentGraph, - }, - prelude::Graph, + db::api::{properties::internal::PropertiesOps, view::BoxedLIter}, }; use chrono::{DateTime, NaiveDateTime, Utc}; use raphtory_api::core::storage::arc_str::ArcStr; @@ -240,12 +236,8 @@ impl PropUnwrap for TemporalPropertyView

{ self.latest().into_ndtime() } - fn into_graph(self) -> Option { - self.latest().into_graph() - } - - fn into_persistent_graph(self) -> Option { - self.latest().into_persistent_graph() + fn into_blob(self) -> Option> { + self.latest().into_blob() } fn into_document(self) -> Option { diff --git a/raphtory/src/db/graph/graph.rs b/raphtory/src/db/graph/graph.rs index 7dca250363..745b3eb600 100644 --- a/raphtory/src/db/graph/graph.rs +++ b/raphtory/src/db/graph/graph.rs @@ -3415,37 +3415,6 @@ mod db_tests { ); } - #[test] - fn persistent_graph_as_prop() { - let g = Graph::new(); - g.add_node(0, 1, [("graph", Prop::Graph(Graph::new()))], None) - .unwrap(); - g.add_node( - 0, - 1, - [("pgraph", Prop::PersistentGraph(PersistentGraph::new()))], - None, - ) - .unwrap(); - g.add_node(0, 1, [("bool", Prop::Bool(true))], None) - .unwrap(); - g.add_node(0, 1, [("u32", Prop::U32(2))], None).unwrap(); - assert_eq!( - g.node(1) - .unwrap() - .properties() - .temporal() - .keys() - .collect::>(), - vec![ - ArcStr("graph".into()), - ArcStr("pgraph".into()), - ArcStr("bool".into()), - ArcStr("u32".into()), - ] - ); - } - #[test] fn test_unique_property() { let g = Graph::new(); diff --git a/raphtory/src/python/types/wrappers/prop.rs b/raphtory/src/python/types/wrappers/prop.rs index 8fb5529607..ed971619d9 100644 --- a/raphtory/src/python/types/wrappers/prop.rs +++ b/raphtory/src/python/types/wrappers/prop.rs @@ -29,8 +29,7 @@ impl<'py> IntoPyObject<'py> for Prop { Prop::F64(f64) => f64.into_pyobject(py)?.into_any(), Prop::DTime(dtime) => dtime.into_pyobject(py)?.into_any(), Prop::NDTime(ndtime) => ndtime.into_pyobject(py)?.into_any(), - Prop::Graph(g) => g.into_pyobject(py)?.into_any(), - Prop::PersistentGraph(g) => g.into_pyobject(py)?.into_any(), + Prop::Blob(blob) => blob.into_pyobject(py)?.into_any(), Prop::Document(d) => PyDocument::from(d).into_pyobject(py)?.into_any(), Prop::I32(v) => v.into_pyobject(py)?.into_any(), Prop::U32(v) => v.into_pyobject(py)?.into_any(), @@ -62,12 +61,6 @@ impl<'source> FromPyObject<'source> for Prop { if let Ok(s) = ob.extract::() { return Ok(Prop::Str(s.into())); } - if let Ok(g) = ob.extract() { - return Ok(Prop::Graph(g)); - } - if let Ok(g) = ob.extract::() { - return Ok(Prop::PersistentGraph(g)); - } if let Ok(d) = ob.extract::() { return Ok(Prop::Document(DocumentInput { content: d.content, @@ -96,8 +89,7 @@ impl Repr for Prop { Prop::F64(v) => v.repr(), Prop::DTime(v) => v.repr(), Prop::NDTime(v) => v.repr(), - Prop::Graph(g) => PyGraphView::from(g.clone()).repr(), - Prop::PersistentGraph(g) => PyGraphView::from(g.clone()).repr(), + Prop::Blob(v) => v.repr(), Prop::Document(d) => d.content.repr(), // We can't reuse the __repr__ defined for PyDocument because it needs to run python code Prop::I32(v) => v.repr(), Prop::U32(v) => v.repr(), diff --git a/raphtory/src/serialise/graph.proto b/raphtory/src/serialise/graph.proto index 4c247923d0..bea1504ab7 100644 --- a/raphtory/src/serialise/graph.proto +++ b/raphtory/src/serialise/graph.proto @@ -173,10 +173,11 @@ message PropType { List = 10; Map = 11; NDTime = 12; - Graph = 13; - PersistentGraph = 14; + Graph = 13 [deprecated=true]; + PersistentGraph = 14 [deprecated=true]; Document = 15; DTime = 16; + Blob = 17; } } @@ -194,11 +195,12 @@ message Prop { bool bool_value = 10; Props prop = 11; Dict map = 12; - Graph graph = 13; // Assuming Data can be represented as bytes. - Graph persistentGraph = 14; // Assuming Data can be represented as bytes. + Graph graph = 13 [deprecated=true]; // Assuming Data can be represented as bytes. + Graph persistentGraph = 14 [deprecated=true]; // Assuming Data can be represented as bytes. NDTime ndTime = 15; string dTime = 16; DocumentInput documentInput = 17; + Blob = 18; } message NDTime{ diff --git a/raphtory/src/serialise/proto_ext.rs b/raphtory/src/serialise/proto_ext.rs index 3ad3548f4c..c28ce819bc 100644 --- a/raphtory/src/serialise/proto_ext.rs +++ b/raphtory/src/serialise/proto_ext.rs @@ -44,33 +44,33 @@ fn as_proto_prop_type(p_type: &PropType) -> SPropType { PropType::List => SPropType::List, PropType::Map => SPropType::Map, PropType::NDTime => SPropType::NdTime, + PropType::Blob => SPropType::Blob, PropType::DTime => SPropType::DTime, - PropType::Graph => SPropType::Graph, - PropType::PersistentGraph => SPropType::PersistentGraph, PropType::Document => SPropType::Document, _ => unimplemented!("Empty prop types not supported!"), } } -pub fn as_prop_type(p_type: SPropType) -> PropType { +pub fn as_prop_type(p_type: SPropType) -> Option { + // for backwards compatibility we may skip some types match p_type { - SPropType::Str => PropType::Str, - SPropType::U8 => PropType::U8, - SPropType::U16 => PropType::U16, - SPropType::U32 => PropType::U32, - SPropType::I32 => PropType::I32, - SPropType::I64 => PropType::I64, - SPropType::U64 => PropType::U64, - SPropType::F32 => PropType::F32, - SPropType::F64 => PropType::F64, - SPropType::Bool => PropType::Bool, - SPropType::List => PropType::List, - SPropType::Map => PropType::Map, - SPropType::NdTime => PropType::NDTime, - SPropType::DTime => PropType::DTime, - SPropType::Graph => PropType::Graph, - SPropType::PersistentGraph => PropType::PersistentGraph, - SPropType::Document => PropType::Document, + SPropType::Str => Some(PropType::Str), + SPropType::U8 => Some(PropType::U8), + SPropType::U16 => Some(PropType::U16), + SPropType::U32 => Some(PropType::U32), + SPropType::I32 => Some(PropType::I32), + SPropType::I64 => Some(PropType::I64), + SPropType::U64 => Some(PropType::U64), + SPropType::F32 => Some(PropType::F32), + SPropType::F64 => Some(PropType::F64), + SPropType::Bool => Some(PropType::Bool), + SPropType::List => Some(PropType::List), + SPropType::Map => Some(PropType::Map), + SPropType::NdTime => Some(PropType::NDTime), + SPropType::DTime => Some(PropType::DTime), + SPropType::Document => Some(PropType::Document), + SPropType::Blob => Some(PropType::Blob), + _ => None, } } @@ -112,7 +112,10 @@ impl UpdateEdgeCProps { } pub fn props(&self) -> impl Iterator> + '_ { - self.properties.iter().map(as_prop) + self.properties + .iter() + .map(as_prop) + .filter_map(|r| r.transpose()) } } @@ -134,7 +137,10 @@ impl UpdateEdgeTProps { } pub fn props(&self) -> impl Iterator> + '_ { - self.properties.iter().map(as_prop) + self.properties + .iter() + .map(as_prop) + .filter_map(|r| r.transpose()) } } @@ -154,7 +160,10 @@ impl UpdateNodeCProps { } pub fn props(&self) -> impl Iterator> + '_ { - self.properties.iter().map(as_prop) + self.properties + .iter() + .map(as_prop) + .filter_map(|r| r.transpose()) } } @@ -168,7 +177,10 @@ impl UpdateNodeTProps { } pub fn props(&self) -> impl Iterator> + '_ { - self.properties.iter().map(as_prop) + self.properties + .iter() + .map(as_prop) + .filter_map(|r| r.transpose()) } } @@ -492,40 +504,44 @@ impl proto::Graph { } } -fn as_prop(prop_pair: &PropPair) -> Result<(usize, Prop), GraphError> { +fn as_prop(prop_pair: &PropPair) -> Result, GraphError> { let PropPair { key, value } = prop_pair; let value = value.as_ref().expect("Missing prop value"); let value = value.value.as_ref(); let value = as_prop_value(value)?; - Ok((*key as usize, value)) + Ok(value.map(|value| (*key as usize, value))) } -fn as_prop_value(value: Option<&prop::Value>) -> Result { +fn as_prop_value(value: Option<&prop::Value>) -> Result, GraphError> { let value = match value.expect("Missing prop value") { - prop::Value::BoolValue(b) => Prop::Bool(*b), - prop::Value::U8(u) => Prop::U8((*u).try_into().unwrap()), - prop::Value::U16(u) => Prop::U16((*u).try_into().unwrap()), - prop::Value::U32(u) => Prop::U32(*u), - prop::Value::I32(i) => Prop::I32(*i), - prop::Value::I64(i) => Prop::I64(*i), - prop::Value::U64(u) => Prop::U64(*u), - prop::Value::F32(f) => Prop::F32(*f), - prop::Value::F64(f) => Prop::F64(*f), - prop::Value::Str(s) => Prop::Str(ArcStr::from(s.as_str())), - prop::Value::Prop(props) => Prop::List(Arc::new( + prop::Value::BoolValue(b) => Some(Prop::Bool(*b)), + prop::Value::U8(u) => Some(Prop::U8((*u).try_into().unwrap())), + prop::Value::U16(u) => Some(Prop::U16((*u).try_into().unwrap())), + prop::Value::U32(u) => Some(Prop::U32(*u)), + prop::Value::I32(i) => Some(Prop::I32(*i)), + prop::Value::I64(i) => Some(Prop::I64(*i)), + prop::Value::U64(u) => Some(Prop::U64(*u)), + prop::Value::F32(f) => Some(Prop::F32(*f)), + prop::Value::F64(f) => Some(Prop::F64(*f)), + prop::Value::Str(s) => Some(Prop::Str(ArcStr::from(s.as_str()))), + prop::Value::Prop(props) => Some(Prop::List(Arc::new( props .properties .iter() - .map(|prop| as_prop_value(prop.value.as_ref())) + .filter_map(|prop| as_prop_value(prop.value.as_ref()).transpose()) .collect::, _>>()?, - )), - prop::Value::Map(dict) => Prop::Map(Arc::new( + ))), + prop::Value::Map(dict) => Some(Prop::Map(Arc::new( dict.map .iter() - .map(|(k, v)| Ok((ArcStr::from(k.as_str()), as_prop_value(v.value.as_ref())?))) + .filter_map(|(k, v)| { + as_prop_value(v.value.as_ref()) + .map(|v| v.map(|v| (ArcStr::from(k.as_str()), v))) + .transpose() + }) .collect::>()?, - )), + ))), prop::Value::NdTime(ndt) => { let prop::NdTime { year, @@ -546,14 +562,12 @@ fn as_prop_value(value: Option<&prop::Value>) -> Result { ) .unwrap(), ); - Prop::NDTime(ndt) - } - prop::Value::DTime(dt) => Prop::DTime(DateTime::parse_from_rfc3339(dt).unwrap().into()), - prop::Value::Graph(graph_proto) => Prop::Graph(Graph::decode_from_proto(graph_proto)?), - prop::Value::PersistentGraph(graph_proto) => { - Prop::PersistentGraph(PersistentGraph::decode_from_proto(graph_proto)?) + Some(Prop::NDTime(ndt)) } - prop::Value::DocumentInput(doc) => Prop::Document(DocumentInput { + prop::Value::DTime(dt) => Some(Prop::DTime( + DateTime::parse_from_rfc3339(dt).unwrap().into(), + )), + prop::Value::DocumentInput(doc) => Some(Prop::Document(DocumentInput { content: doc.content.clone(), life: doc .life @@ -569,7 +583,9 @@ fn as_prop_value(value: Option<&prop::Value>) -> Result { None => Lifespan::Inherited, }) .unwrap_or(Lifespan::Inherited), - }), + })), + prop::Value::Blob(blob) => Some(Prop::Blob(blob)), + _ => None, }; Ok(value) } @@ -585,7 +601,10 @@ fn collect_proto_props( pub fn collect_props<'a>( iter: impl IntoIterator, ) -> Result, GraphError> { - iter.into_iter().map(as_prop).collect() + iter.into_iter() + .map(as_prop) + .filter_map(|r| r.transpose()) + .collect() } fn as_proto_prop(prop: &Prop) -> proto::Prop { @@ -634,8 +653,7 @@ fn as_proto_prop(prop: &Prop) -> proto::Prop { Prop::DTime(dt) => { prop::Value::DTime(dt.to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true)) } - Prop::Graph(g) => prop::Value::Graph(g.encode_to_proto()), - Prop::PersistentGraph(g) => prop::Value::PersistentGraph(g.encode_to_proto()), + Prop::Blob(blob) => prop::Value::Blob(blob.clone()), Prop::Document(doc) => { let life = match doc.life { Lifespan::Interval { start, end } => { diff --git a/raphtory/src/serialise/serialise.rs b/raphtory/src/serialise/serialise.rs index 5111d06f33..abd2e86f3a 100644 --- a/raphtory/src/serialise/serialise.rs +++ b/raphtory/src/serialise/serialise.rs @@ -1083,13 +1083,5 @@ mod proto_test { }, }), )); - let graph = Graph::new(); - graph.add_edge(1, "a", "b", NO_PROPS, None).unwrap(); - props.push(("graph", Prop::Graph(graph))); - - let graph = Graph::new().persistent_graph(); - graph.add_edge(1, "a", "b", NO_PROPS, None).unwrap(); - graph.delete_edge(2, "a", "b", None).unwrap(); - props.push(("p_graph", Prop::PersistentGraph(graph))); } } diff --git a/raphtory/src/vectors/template.rs b/raphtory/src/vectors/template.rs index 69d4916183..d1b5a799b8 100644 --- a/raphtory/src/vectors/template.rs +++ b/raphtory/src/vectors/template.rs @@ -146,14 +146,13 @@ impl From for Value { Prop::Str(value) => Value::from(value.0.to_owned()), Prop::DTime(value) => Value::from(value.timestamp_millis()), Prop::NDTime(value) => Value::from(value.and_utc().timestamp_millis()), + Prop::Blob(value) => Value::from(value), Prop::List(value) => value.iter().cloned().collect(), Prop::Map(value) => value .iter() .map(|(key, value)| (key.to_string(), value.clone())) .collect(), Prop::Document(value) => Value::from(value.content), - Prop::Graph(value) => Value::from(value.to_string()), - Prop::PersistentGraph(value) => Value::from(value.to_string()), } } }