Merge pull request #923 from veryl-lang/std_crate

Extract veryl-std from veryl-metadata

Cargo.toml

@@ -7,7 +7,9 @@ members = [
     "crates/mdbook",
     "crates/metadata",
     "crates/parser",
+    "crates/path",
     "crates/sourcemap",
+    "crates/std",
     "crates/tests",
     "crates/veryl",
 ]
@@ -45,6 +47,7 @@ thiserror       = "1.0"
 tokio           = {version = "1.40.0", features = ["full"]}
 toml            = "0.8.19"
 url             = {version = "2.5", features = ["serde"]}
+walkdir         = "2.5.0"
 
 [workspace.metadata.release]
 tag = false

crates/languageserver/Cargo.toml

@@ -28,3 +28,4 @@ veryl-analyzer    = {version = "0.12.0", path = "../analyzer"}
 veryl-formatter   = {version = "0.12.0", path = "../formatter"}
 veryl-metadata    = {version = "0.12.0", path = "../metadata"}
 veryl-parser      = {version = "0.12.0", path = "../parser"}
+veryl-path        = {version = "0.12.0", path = "../path"}

crates/languageserver/src/server.rs

@@ -12,10 +12,11 @@ use veryl_analyzer::symbol::SymbolKind as VerylSymbolKind;
 use veryl_analyzer::symbol_path::SymbolPath;
 use veryl_analyzer::{namespace_table, symbol_table, Analyzer, AnalyzerError};
 use veryl_formatter::Formatter;
-use veryl_metadata::{Metadata, PathPair};
+use veryl_metadata::Metadata;
 use veryl_parser::veryl_token::Token;
 use veryl_parser::veryl_walker::VerylWalker;
 use veryl_parser::{resource_table, Finder, Parser, ParserError};
+use veryl_path::PathPair;
 
 pub enum MsgToServer {
     DidOpen {
@@ -119,7 +120,7 @@ impl Server {
             document_map: DashMap::new(),
             parser_map: DashMap::new(),
             metadata_map: DashMap::new(),
-            cache_dir: Metadata::cache_path(),
+            cache_dir: veryl_path::cache_path(),
             lsp_token: 0,
             background_tasks: VecDeque::new(),
             background_done: true,

crates/metadata/Cargo.toml

@@ -13,12 +13,10 @@ edition               = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-directories    = "5.0"
 git-repository = {version = "0.35.0", optional = true, features = ["blocking-network-client", "blocking-http-transport-reqwest", "blocking-http-transport-reqwest-rust-tls"]}
 log            = {workspace = true}
 once_cell      = {workspace = true}
 regex          = {workspace = true}
-rust-embed     = {version = "8.5", features = ["debug-embed", "compression", "include-exclude"]}
 semver         = {workspace = true}
 serde          = {workspace = true}
 serde_regex    = "1.1"
@@ -28,21 +26,18 @@ toml           = {workspace = true}
 url            = {workspace = true}
 uuid           = {version = "1.10", default-features = false, features = ["v5", "serde"]}
 veryl-parser   = {version = "0.12.0", path = "../parser"}
-walkdir        = "2.5.0"
+veryl-path     = {version = "0.12.0", path = "../path"}
+veryl-std      = {version = "0.12.0", path = "../std"}
 
 [target.'cfg(target_family = "wasm")'.dependencies]
 miette         = {workspace = true, features = ["fancy-no-syscall"]}
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 miette         = {workspace = true, features = ["fancy"]}
-fs4            = {version = "0.9.1", features = ["sync"]}
 
 [dev-dependencies]
 tempfile       = {workspace = true}
 
-[build-dependencies]
-merkle_hash    = "3.7"
-
 [features]
 default      = ["git-command"]
 git-command  = []

crates/metadata/src/lib.rs

@@ -9,17 +9,15 @@ mod metadata_error;
 mod project;
 mod pubfile;
 mod publish;
-mod stdlib;
 mod test;
 #[cfg(test)]
 mod tests;
-mod utils;
 pub use build::{Build, BuiltinType, ClockType, FilelistType, ResetType, SourceMapTarget, Target};
 pub use doc::Doc;
 pub use format::Format;
 pub use lint::{Case, Lint};
 pub use lockfile::Lockfile;
-pub use metadata::{BumpKind, Metadata, PathPair};
+pub use metadata::{BumpKind, Metadata};
 pub use metadata_error::MetadataError;
 pub use project::Project;
 pub use pubfile::{Pubfile, Release};

crates/metadata/src/lockfile.rs

@@ -2,7 +2,6 @@ use crate::git::Git;
 use crate::metadata::{Dependency, Metadata};
 use crate::metadata_error::MetadataError;
 use crate::pubfile::{Pubfile, Release};
-use crate::{utils, PathPair};
 use log::info;
 use semver::{Version, VersionReq};
 use serde::{Deserialize, Serialize};
@@ -12,6 +11,7 @@ use std::path::Path;
 use std::str::FromStr;
 use url::Url;
 use uuid::Uuid;
+use veryl_path::PathPair;
 
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -160,7 +160,7 @@ impl Lockfile {
                 let metadata = self.get_metadata(&lock.url, &lock.revision)?;
                 let path = metadata.project_path();
 
-                for src in &utils::gather_files_with_extension(&path, "veryl", false)? {
+                for src in &veryl_path::gather_files_with_extension(&path, "veryl", false)? {
                     let rel = src.strip_prefix(&path)?;
                     let mut dst = base_dst.join(&lock.name);
                     dst.push(rel);
@@ -334,7 +334,7 @@ impl Lockfile {
         url: &Url,
         version_req: &VersionReq,
     ) -> Result<Release, MetadataError> {
-        let resolve_dir = Metadata::cache_path().join("resolve");
+        let resolve_dir = veryl_path::cache_path().join("resolve");
 
         if !resolve_dir.exists() {
             fs::create_dir_all(&resolve_dir)?;
@@ -343,11 +343,11 @@ impl Lockfile {
         let uuid = Self::gen_uuid(url, "")?;
 
         let path = resolve_dir.join(uuid.simple().encode_lower(&mut Uuid::encode_buffer()));
-        let lock = utils::lock_dir("resolve")?;
+        let lock = veryl_path::lock_dir("resolve")?;
         let git = Git::clone(url, &path)?;
         git.fetch()?;
         git.checkout(None)?;
-        utils::unlock_dir(lock)?;
+        veryl_path::unlock_dir(lock)?;
 
         let toml = path.join("Veryl.pub");
         let mut pubfile = Pubfile::load(toml)?;
@@ -367,7 +367,7 @@ impl Lockfile {
     }
 
     fn get_metadata(&self, url: &Url, revision: &str) -> Result<Metadata, MetadataError> {
-        let dependencies_dir = Metadata::cache_path().join("dependencies");
+        let dependencies_dir = veryl_path::cache_path().join("dependencies");
 
         if !dependencies_dir.exists() {
             fs::create_dir_all(&dependencies_dir)?;
@@ -379,23 +379,23 @@ impl Lockfile {
         let toml = path.join("Veryl.toml");
 
         if !path.exists() {
-            let lock = utils::lock_dir("dependencies")?;
+            let lock = veryl_path::lock_dir("dependencies")?;
             let git = Git::clone(url, &path)?;
             git.fetch()?;
             git.checkout(Some(revision))?;
-            utils::unlock_dir(lock)?;
+            veryl_path::unlock_dir(lock)?;
         } else {
             let git = Git::open(&path)?;
             let ret = git.is_clean().map_or(false, |x| x);
 
             // If the existing path is not git repository, cleanup and re-try
             if !ret || !toml.exists() {
-                let lock = utils::lock_dir("dependencies")?;
+                let lock = veryl_path::lock_dir("dependencies")?;
                 fs::remove_dir_all(&path)?;
                 let git = Git::clone(url, &path)?;
                 git.fetch()?;
                 git.checkout(Some(revision))?;
-                utils::unlock_dir(lock)?;
+                veryl_path::unlock_dir(lock)?;
             }
         }
 

crates/metadata/src/metadata.rs

@@ -7,10 +7,8 @@ use crate::lockfile::Lockfile;
 use crate::project::Project;
 use crate::pubfile::{Pubfile, Release};
 use crate::publish::Publish;
-use crate::stdlib;
 use crate::test::Test;
-use crate::{utils, FilelistType, MetadataError};
-use directories::ProjectDirs;
+use crate::{FilelistType, MetadataError};
 use log::{debug, info};
 use once_cell::sync::Lazy;
 use regex::Regex;
@@ -23,13 +21,7 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use url::Url;
-
-#[derive(Clone, Debug)]
-pub struct PathPair {
-    pub prj: String,
-    pub src: PathBuf,
-    pub dst: PathBuf,
-}
+use veryl_path::PathPair;
 
 #[derive(Clone, Copy, Debug)]
 pub enum BumpKind {
@@ -231,7 +223,7 @@ impl Metadata {
         let base = self.project_path();
 
         let src_files = if files.is_empty() {
-            utils::gather_files_with_extension(&base, "veryl", symlink)?
+            veryl_path::gather_files_with_extension(&base, "veryl", symlink)?
         } else {
             let mut ret = Vec::new();
             for file in files {
@@ -264,8 +256,8 @@ impl Metadata {
         }
 
         if !self.build.exclude_std {
-            stdlib::expand()?;
-            ret.append(&mut stdlib::paths(&base_dst)?);
+            veryl_std::expand()?;
+            ret.append(&mut veryl_std::paths(&base_dst)?);
         }
 
         self.update_lockfile()?;
@@ -288,11 +280,6 @@ version = "0.1.0""###
         ))
     }
 
-    pub fn cache_path() -> PathBuf {
-        let project_dir = ProjectDirs::from("org", "veryl-lang", "veryl").unwrap();
-        project_dir.cache_dir().to_path_buf()
-    }
-
     pub fn project_path(&self) -> PathBuf {
         self.metadata_path.parent().unwrap().to_path_buf()
     }

crates/metadata/src/metadata_error.rs

@@ -3,6 +3,7 @@ use semver::Version;
 use std::path::PathBuf;
 use thiserror::Error;
 use url::Url;
+use veryl_path::PathError;
 
 #[derive(Error, Diagnostic, Debug)]
 pub enum MetadataError {
@@ -18,10 +19,6 @@ pub enum MetadataError {
     #[error("toml load failed")]
     Deserialize(#[from] toml::de::Error),
 
-    #[diagnostic(code(MetadataError::Walkdir), help(""))]
-    #[error("walkdir error")]
-    Walkdir(#[from] walkdir::Error),
-
     #[diagnostic(code(MetadataError::StripPrefix), help(""))]
     #[error("strip prefix error")]
     StripPrefix(#[from] std::path::StripPrefixError),
@@ -67,4 +64,8 @@ pub enum MetadataError {
     #[diagnostic(code(MetadataError::NameConflict), help(""))]
     #[error("project name \"{0}\" is used multiply in dependencies")]
     NameConflict(String),
+
+    #[diagnostic(code(MetadataError::Path), help(""))]
+    #[error("path error")]
+    Path(#[from] PathError),
 }

crates/parser/Cargo.toml

@@ -31,4 +31,3 @@ miette        = {workspace = true, features = ["fancy"]}
 [build-dependencies]
 parol         =  "0.31.0"
 parol_runtime = {version = "0.24.1", features = ["auto_generation"]}
-walkdir       = "2.5.0"

crates/path/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name                  = "veryl-path"
+version               = "0.12.0"
+authors.workspace     = true
+repository.workspace  = true
+keywords.workspace    = true
+categories.workspace  = true
+license.workspace     = true
+readme.workspace      = true
+description.workspace = true
+edition               = "2021"
+
+[dependencies]
+directories = "5.0"
+log         = {workspace = true}
+thiserror   = {workspace = true}
+walkdir     = {workspace = true}
+
+[target.'cfg(target_family = "wasm")'.dependencies]
+miette      = {workspace = true, features = ["fancy-no-syscall"]}
+
+[target.'cfg(not(target_family = "wasm"))'.dependencies]
+miette      = {workspace = true, features = ["fancy"]}
+fs4         = {version = "0.9.1", features = ["sync"]}