Add tokenizer module

efmt_core/src/lib.rs

@@ -7,7 +7,7 @@ pub mod span;
 pub fn format_text<T: crate::parse::Parse + crate::format::Format>(
     text: &str,
 ) -> crate::parse::Result<String> {
-    let tokenizer = erl_tokenize::Tokenizer::new(text.to_owned());
+    let tokenizer = crate::parse::Tokenizer::new(text.to_owned());
     let mut ts = crate::parse::TokenStream::new(tokenizer);
     let item: T = ts.parse()?;
     let mut formatter = crate::format::Formatter::new(ts);

efmt_core/src/parse.rs

@@ -4,11 +4,13 @@ use std::path::PathBuf;
 use std::sync::Arc;
 
 pub use self::token_stream::TokenStream;
+pub use self::tokenizer::Tokenizer;
 
 /// A procedural macro to derive [Parse].
 pub use efmt_derive::Parse;
 
 pub(crate) mod token_stream;
+pub(crate) mod tokenizer;
 
 /// Possible errors.
 #[derive(Debug, Clone, thiserror::Error)]

efmt_core/src/parse/token_stream.rs

@@ -6,17 +6,18 @@ use crate::items::tokens::{
     AtomToken, CharToken, CommentToken, FloatToken, IntegerToken, KeywordToken, LexicalToken,
     SigilStringToken, StringToken, SymbolToken, VariableToken,
 };
+use crate::parse::Tokenizer;
 use crate::parse::{Error, Parse, Result, ResumeParse};
 use crate::span::{Position, Span};
 use erl_tokenize::values::Symbol;
-use erl_tokenize::{PositionRange as _, Tokenizer};
+use erl_tokenize::PositionRange as _;
 use std::collections::{BTreeMap, HashSet};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 #[derive(Debug)]
 pub struct TokenStream {
-    tokenizer: Tokenizer<String>,
+    tokenizer: Tokenizer,
     tokens: Vec<LexicalToken>,
     current_token_index: usize,
     comments: BTreeMap<Position, CommentToken>,
@@ -33,7 +34,7 @@ pub struct TokenStream {
 }
 
 impl TokenStream {
-    pub fn new(tokenizer: Tokenizer<String>) -> Self {
+    pub fn new(tokenizer: Tokenizer) -> Self {
         let text = Arc::new(tokenizer.text().to_owned());
         let path = tokenizer
             .next_position()

efmt_core/src/parse/tokenizer.rs

@@ -0,0 +1,35 @@
+use erl_tokenize::{Position, Token};
+use std::path::Path;
+
+#[derive(Debug)]
+pub struct Tokenizer {
+    inner: erl_tokenize::Tokenizer<String>,
+}
+
+impl Tokenizer {
+    pub fn new(text: String) -> Self {
+        Tokenizer {
+            inner: erl_tokenize::Tokenizer::new(text),
+        }
+    }
+
+    pub fn text(&self) -> &str {
+        self.inner.text()
+    }
+
+    pub fn next_position(&self) -> Position {
+        self.inner.next_position()
+    }
+
+    pub fn set_filepath<P: AsRef<Path>>(&mut self, path: P) {
+        self.inner.set_filepath(path)
+    }
+}
+
+impl Iterator for Tokenizer {
+    type Item = erl_tokenize::Result<Token>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner.next()
+    }
+}

src/files.rs

@@ -1,5 +1,5 @@
 use efmt_core::items::{Config, Expr};
-use efmt_core::parse::TokenStream;
+use efmt_core::parse::{TokenStream, Tokenizer};
 use ignore::Walk;
 use std::path::{Path, PathBuf};
 
@@ -13,7 +13,7 @@ pub fn find_rebar_config_dir() -> Option<PathBuf> {
 
 pub fn load_rebar_config<P: AsRef<Path>>(path: P) -> anyhow::Result<Vec<RebarConfigValue>> {
     let text = std::fs::read_to_string(&path)?;
-    let mut tokenizer = erl_tokenize::Tokenizer::new(text);
+    let mut tokenizer = Tokenizer::new(text);
     tokenizer.set_filepath(path);
     let mut ts = TokenStream::new(tokenizer);
     let config: Config = ts.parse()?;

src/lib.rs

@@ -1,5 +1,5 @@
 use efmt_core::format::{Format, Formatter};
-use efmt_core::parse::{Parse, TokenStream};
+use efmt_core::parse::{Parse, TokenStream, Tokenizer};
 use std::path::Path;
 
 pub mod diff;
@@ -34,20 +34,17 @@ impl Options {
 
     pub fn format_file<T: Parse + Format, P: AsRef<Path>>(self, path: P) -> anyhow::Result<String> {
         let text = std::fs::read_to_string(&path)?;
-        let mut tokenizer = erl_tokenize::Tokenizer::new(text);
+        let mut tokenizer = Tokenizer::new(text);
         tokenizer.set_filepath(path);
         self.format::<T>(tokenizer)
     }
 
     pub fn format_text<T: Parse + Format>(self, text: &str) -> anyhow::Result<String> {
-        let tokenizer = erl_tokenize::Tokenizer::new(text.to_owned());
+        let tokenizer = Tokenizer::new(text.to_owned());
         self.format::<T>(tokenizer)
     }
 
-    fn format<T: Parse + Format>(
-        self,
-        tokenizer: erl_tokenize::Tokenizer<String>,
-    ) -> anyhow::Result<String> {
+    fn format<T: Parse + Format>(self, tokenizer: Tokenizer) -> anyhow::Result<String> {
         let mut ts = TokenStream::new(tokenizer);
         let item: T = ts.parse()?;
         let mut formatter = Formatter::new(ts);