Support OTP-27.0-rc1 (triple-quoted strings and sigil string literals)

Cargo.toml

@@ -13,7 +13,7 @@ exclude = ["/rebar3_efmt/"]
 [dependencies]
 anyhow = "1"
 clap = { version = "4", features = ["derive"] }
-erl_tokenize = "0.5"
+erl_tokenize = "0.6"
 efmt_core = { path = "efmt_core", version = "0.1.0" }
 env_logger = "0.11"
 log = "0.4"

efmt_core/Cargo.toml

@@ -10,7 +10,7 @@ repository = "https://github.com/sile/efmt"
 readme = "README.md"
 
 [dependencies]
-erl_tokenize = "0.5"
+erl_tokenize = "0.6"
 efmt_derive = { path = "../efmt_derive", version = "0.1.0" }
 log = "0.4"
 thiserror = "1"

efmt_core/src/items/expressions.rs

@@ -3,7 +3,8 @@ use crate::format::Format;
 use crate::items::components::{Either, Element, Parenthesized};
 use crate::items::symbols::OpenBraceSymbol;
 use crate::items::tokens::{
-    AtomToken, CharToken, FloatToken, IntegerToken, LexicalToken, SymbolToken, VariableToken,
+    AtomToken, CharToken, FloatToken, IntegerToken, LexicalToken, SigilStringToken, SymbolToken,
+    VariableToken,
 };
 use crate::items::Expr;
 use crate::parse::{self, Parse};
@@ -190,13 +191,14 @@ impl Element for FullExpr {
     }
 }
 
-/// [AtomToken] | [CharToken] | [FloatToken] | [IntegerToken] | [VariableToken] | [StringExpr]
+/// [AtomToken] | [CharToken] | [FloatToken] | [IntegerToken] | [VariableToken] | [StringExpr] | [SigilStringToken]
 #[derive(Debug, Clone, Span, Parse, Format, Element)]
 pub enum LiteralExpr {
     Atom(AtomToken),
     Char(CharToken),
     Float(FloatToken),
     Integer(IntegerToken),
+    SigilString(SigilStringToken),
     String(StringExpr),
     Variable(VariableToken),
 }

efmt_core/src/items/tokens.rs

@@ -13,6 +13,7 @@ pub enum LexicalToken {
     Float(FloatToken),
     Integer(IntegerToken),
     Keyword(KeywordToken),
+    SigilString(SigilStringToken),
     String(StringToken),
     Symbol(SymbolToken),
     Variable(VariableToken),
@@ -26,6 +27,7 @@ impl LexicalToken {
             Self::Float(x) => (&mut x.start, &mut x.end),
             Self::Integer(x) => (&mut x.start, &mut x.end),
             Self::Keyword(x) => (&mut x.start, &mut x.end),
+            Self::SigilString(x) => (&mut x.start, &mut x.end),
             Self::String(x) => (&mut x.start, &mut x.end),
             Self::Symbol(x) => (&mut x.start, &mut x.end),
             Self::Variable(x) => (&mut x.start, &mut x.end),
@@ -188,6 +190,26 @@ impl KeywordToken {
 
 impl_traits!(KeywordToken, Keyword);
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct SigilStringToken {
+    start: Position,
+    end: Position,
+}
+
+impl SigilStringToken {
+    pub fn new(start: Position, end: Position) -> Self {
+        Self { start, end }
+    }
+}
+
+impl_traits!(SigilStringToken, SigilString);
+
+impl Element for SigilStringToken {
+    fn is_packable(&self) -> bool {
+        true
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct StringToken {
     value: String,

efmt_core/src/parse/token_stream.rs

@@ -4,7 +4,7 @@ use crate::items::macros::{Macro, MacroName};
 use crate::items::symbols::{OpenParenSymbol, QuestionSymbol};
 use crate::items::tokens::{
     AtomToken, CharToken, CommentToken, FloatToken, IntegerToken, KeywordToken, LexicalToken,
-    StringToken, SymbolToken, VariableToken,
+    SigilStringToken, StringToken, SymbolToken, VariableToken,
 };
 use crate::parse::{Error, Parse, Result, ResumeParse};
 use crate::span::{Position, Span};
@@ -247,6 +247,9 @@ impl TokenStream {
                 erl_tokenize::Token::Keyword(x) => {
                     KeywordToken::new(x.value(), start_position, end_position).into()
                 }
+                erl_tokenize::Token::SigilString(_) => {
+                    SigilStringToken::new(start_position, end_position).into()
+                }
                 erl_tokenize::Token::String(x) => {
                     StringToken::new(x.value(), start_position, end_position).into()
                 }

tests/testdata/otp27_strings.erl

@@ -0,0 +1,29 @@
+-module(otp27_strings).
+
+-doc """
+    First line
+    Second line with "\*not emphasized\* Markdown"
+    Third line
+    """.
+
+
+triple_quoted_strings() ->
+    X = <<"""
+        Line 1
+        Line 2
+        """/utf8>>,
+    X = """"
+        ++ foo() ++
+        """",
+    ok.
+
+
+sigil_strings() ->
+    ~"abc\d",
+    ~'abc"d',
+    ~b[abc"d],
+    ~R/^from: /i,
+    ~""""
+     ++ foo() ++
+     """",
+    ok.