From bbb5800b4cf56d2996691edc63edc2783e19f427 Mon Sep 17 00:00:00 2001 From: Azat Alimov <32402726+mkslanc@users.noreply.github.com> Date: Fri, 5 Aug 2022 17:19:28 +0400 Subject: [PATCH] fix: Rust identifiers normally recognised now; generics highlight support; doc comments support (#4868) --- demo/kitchen-sink/docs/rust.rs | 38 +++++ src/mode/_test/tokens_rust.json | 138 ++++++++++------ src/mode/rust_highlight_rules.js | 274 +++++++++++++++++++++---------- 3 files changed, 312 insertions(+), 138 deletions(-) diff --git a/demo/kitchen-sink/docs/rust.rs b/demo/kitchen-sink/docs/rust.rs index 7ab5418cdfb..24e3f7eff76 100644 --- a/demo/kitchen-sink/docs/rust.rs +++ b/demo/kitchen-sink/docs/rust.rs @@ -18,3 +18,41 @@ fn map(vector: &[T], function: &fn(v: &T) -> U) -> ~[U] { } return accumulator; } + +struct ConstGenericStruct([(); N]); +// T constrains by being an argument to GenericTrait. +impl GenericTrait for i32 { /* ... */ } + +// T constrains by being an arguement to GenericStruct +impl Trait for GenericStruct { /* ... */ } + +// Likewise, N constrains by being an argument to ConstGenericStruct +impl Trait for ConstGenericStruct { /* ... */ } + +// T constrains by being in an associated type in a bound for type `U` which is +// itself a generic parameter constraining the trait. +impl GenericTrait for u32 where U: HasAssocType { /* ... */ } + +// Like previous, except the type is `(U, isize)`. `U` appears inside the type +// that includes `T`, and is not the type itself. +impl GenericStruct where (U, isize): HasAssocType { /* ... */ } + +//! - Inner line doc +//!! - Still an inner line doc (but with a bang at the beginning) + +/*! - Inner block doc */ +/*!! - Still an inner block doc (but with a bang at the beginning) */ + +/** - Outer block doc (exactly) 2 asterisks */ + +macro_rules! mac_variant { + ($vis:vis $name:ident) => { + enum $name { + $vis Unit, + + $vis Tuple(u8, u16), + + $vis Struct { f: u8 }, + } + } +} \ No newline at end of file diff --git a/src/mode/_test/tokens_rust.json b/src/mode/_test/tokens_rust.json index 68add464f9d..04bbad49ab6 100644 --- a/src/mode/_test/tokens_rust.json +++ b/src/mode/_test/tokens_rust.json @@ -3,7 +3,7 @@ ["keyword.source.rust","use"], ["text"," "], ["support.constant","core::rand::"], - ["text","RngUtil"], + ["identifier","RngUtil"], ["punctuation.operator",";"] ],[ "start" @@ -31,9 +31,10 @@ ["string.quoted.double.source.rust","\"Carol\""], ["paren.rparen","]"], ["punctuation.operator","."], - ["text","each "], + ["identifier","each"], + ["text"," "], ["keyword.operator","|&"], - ["text","name"], + ["identifier","name"], ["keyword.operator","|"], ["text"," "], ["paren.lparen","{"] @@ -41,21 +42,25 @@ "start", ["text"," "], ["keyword.source.rust","do"], - ["text"," spawn "], + ["text"," "], + ["identifier","spawn"], + ["text"," "], ["paren.lparen","{"] ],[ "start", ["text"," "], ["keyword.source.rust","let"], - ["text"," v "], + ["text"," "], + ["identifier","v"], + ["text"," "], ["keyword.operator","="], ["text"," "], ["support.constant","rand::"], - ["text","Rng"], + ["identifier","Rng"], ["paren.lparen","("], ["paren.rparen",")"], ["punctuation.operator","."], - ["text","shuffle"], + ["identifier","shuffle"], ["paren.lparen","(["], ["constant.numeric.source.rust","1"], ["punctuation.operator",","], @@ -70,28 +75,34 @@ "start", ["text"," "], ["keyword.source.rust","for"], - ["text"," v"], + ["text"," "], + ["identifier","v"], ["punctuation.operator","."], - ["text","each "], + ["identifier","each"], + ["text"," "], ["keyword.operator","|&"], - ["text","num"], + ["identifier","num"], ["keyword.operator","|"], ["text"," "], ["paren.lparen","{"] ],[ "start", - ["text"," print"], + ["text"," "], + ["identifier","print"], ["paren.lparen","("], - ["text","fmt"], + ["identifier","fmt"], ["keyword.operator","!"], ["paren.lparen","("], ["string.quoted.double.source.rust","\"%s says: '%d'"], - ["constant.character.escape.source.rust","\\n\\\\"], + ["constant.character.escape.source.rust","\\n"], ["string.quoted.double.source.rust","\""], ["punctuation.operator",","], - ["text"," name"], + ["text"," "], + ["identifier","name"], ["punctuation.operator",","], - ["text"," num "], + ["text"," "], + ["identifier","num"], + ["text"," "], ["keyword.operator","+"], ["text"," "], ["constant.numeric.source.rust","1"], @@ -112,20 +123,31 @@ "start" ],[ "start", + ["text"," "], ["keyword.source.rust","let"], - ["text"," _ "], + ["text"," "], + ["identifier","_"], ["punctuation.operator",":"], + ["text"," "], ["storage.type.source.rust","i128"], - ["keyword.operator","=-"], + ["text"," "], + ["keyword.operator","="], + ["text"," "], + ["keyword.operator","-"], ["constant.numeric.source.rust","42i128"], ["punctuation.operator",";"] ],[ "start", + ["text"," "], ["keyword.source.rust","let"], - ["text"," _ "], + ["text"," "], + ["identifier","_"], ["punctuation.operator",":"], + ["text"," "], ["storage.type.source.rust","u128"], + ["text"," "], ["keyword.operator","="], + ["text"," "], ["constant.numeric.source.rust","42u128"], ["punctuation.operator",";"] ],[ @@ -152,41 +174,44 @@ ["keyword.source.rust","fn"], ["text"," "], ["entity.name.function.source.rust","map"], - ["keyword.operator","<"], - ["text","T"], + ["punctuation","<"], + ["identifier","T"], ["punctuation.operator",","], - ["text"," U"], - ["keyword.operator",">"], + ["text"," "], + ["identifier","U"], + ["punctuation",">"], ["paren.lparen","("], - ["text","vector"], + ["identifier","vector"], ["punctuation.operator",":"], ["text"," "], ["keyword.operator","&"], ["paren.lparen","["], - ["text","T"], + ["identifier","T"], ["paren.rparen","]"], ["punctuation.operator",","], - ["text"," function"], + ["text"," "], + ["identifier","function"], ["punctuation.operator",":"], ["text"," "], ["keyword.operator","&"], - ["text","fn"], + ["identifier","fn"], ["paren.lparen","("], - ["text","v"], + ["identifier","v"], ["punctuation.operator",":"], ["text"," "], ["keyword.operator","&"], - ["text","T"], + ["identifier","T"], ["paren.rparen",")"], ["text"," "], ["keyword.operator","->"], - ["text"," U"], + ["text"," "], + ["identifier","U"], ["paren.rparen",")"], ["text"," "], ["keyword.operator","->"], ["text"," ~"], ["paren.lparen","["], - ["text","U"], + ["identifier","U"], ["paren.rparen","]"], ["text"," "], ["paren.lparen","{"] @@ -196,7 +221,9 @@ ["keyword.source.rust","let"], ["text"," "], ["keyword.source.rust","mut"], - ["text"," accumulator "], + ["text"," "], + ["identifier","accumulator"], + ["text"," "], ["keyword.operator","="], ["text"," ~"], ["paren.lparen","["], @@ -208,25 +235,26 @@ ["keyword.source.rust","for"], ["text"," "], ["support.constant","vec::"], - ["text","each"], + ["identifier","each"], ["paren.lparen","("], - ["text","vector"], + ["identifier","vector"], ["paren.rparen",")"], ["text"," "], ["keyword.operator","|"], - ["text","element"], + ["identifier","element"], ["keyword.operator","|"], ["text"," "], ["paren.lparen","{"] ],[ "start", - ["text"," accumulator"], + ["text"," "], + ["identifier","accumulator"], ["punctuation.operator","."], - ["text","push"], + ["identifier","push"], ["paren.lparen","("], - ["text","function"], + ["identifier","function"], ["paren.lparen","("], - ["text","element"], + ["identifier","element"], ["paren.rparen","))"], ["punctuation.operator",";"] ],[ @@ -237,7 +265,8 @@ "start", ["text"," "], ["keyword.source.rust","return"], - ["text"," accumulator"], + ["text"," "], + ["identifier","accumulator"], ["punctuation.operator",";"] ],[ "start", @@ -276,24 +305,37 @@ "start", ["constant.numeric.source.rust","14"], ["punctuation.operator","."], - ["text","_E"], + ["identifier","_E"], ["keyword.operator","-"], ["constant.numeric.source.rust","111_f64"], ["punctuation.operator",";"], - ["text","0xi32"], + ["text"," 0"], + ["identifier","xi32"], ["punctuation.operator",";"], - ["text","0b777u"] + ["text"," 0"], + ["identifier","b777u"] +],[ + "start" +],[ + "start", + ["comment.line.double-dash.source.rust","// identifiers ending in constant.numeric"] ],[ "start", - ["text","foo1"], + ["identifier","foo1"], ["punctuation.operator",";"], - ["text","foo1u32"], + ["text"," "], + ["identifier","foo1u32"], ["punctuation.operator",";"], - ["text","foo1f32"], + ["text"," "], + ["identifier","foo1f32"], ["punctuation.operator",";"], - ["text","foo0xF"], + ["text"," "], + ["identifier","foo0xF"], ["punctuation.operator",";"], - ["text","foo1"], + ["text"," "], + ["identifier","foo1"], ["punctuation.operator","."], ["constant.numeric.source.rust","0"] -]] +],[ + "start" +]] \ No newline at end of file diff --git a/src/mode/rust_highlight_rules.js b/src/mode/rust_highlight_rules.js index 324945ac268..f06806720f9 100644 --- a/src/mode/rust_highlight_rules.js +++ b/src/mode/rust_highlight_rules.js @@ -4,105 +4,199 @@ var oop = require("../lib/oop"); var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules; +var DocCommentHighlightRules = require("./doc_comment_highlight_rules").DocCommentHighlightRules; var stringEscape = /\\(?:[nrt0'"\\]|x[\da-fA-F]{2}|u\{[\da-fA-F]{6}\})/.source; var wordPattern = /[a-zA-Z_\xa1-\uffff][a-zA-Z0-9_\xa1-\uffff]*/.source; var RustHighlightRules = function() { // regexp must not have capturing parentheses. Use (?:) instead. // regexps are ordered -> the first match is used + var keywordMapper = this.createKeywordMapper({ + "keyword.source.rust": "abstract|alignof|as|async|await|become|box|break|catch|continue|const|crate|" + + "default|do|dyn|else|enum|extern|for|final|if|impl|in|let|loop|macro|match|mod|move|mut|offsetof|" + + "override|priv|proc|pub|pure|ref|return|self|sizeof|static|struct|super|trait|type|typeof|union|" + + "unsafe|unsized|use|virtual|where|while|yield|try", + "storage.type.source.rust": "Self|isize|usize|char|bool|u8|u16|u32|u64|u128|f16|f32|f64|i8|i16|i32|i64|" + + "i128|str|option|either|c_float|c_double|c_void|FILE|fpos_t|DIR|dirent|c_char|c_schar|c_uchar|c_short|" + + "c_ushort|c_int|c_uint|c_long|c_ulong|size_t|ptrdiff_t|clock_t|time_t|c_longlong|c_ulonglong|intptr_t|" + + "uintptr_t|off_t|dev_t|ino_t|pid_t|mode_t|ssize_t", + "constant.language.source.rust": "true|false|Some|None|Ok|Err|FALSE|TRUE", + "support.constant.source.rust": "EXIT_FAILURE|EXIT_SUCCESS|RAND_MAX|EOF|SEEK_SET|SEEK_CUR|SEEK_END|_IOFBF|" + + "_IONBF|_IOLBF|" + + "BUFSIZ|FOPEN_MAX|FILENAME_MAX|L_tmpnam|TMP_MAX|O_RDONLY|O_WRONLY|O_RDWR|O_APPEND|O_CREAT|O_EXCL|O_TRUNC|" + + "S_IFIFO|S_IFCHR|S_IFBLK|S_IFDIR|S_IFREG|S_IFMT|S_IEXEC|S_IWRITE|S_IREAD|S_IRWXU|S_IXUSR|S_IWUSR|S_IRUSR|" + + "F_OK|R_OK|W_OK|X_OK|STDIN_FILENO|STDOUT_FILENO|STDERR_FILENO", + "constant.language": "macro_rules|mac_variant" + }, "identifier"); - this.$rules = { start: - [ { token: 'variable.other.source.rust', - // `(?![\\\'])` to keep a lifetime name highlighting from continuing one character - // past the name. The end `\'` will block this from matching for a character like - // `'a'` (it should have character highlighting, not variable highlighting). - regex: '\'' + wordPattern + '(?![\\\'])' }, - { token: 'string.quoted.single.source.rust', - regex: "'(?:[^'\\\\]|" + stringEscape + ")'" }, - { token: 'identifier', - regex: "r#" + wordPattern + "\\b" }, - { - stateName: "bracketedComment", - onMatch : function(value, currentState, stack){ - stack.unshift(this.next, value.length - 1, currentState); - return "string.quoted.raw.source.rust"; - }, - regex : /r#*"/, - next : [ - { - onMatch : function(value, currentState, stack) { - var token = "string.quoted.raw.source.rust"; - if (value.length >= stack[1]) { - if (value.length > stack[1]) - token = "invalid"; - stack.shift(); - stack.shift(); - this.next = stack.shift(); - } else { - this.next = ""; - } - return token; - }, - regex : /"#*/, - next : "start" - }, { - defaultToken : "string.quoted.raw.source.rust" - } - ] - }, - { token: 'string.quoted.double.source.rust', - regex: '"', - push: - [ { token: 'string.quoted.double.source.rust', + this.$rules = { + start: [ + { + token: 'variable.other.source.rust', // `(?![\\\'])` to keep a lifetime name highlighting from continuing one character + // past the name. The end `\'` will block this from matching for a character like + // `'a'` (it should have character highlighting, not variable highlighting). + regex: '\'' + wordPattern + '(?![\\\'])' + }, { + token: 'string.quoted.single.source.rust', + regex: "'(?:[^'\\\\]|" + stringEscape + ")'" + }, { + token: 'identifier', + regex: "r#" + wordPattern + "\\b" + }, { + stateName: "bracketedComment", + onMatch: function (value, currentState, stack) { + stack.unshift(this.next, value.length - 1, currentState); + return "string.quoted.raw.source.rust"; + }, + regex: /r#*"/, + next: [ + { + onMatch: function (value, currentState, stack) { + var token = "string.quoted.raw.source.rust"; + if (value.length >= stack[1]) { + if (value.length > stack[1]) token = "invalid"; + stack.shift(); + stack.shift(); + this.next = stack.shift(); + } + else { + this.next = ""; + } + return token; + }, + regex: /"#*/, + next: "start" + }, { + defaultToken: "string.quoted.raw.source.rust" + } + ] + }, { + token: 'string.quoted.double.source.rust', regex: '"', - next: 'pop' }, - { token: 'constant.character.escape.source.rust', - regex: stringEscape }, - { defaultToken: 'string.quoted.double.source.rust' } ] }, - { token: [ 'keyword.source.rust', 'text', 'entity.name.function.source.rust' ], - regex: '\\b(fn)(\\s+)((?:r#)?'+ wordPattern + ')' }, - { token: 'support.constant', regex: wordPattern + '::' }, - { token: 'keyword.source.rust', - regex: '\\b(?:abstract|alignof|as|async|await|become|box|break|catch|continue|const|crate|default|do|dyn|else|enum|extern|for|final|if|impl|in|let|loop|macro|match|mod|move|mut|offsetof|override|priv|proc|pub|pure|ref|return|self|sizeof|static|struct|super|trait|type|typeof|union|unsafe|unsized|use|virtual|where|while|yield)\\b' }, - { token: 'storage.type.source.rust', - regex: '\\b(?:Self|isize|usize|char|bool|u8|u16|u32|u64|u128|f16|f32|f64|i8|i16|i32|i64|i128|str|option|either|c_float|c_double|c_void|FILE|fpos_t|DIR|dirent|c_char|c_schar|c_uchar|c_short|c_ushort|c_int|c_uint|c_long|c_ulong|size_t|ptrdiff_t|clock_t|time_t|c_longlong|c_ulonglong|intptr_t|uintptr_t|off_t|dev_t|ino_t|pid_t|mode_t|ssize_t)\\b' }, - { token: 'variable.language.source.rust', regex: '\\bself\\b' }, - - { token: 'comment.line.doc.source.rust', - regex: '//!.*$' }, - { token: 'comment.line.double-dash.source.rust', - regex: '//.*$' }, - { token: 'comment.start.block.source.rust', - regex: '/\\*', - stateName: 'comment', - push: - [ { token: 'comment.start.block.source.rust', + push: [ + { + token: 'string.quoted.double.source.rust', + regex: '"', + next: 'pop' + }, { + token: 'constant.character.escape.source.rust', + regex: stringEscape + }, {defaultToken: 'string.quoted.double.source.rust'} + ] + }, { + token: ['keyword.source.rust', 'text', 'entity.name.function.source.rust', 'punctuation'], + regex: '\\b(fn)(\\s+)((?:r#)?' + wordPattern + ')(<)', + push: "generics" + }, { + token: ['keyword.source.rust', 'text', 'entity.name.function.source.rust'], + regex: '\\b(fn)(\\s+)((?:r#)?' + wordPattern + ')' + }, { + token: ['support.constant', "punctuation"], + regex: "(" + wordPattern + '::)(<)', + push: "generics" + }, { + token: 'support.constant', + regex: wordPattern + '::' + }, { + token: 'variable.language.source.rust', + regex: '\\bself\\b' + }, DocCommentHighlightRules.getStartRule("doc-start"), { + token: 'comment.line.doc.source.rust', + regex: '///.*$' + }, { + token: 'comment.line.doc.source.rust', + regex: '//!.*$' + }, { + token: 'comment.line.double-dash.source.rust', + regex: '//.*$' + }, { + token: 'comment.start.block.source.rust', regex: '/\\*', - push: 'comment' }, - { token: 'comment.end.block.source.rust', - regex: '\\*/', - next: 'pop' }, - { defaultToken: 'comment.block.source.rust' } ] }, - - { token: 'keyword.operator', - // `[*/](?![*/])=?` is separated because `//` and `/* */` become comments and must be - // guarded against. This states either `*` or `/` may be matched as long as the match - // it isn't followed by either of the two. An `=` may be on the end. - regex: /\$|[-=]>|[-+%^=!&|<>]=?|[*/](?![*/])=?/ }, - { token : "punctuation.operator", regex : /[?:,;.]/ }, - { token : "paren.lparen", regex : /[\[({]/ }, - { token : "paren.rparen", regex : /[\])}]/ }, - { token: 'constant.language.source.rust', - regex: '\\b(?:true|false|Some|None|Ok|Err)\\b' }, - { token: 'support.constant.source.rust', - regex: '\\b(?:EXIT_FAILURE|EXIT_SUCCESS|RAND_MAX|EOF|SEEK_SET|SEEK_CUR|SEEK_END|_IOFBF|_IONBF|_IOLBF|BUFSIZ|FOPEN_MAX|FILENAME_MAX|L_tmpnam|TMP_MAX|O_RDONLY|O_WRONLY|O_RDWR|O_APPEND|O_CREAT|O_EXCL|O_TRUNC|S_IFIFO|S_IFCHR|S_IFBLK|S_IFDIR|S_IFREG|S_IFMT|S_IEXEC|S_IWRITE|S_IREAD|S_IRWXU|S_IXUSR|S_IWUSR|S_IRUSR|F_OK|R_OK|W_OK|X_OK|STDIN_FILENO|STDOUT_FILENO|STDERR_FILENO)\\b' }, - { token: 'meta.preprocessor.source.rust', - regex: '\\b\\w\\(\\w\\)*!|#\\[[\\w=\\(\\)_]+\\]\\b' }, - { token: 'constant.numeric.source.rust', - regex: /\b(?:0x[a-fA-F0-9_]+|0o[0-7_]+|0b[01_]+|[0-9][0-9_]*(?!\.))(?:[iu](?:size|8|16|32|64|128))?\b/ }, - { token: 'constant.numeric.source.rust', - regex: /\b(?:[0-9][0-9_]*)(?:\.[0-9][0-9_]*)?(?:[Ee][+-][0-9][0-9_]*)?(?:f32|f64)?\b/ } ] }; - + stateName: 'comment', + push: [ + { + token: 'comment.start.block.source.rust', + regex: '/\\*', + push: 'comment' + }, { + token: 'comment.end.block.source.rust', + regex: '\\*/', + next: 'pop' + }, {defaultToken: 'comment.block.source.rust'} + ] + }, { + token: ["keyword.source.rust", "identifier", "punctuaction"], + regex: "(?:(impl)|(" + wordPattern + "))(<)", + stateName: 'generics', + push: [ + { + token: "punctuaction", + regex: "<", + push: "generics" + }, { + token: 'variable.other.source.rust', // `(?![\\\'])` to keep a lifetime name highlighting from continuing one character + // past the name. The end `\'` will block this from matching for a character like + // `'a'` (it should have character highlighting, not variable highlighting). + regex: '\'' + wordPattern + '(?![\\\'])' + }, { + token: "storage.type.source.rust", + regex: "\\b(u8|u16|u32|u64|u128|usize|i8|i16|i32|i64|i128|isize|char|bool)\\b" + }, { + token: "punctuation.operator", + regex: "[,:]" + }, { + token: "keyword", + regex: "\\b(?:const|dyn)\\b" + }, { + token: "punctuation", + regex: ">", + next: "pop" + }, { + token: "paren.lparen", + regex: "[(]" + }, { + token: "paren.rparen", + regex: "[)]" + }, { + token: "identifier", + regex: "\\b"+wordPattern+"\\b" + }, { + token: 'keyword.operator', + regex: "=" + } + ] + }, { + token: keywordMapper, + regex: wordPattern + }, { + token: 'keyword.operator', // `[*/](?![*/])=?` is separated because `//` and `/* */` become comments and must be + // guarded against. This states either `*` or `/` may be matched as long as the match + // it isn't followed by either of the two. An `=` may be on the end. + regex: /\$|[-=]>|[-+%^=!&|<>]=?|[*/](?![*/])=?/ + }, { + token: "punctuation.operator", + regex: /[?:,;.]/ + }, { + token: "paren.lparen", + regex: /[\[({]/ + }, { + token: "paren.rparen", + regex: /[\])}]/ + }, { + token: 'meta.preprocessor.source.rust', + regex: '\\b\\w\\(\\w\\)*!|#\\[[\\w=\\(\\)_]+\\]\\b' + }, { + token: 'constant.numeric.source.rust', + regex: /\b(?:0x[a-fA-F0-9_]+|0o[0-7_]+|0b[01_]+|[0-9][0-9_]*(?!\.))(?:[iu](?:size|8|16|32|64|128))?\b/ + }, { + token: 'constant.numeric.source.rust', + regex: /\b(?:[0-9][0-9_]*)(?:\.[0-9][0-9_]*)?(?:[Ee][+-][0-9][0-9_]*)?(?:f32|f64)?\b/ + } + ] + }; + + this.embedRules(DocCommentHighlightRules, "doc-", + [ DocCommentHighlightRules.getEndRule("start") ]); this.normalizeRules(); };