Skip to content

Commit

Permalink
Merge pull request #14 from Microsoft/kieferrm/support-while
Browse files Browse the repository at this point in the history 
Support for grammars with begin/while rules
  • Loading branch information
@alexdima
alexdima committed May 23, 2016
2 parents 214c326 + 2874c53 commit b14ff75
Show file tree
Hide file tree
Showing 7 changed files with 261 additions and 22 deletions.
1 change: 1 addition & 0 deletions release/main.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ export interface StackElement {
ruleId: number;
enterPos: number;
endRule: string;
whileRule: string;
scopeName: string;
contentName: string;

Expand Down
118 changes: 107 additions & 11 deletions release/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -767,6 +767,60 @@ var BeginEndRule = (function (_super) {
return BeginEndRule;
}(Rule));
exports.BeginEndRule = BeginEndRule;
var BeginWhileRule = (function (_super) {
__extends(BeginWhileRule, _super);
function BeginWhileRule(id, name, contentName, begin, beginCaptures, _while, patterns) {
_super.call(this, id, name, contentName);
this._begin = new RegExpSource(begin, this.id);
this.beginCaptures = beginCaptures;
this._while = new RegExpSource(_while, -2);
this.whileHasBackReferences = this._while.hasBackReferences;
this.patterns = patterns.patterns;
this.hasMissingPatterns = patterns.hasMissingPatterns;
this._cachedCompiledPatterns = null;
this._cachedCompiledWhilePatterns = null;
}
BeginWhileRule.prototype.getWhileWithResolvedBackReferences = function (lineText, captureIndices) {
return this._while.resolveBackReferences(lineText, captureIndices);
};
BeginWhileRule.prototype.collectPatternsRecursive = function (grammar, out, isFirst) {
if (isFirst) {
var i = void 0, len = void 0, rule = void 0;
for (i = 0, len = this.patterns.length; i < len; i++) {
rule = grammar.getRule(this.patterns[i]);
rule.collectPatternsRecursive(grammar, out, false);
}
}
else {
out.push(this._begin);
}
};
BeginWhileRule.prototype.compile = function (grammar, endRegexSource, allowA, allowG) {
this._precompile(grammar);
return this._cachedCompiledPatterns.compile(grammar, allowA, allowG);
};
BeginWhileRule.prototype._precompile = function (grammar) {
if (!this._cachedCompiledPatterns) {
this._cachedCompiledPatterns = new RegExpSourceList();
this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true);
}
};
BeginWhileRule.prototype.compileWhile = function (grammar, endRegexSource, allowA, allowG) {
this._precompileWhile(grammar);
if (this._while.hasBackReferences) {
this._cachedCompiledWhilePatterns.setSource(0, endRegexSource);
}
return this._cachedCompiledWhilePatterns.compile(grammar, allowA, allowG);
};
BeginWhileRule.prototype._precompileWhile = function (grammar) {
if (!this._cachedCompiledWhilePatterns) {
this._cachedCompiledWhilePatterns = new RegExpSourceList();
this._cachedCompiledWhilePatterns.push(this._while.hasBackReferences ? this._while.clone() : this._while);
}
};
return BeginWhileRule;
}(Rule));
exports.BeginWhileRule = BeginWhileRule;
var RuleFactory = (function () {
function RuleFactory() {
}
Expand All @@ -788,6 +842,9 @@ var RuleFactory = (function () {
}
return new IncludeOnlyRule(desc.id, desc.name, desc.contentName, RuleFactory._compilePatterns(desc.patterns, helper, repository));
}
if (desc.while) {
return new BeginWhileRule(desc.id, desc.name, desc.contentName, desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository), desc.while, RuleFactory._compilePatterns(desc.patterns, helper, repository));
}
return new BeginEndRule(desc.id, desc.name, desc.contentName, desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository), desc.end, RuleFactory._compileCaptures(desc.endCaptures || desc.captures, helper, repository), desc.applyEndPatternLast, RuleFactory._compilePatterns(desc.patterns, helper, repository));
});
}
Expand Down Expand Up @@ -874,12 +931,7 @@ var RuleFactory = (function () {
if (patternId !== -1) {
rule = helper.getRule(patternId);
skipRule = false;
if (rule instanceof IncludeOnlyRule) {
if (rule.hasMissingPatterns && rule.patterns.length === 0) {
skipRule = true;
}
}
else if (rule instanceof BeginEndRule) {
if (rule instanceof IncludeOnlyRule || rule instanceof BeginEndRule || rule instanceof BeginWhileRule) {
if (rule.hasMissingPatterns && rule.patterns.length === 0) {
skipRule = true;
}
Expand Down Expand Up @@ -1178,7 +1230,26 @@ function matchInjections(injections, grammar, lineText, isFirstLine, linePos, st
}
function matchRule(grammar, lineText, isFirstLine, linePos, stack, anchorPosition) {
var stackElement = stack[stack.length - 1];
var ruleScanner = grammar.getRule(stackElement.ruleId).compile(grammar, stackElement.endRule, isFirstLine, linePos === anchorPosition);
var rule = grammar.getRule(stackElement.ruleId);
if (rule instanceof rule_1.BeginWhileRule && stackElement.enterPos === -1) {
var ruleScanner_1 = rule.compileWhile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
var r_1 = ruleScanner_1.scanner._findNextMatchSync(lineText, linePos);
var doNotContinue = {
captureIndices: null,
matchedRuleId: -3
};
if (r_1) {
var matchedRuleId = ruleScanner_1.rules[r_1.index];
if (matchedRuleId != -2) {
// we shouldn't end up here
return doNotContinue;
}
}
else {
return doNotContinue;
}
}
var ruleScanner = rule.compile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
var r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
if (r) {
return {
Expand Down Expand Up @@ -1232,7 +1303,7 @@ function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTok
}
var captureIndices = r.captureIndices;
var matchedRuleId = r.matchedRuleId;
var hasAdvanced = (captureIndices[0].end > linePos);
var hasAdvanced = (captureIndices && captureIndices.length > 0) ? (captureIndices[0].end > linePos) : false;
if (matchedRuleId === -1) {
// We matched the `end` for this rule => pop it
var poppedRule = grammar.getRule(stackElement.ruleId);
Expand All @@ -1250,12 +1321,17 @@ function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTok
return false;
}
}
else if (matchedRuleId === -3) {
// A while clause failed
stack.pop();
return true;
}
else {
// We matched a rule!
var _rule = grammar.getRule(matchedRuleId);
lineTokens.produce(stack, captureIndices[0].start);
// push it on the stack rule
stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(rule_1.getString(lineText), captureIndices), null));
stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(rule_1.getString(lineText), captureIndices), null, null));
if (_rule instanceof rule_1.BeginEndRule) {
var pushedRule = _rule;
handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices);
Expand All @@ -1274,6 +1350,24 @@ function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTok
return false;
}
}
else if (_rule instanceof rule_1.BeginWhileRule) {
var pushedRule = _rule;
handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices);
lineTokens.produce(stack, captureIndices[0].end);
anchorPosition = captureIndices[0].end;
stack[stack.length - 1].contentName = pushedRule.getContentName(rule_1.getString(lineText), captureIndices);
if (pushedRule.whileHasBackReferences) {
stack[stack.length - 1].whileRule = pushedRule.getWhileWithResolvedBackReferences(rule_1.getString(lineText), captureIndices);
}
if (!hasAdvanced && stackElement.ruleId === stack[stack.length - 1].ruleId) {
// Grammar pushed the same rule without advancing
console.error('Grammar is in an endless loop - case 2');
stack.pop();
lineTokens.produce(stack, lineLength);
linePos = lineLength;
return false;
}
}
else {
var matchingRule = _rule;
handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, matchingRule.captures, captureIndices);
Expand Down Expand Up @@ -1301,15 +1395,17 @@ function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTok
}
}
var StackElement = (function () {
function StackElement(ruleId, enterPos, endRule, scopeName, contentName) {
function StackElement(ruleId, enterPos, endRule, scopeName, contentName, whileRule) {
if (whileRule === void 0) { whileRule = null; }
this.ruleId = ruleId;
this.enterPos = enterPos;
this.endRule = endRule;
this.scopeName = scopeName;
this.contentName = contentName;
this.whileRule = whileRule;
}
StackElement.prototype.clone = function () {
return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName);
return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName, this.whileRule);
};
StackElement.prototype.matches = function (scopeName) {
if (!this.scopeName) {
Expand Down
62 changes: 56 additions & 6 deletions src/grammar.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import {clone} from './utils';
import {IRawGrammar, IRawRepository, IRawRule} from './types';
import {IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, MatchRule, ICompiledRule, createOnigString, getString} from './rule';
import {IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, BeginWhileRule, MatchRule, ICompiledRule, createOnigString, getString} from './rule';
import {IOnigCaptureIndex, IOnigNextMatchResult, OnigString} from 'oniguruma';
import {createMatcher, Matcher} from './matcher';

Expand Down Expand Up @@ -381,7 +381,31 @@ interface IMatchResult {

function matchRule(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement[], anchorPosition:number): IMatchResult {
let stackElement = stack[stack.length - 1];
let ruleScanner = grammar.getRule(stackElement.ruleId).compile(grammar, stackElement.endRule, isFirstLine, linePos === anchorPosition);
let rule = grammar.getRule(stackElement.ruleId);

if (rule instanceof BeginWhileRule && stackElement.enterPos === -1) {

let ruleScanner = rule.compileWhile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);

let doNotContinue: IMatchResult = {
captureIndices: null,
matchedRuleId: -3
};

if (r) {
let matchedRuleId = ruleScanner.rules[r.index];
if (matchedRuleId != -2) {
// we shouldn't end up here
return doNotContinue;
}
} else {
return doNotContinue;
}
}


let ruleScanner = rule.compile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);

if (r) {
Expand Down Expand Up @@ -450,7 +474,7 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
let captureIndices: IOnigCaptureIndex[] = r.captureIndices;
let matchedRuleId: number = r.matchedRuleId;

let hasAdvanced = (captureIndices[0].end > linePos);
let hasAdvanced = (captureIndices && captureIndices.length > 0) ? (captureIndices[0].end > linePos) : false;

if (matchedRuleId === -1) {
// We matched the `end` for this rule => pop it
Expand All @@ -471,6 +495,10 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
linePos = lineLength;
return false;
}
} else if (matchedRuleId === -3) {
// A while clause failed
stack.pop();
return true;

} else {
// We matched a rule!
Expand All @@ -479,7 +507,7 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
lineTokens.produce(stack, captureIndices[0].start);

// push it on the stack rule
stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(getString(lineText), captureIndices), null));
stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(getString(lineText), captureIndices), null, null));

if (_rule instanceof BeginEndRule) {
let pushedRule = <BeginEndRule>_rule;
Expand All @@ -493,6 +521,26 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
stack[stack.length-1].endRule = pushedRule.getEndWithResolvedBackReferences(getString(lineText), captureIndices);
}

if (!hasAdvanced && stackElement.ruleId === stack[stack.length - 1].ruleId) {
// Grammar pushed the same rule without advancing
console.error('Grammar is in an endless loop - case 2');
stack.pop();
lineTokens.produce(stack, lineLength);
linePos = lineLength;
return false;
}
} else if (_rule instanceof BeginWhileRule) {
let pushedRule = <BeginWhileRule>_rule;

handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices);
lineTokens.produce(stack, captureIndices[0].end);
anchorPosition = captureIndices[0].end;
stack[stack.length - 1].contentName = pushedRule.getContentName(getString(lineText), captureIndices);

if (pushedRule.whileHasBackReferences) {
stack[stack.length - 1].whileRule = pushedRule.getWhileWithResolvedBackReferences(getString(lineText), captureIndices);
}

if (!hasAdvanced && stackElement.ruleId === stack[stack.length - 1].ruleId) {
// Grammar pushed the same rule without advancing
console.error('Grammar is in an endless loop - case 2');
Expand Down Expand Up @@ -538,19 +586,21 @@ export class StackElement {
public endRule: string;
public scopeName: string;
public contentName: string;
public whileRule: string;

private scopeNameSegments: { [segment:string]:boolean };

constructor (ruleId:number, enterPos:number, endRule:string, scopeName:string, contentName:string) {
constructor(ruleId:number, enterPos:number, endRule:string, scopeName:string, contentName: string, whileRule: string = null) {
this.ruleId = ruleId;
this.enterPos = enterPos;
this.endRule = endRule;
this.scopeName = scopeName;
this.contentName = contentName;
this.whileRule = whileRule;
}

public clone(): StackElement {
return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName);
return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName, this.whileRule);
}

public matches(scopeName: string) : boolean {
Expand Down
Loading

0 comments on commit b14ff75

Please sign in to comment.