Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add line and column information to scanner and visitor #17

Merged
merged 3 commits into from
Mar 21, 2019
Merged

add line and column information to scanner and visitor #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7971774
add onLineBreak visit function
P0lip Mar 14, 2019
7479e45
sort out whitespaces in readme
P0lip Mar 20, 2019
85b5249
getTokenStartLine/StartCharacter
P0lip Mar 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ export interface JSONScanner {
* The length of the last read token.
*/
getTokenLength(): number;
/**
* The zero-based start line number of the last read token.
*/
getTokenStartLine(): number;
/**
* The zero-based start character (column) of the last read token.
*/
getTokenStartCharacter(): number;
/**
* An error code of the last scan.
*/
Expand Down Expand Up @@ -96,39 +104,39 @@ export interface JSONVisitor {
/**
* Invoked when an open brace is encountered and an object is started. The offset and length represent the location of the open brace.
*/
onObjectBegin?: (offset: number, length: number) => void;
onObjectBegin?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a property is encountered. The offset and length represent the location of the property name.
*/
onObjectProperty?: (property: string, offset: number, length: number) => void;
onObjectProperty?: (property: string, offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a closing brace is encountered and an object is completed. The offset and length represent the location of the closing brace.
*/
onObjectEnd?: (offset: number, length: number) => void;
onObjectEnd?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when an open bracket is encountered. The offset and length represent the location of the open bracket.
*/
onArrayBegin?: (offset: number, length: number) => void;
onArrayBegin?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a closing bracket is encountered. The offset and length represent the location of the closing bracket.
*/
onArrayEnd?: (offset: number, length: number) => void;
onArrayEnd?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a literal value is encountered. The offset and length represent the location of the literal value.
*/
onLiteralValue?: (value: any, offset: number, length: number) => void;
onLiteralValue?: (value: any, offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked when a comma or colon separator is encountered. The offset and length represent the location of the separator.
*/
onSeparator?: (charcter: string, offset: number, length: number) => void;
/**
* When comments are allowed, invoked when a line or block comment is encountered. The offset and length represent the location of the comment.
*/
onComment?: (offset: number, length: number) => void;
onSeparator?: (character: string, offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* When comments are allowed, invoked when a line or block comment is encountered. The offset and length represent the location of the comment.
*/
onComment?: (offset: number, length: number, startLine: number, startCharacter: number) => void;
/**
* Invoked on an error.
*/
onError?: (error: ParseErrorCode, offset: number, length: number) => void;
onError?: (error: ParseErrorCode, offset: number, length: number, startLine: number, startCharacter: number) => void;
}

/**
Expand Down Expand Up @@ -298,4 +306,4 @@ License

(MIT License)

Copyright 2018, Microsoft
Copyright 2018, Microsoft
23 changes: 16 additions & 7 deletions src/impl/parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,17 @@

import { createScanner } from './scanner';
import {
ScanError, SyntaxKind, Node, NodeType, Edit, JSONPath, FormattingOptions,
ModificationOptions, ParseError, ParseErrorCode, Location, Segment, ParseOptions, JSONVisitor
JSONPath,
JSONVisitor,
Location,
Node,
NodeType,
ParseError,
ParseErrorCode,
ParseOptions,
ScanError,
Segment,
SyntaxKind
} from '../main';

namespace ParseOptions {
Expand Down Expand Up @@ -376,11 +385,11 @@ export function visit(text: string, visitor: JSONVisitor, options: ParseOptions

let _scanner = createScanner(text, false);

function toNoArgVisit(visitFunction?: (offset: number, length: number) => void): () => void {
return visitFunction ? () => visitFunction(_scanner.getTokenOffset(), _scanner.getTokenLength()) : () => true;
function toNoArgVisit(visitFunction?: (offset: number, length: number, startLine: number, startCharacter: number) => void): () => void {
return visitFunction ? () => visitFunction(_scanner.getTokenOffset(), _scanner.getTokenLength(), _scanner.getTokenStartLine(), _scanner.getTokenStartCharacter()) : () => true;
}
function toOneArgVisit<T>(visitFunction?: (arg: T, offset: number, length: number) => void): (arg: T) => void {
return visitFunction ? (arg: T) => visitFunction(arg, _scanner.getTokenOffset(), _scanner.getTokenLength()) : () => true;
function toOneArgVisit<T>(visitFunction?: (arg: T, offset: number, length: number, startLine: number, startCharacter: number) => void): (arg: T) => void {
return visitFunction ? (arg: T) => visitFunction(arg, _scanner.getTokenOffset(), _scanner.getTokenLength(), _scanner.getTokenStartLine(), _scanner.getTokenStartCharacter()) : () => true;
}

let onObjectBegin = toNoArgVisit(visitor.onObjectBegin),
Expand Down Expand Up @@ -650,4 +659,4 @@ function getLiteralNodeType(value: any): NodeType {
case 'string': return 'string';
default: return 'null';
}
}
}
22 changes: 21 additions & 1 deletion src/impl/scanner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
value: string = '',
tokenOffset = 0,
token: SyntaxKind = SyntaxKind.Unknown,
lineNumber = 0,
lineStartOffset = 0,
tokenLineStartOffset = 0,
prevTokenLineStartOffset = 0,
scanError: ScanError = ScanError.None;

function scanHexDigits(count: number, exact?: boolean): number {
Expand Down Expand Up @@ -179,6 +183,8 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
scanError = ScanError.None;

tokenOffset = pos;
lineStartOffset = lineNumber;
prevTokenLineStartOffset = tokenLineStartOffset;

if (pos >= len) {
// at the end
Expand Down Expand Up @@ -206,6 +212,8 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
pos++;
value += '\n';
}
lineNumber++;
tokenLineStartOffset = pos;
return token = SyntaxKind.LineBreakTrivia;
}

Expand Down Expand Up @@ -268,7 +276,17 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
commentClosed = true;
break;
}

pos++;

if (isLineBreak(ch)) {
if (ch === CharacterCodes.carriageReturn && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
pos++;
}

lineNumber++;
tokenLineStartOffset = pos;
}
}

if (!commentClosed) {
Expand Down Expand Up @@ -365,7 +383,9 @@ export function createScanner(text: string, ignoreTrivia: boolean = false): JSON
getTokenValue: () => value,
getTokenOffset: () => tokenOffset,
getTokenLength: () => pos - tokenOffset,
getTokenError: () => scanError
getTokenStartLine: () => lineStartOffset,
getTokenStartCharacter: () => tokenOffset - prevTokenLineStartOffset,
getTokenError: () => scanError,
};
}

Expand Down
26 changes: 17 additions & 9 deletions src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,14 @@ export interface JSONScanner {
* The length of the last read token.
*/
getTokenLength(): number;
/**
* The zero-based start line number of the last read token.
*/
getTokenStartLine(): number;
/**
* The zero-based start character (column) of the last read token.
*/
getTokenStartCharacter(): number;
/**
* An error code of the last scan.
*/
Expand Down Expand Up @@ -225,47 +233,47 @@ export interface JSONVisitor {
/**
* Invoked when an open brace is encountered and an object is started. The offset and length represent the location of the open brace.
*/
onObjectBegin?: (offset: number, length: number) => void;
onObjectBegin?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a property is encountered. The offset and length represent the location of the property name.
*/
onObjectProperty?: (property: string, offset: number, length: number) => void;
onObjectProperty?: (property: string, offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a closing brace is encountered and an object is completed. The offset and length represent the location of the closing brace.
*/
onObjectEnd?: (offset: number, length: number) => void;
onObjectEnd?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when an open bracket is encountered. The offset and length represent the location of the open bracket.
*/
onArrayBegin?: (offset: number, length: number) => void;
onArrayBegin?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a closing bracket is encountered. The offset and length represent the location of the closing bracket.
*/
onArrayEnd?: (offset: number, length: number) => void;
onArrayEnd?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a literal value is encountered. The offset and length represent the location of the literal value.
*/
onLiteralValue?: (value: any, offset: number, length: number) => void;
onLiteralValue?: (value: any, offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked when a comma or colon separator is encountered. The offset and length represent the location of the separator.
*/
onSeparator?: (character: string, offset: number, length: number) => void;
onSeparator?: (character: string, offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* When comments are allowed, invoked when a line or block comment is encountered. The offset and length represent the location of the comment.
*/
onComment?: (offset: number, length: number) => void;
onComment?: (offset: number, length: number, startLine: number, startCharacter: number) => void;

/**
* Invoked on an error.
*/
onError?: (error: ParseErrorCode, offset: number, length: number) => void;
onError?: (error: ParseErrorCode, offset: number, length: number, startLine: number, startCharacter: number) => void;
}

/**
Expand Down
97 changes: 81 additions & 16 deletions src/test/json.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,20 @@ function assertTree(input: string, expected: any, expectedErrors: ParseError[] =
interface VisitorCallback {
id: keyof JSONVisitor,
text: string;
startLine: number;
startCharacter: number;
arg?: any;
};
interface VisitorError extends ParseError {
startLine: number;
startCharacter: number;
}

function assertVisit(input: string, expected: VisitorCallback[], expectedErrors: ParseError[] = [], disallowComments = false): void {
let errors: ParseError[] = [];
function assertVisit(input: string, expected: VisitorCallback[], expectedErrors: VisitorError[] = [], disallowComments = false): void {
let errors: VisitorError[] = [];
let actuals: VisitorCallback[] = [];
let noArgHalder = (id: keyof JSONVisitor) => (offset: number, length: number) => actuals.push({ id, text: input.substr(offset, length) });
let oneArgHalder = (id: keyof JSONVisitor) => (arg: any, offset: number, length: number) => actuals.push({ id, text: input.substr(offset, length), arg });
let noArgHalder = (id: keyof JSONVisitor) => (offset: number, length: number, startLine: number, startCharacter: number) => actuals.push({ id, text: input.substr(offset, length), startLine, startCharacter });
let oneArgHalder = (id: keyof JSONVisitor) => (arg: any, offset: number, length: number, startLine: number, startCharacter: number) => actuals.push({ id, text: input.substr(offset, length), startLine, startCharacter, arg });
visit(input, {
onObjectBegin: noArgHalder('onObjectBegin'),
onObjectProperty: oneArgHalder('onObjectProperty'),
Expand All @@ -87,8 +93,8 @@ function assertVisit(input: string, expected: VisitorCallback[], expectedErrors:
onLiteralValue: oneArgHalder('onLiteralValue'),
onSeparator: oneArgHalder('onSeparator'),
onComment: noArgHalder('onComment'),
onError: (error: ParseErrorCode, offset: number, length: number) => {
errors.push({ error, offset, length })
onError: (error: ParseErrorCode, offset: number, length: number, startLine: number, startCharacter: number) => {
errors.push({ error, offset, length, startLine, startCharacter })
}
}, {
disallowComments
Expand Down Expand Up @@ -282,7 +288,7 @@ suite('JSON', () => {
assertInvalidParse('[ ,1, 2, 3, ]', [1, 2, 3]);
});

test('parse: disallow commments', () => {
test('parse: disallow comments', () => {
let options = { disallowComments: true };

assertValidParse('[ 1, 2, null, "foo" ]', [1, 2, null, 'foo'], options);
Expand Down Expand Up @@ -422,21 +428,80 @@ suite('JSON', () => {
});

test('visit: object', () => {
assertVisit('{ }', [{ id: 'onObjectBegin', text: '{' }, { id: 'onObjectEnd', text: '}' }]);
assertVisit('{ "foo": "bar" }', [{ id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"foo"', arg: 'foo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onLiteralValue', text: '"bar"', arg: 'bar' }, { id: 'onObjectEnd', text: '}' }]);
assertVisit('{ "foo": { "goo": 3 } }', [{ id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"foo"', arg: 'foo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"goo"', arg: 'goo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onLiteralValue', text: '3', arg: 3 }, { id: 'onObjectEnd', text: '}' }, { id: 'onObjectEnd', text: '}' }]);
assertVisit('{ }', [{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 0 }, { id: 'onObjectEnd', text: '}', startLine: 0, startCharacter: 2 }]);
assertVisit('{ "foo": "bar" }', [
{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 0 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 0, startCharacter: 2, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 0, startCharacter: 7, arg: ':' },
{ id: 'onLiteralValue', text: '"bar"', startLine: 0, startCharacter: 9, arg: 'bar' },
{ id: 'onObjectEnd', text: '}', startLine: 0, startCharacter: 15 },
]);
assertVisit('{ "foo": { "goo": 3 } }', [
{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 0 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 0, startCharacter: 2, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 0, startCharacter: 7, arg: ':' },
{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 9 },
{ id: 'onObjectProperty', text: '"goo"', startLine: 0, startCharacter: 11, arg: 'goo' },
{ id: 'onSeparator', text: ':', startLine: 0, startCharacter: 16, arg: ':' },
{ id: 'onLiteralValue', text: '3', startLine: 0, startCharacter: 18, arg: 3 },
{ id: 'onObjectEnd', text: '}', startLine: 0, startCharacter: 20 },
{ id: 'onObjectEnd', text: '}', startLine: 0, startCharacter: 22 },
]);
});

test('visit: array', () => {
assertVisit('[]', [{ id: 'onArrayBegin', text: '[' }, { id: 'onArrayEnd', text: ']' }]);
assertVisit('[ true, null, [] ]', [{ id: 'onArrayBegin', text: '[' }, { id: 'onLiteralValue', text: 'true', arg: true }, { id: 'onSeparator', text: ',', arg: ',' }, { id: 'onLiteralValue', text: 'null', arg: null }, { id: 'onSeparator', text: ',', arg: ',' }, { id: 'onArrayBegin', text: '[' }, { id: 'onArrayEnd', text: ']' }, { id: 'onArrayEnd', text: ']' }]);
assertVisit('[]', [{ id: 'onArrayBegin', text: '[', startLine: 0, startCharacter: 0 }, { id: 'onArrayEnd', text: ']', startLine: 0, startCharacter: 1 }]);
assertVisit('[ true, null, [] ]', [
{ id: 'onArrayBegin', text: '[', startLine: 0, startCharacter: 0 },
{ id: 'onLiteralValue', text: 'true', startLine: 0, startCharacter: 2, arg: true },
{ id: 'onSeparator', text: ',', startLine: 0, startCharacter: 6, arg: ',' },
{ id: 'onLiteralValue', text: 'null', startLine: 0, startCharacter: 8, arg: null },
{ id: 'onSeparator', text: ',', startLine: 0, startCharacter: 12, arg: ',' },
{ id: 'onArrayBegin', text: '[', startLine: 0, startCharacter: 14 },
{ id: 'onArrayEnd', text: ']', startLine: 0, startCharacter: 15 },
{ id: 'onArrayEnd', text: ']', startLine: 0, startCharacter: 17 },
]);
assertVisit('[\r\n0,\r\n1,\r\n2\r\n]', [
{ id: 'onArrayBegin', text: '[', startLine: 0, startCharacter: 0 },
{ id: 'onLiteralValue', text: '0', startLine: 1, startCharacter: 0, arg: 0 },
{ id: 'onSeparator', text: ',', startLine: 1, startCharacter: 1, arg: ',' },
{ id: 'onLiteralValue', text: '1', startLine: 2, startCharacter: 0, arg: 1 },
{ id: 'onSeparator', text: ',', startLine: 2, startCharacter: 1, arg: ',' },
{ id: 'onLiteralValue', text: '2', startLine: 3, startCharacter: 0, arg: 2 },
{ id: 'onArrayEnd', text: ']', startLine: 4, startCharacter: 0 }]);
});

test('visit: comment', () => {
assertVisit('/* g */ { "foo": //f\n"bar" }', [{ id: 'onComment', text: '/* g */' }, { id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"foo"', arg: 'foo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onComment', text: '//f' }, { id: 'onLiteralValue', text: '"bar"', arg: 'bar' }, { id: 'onObjectEnd', text: '}' }]);
assertVisit('/* g */ { "foo": //f\n"bar" }',
[{ id: 'onObjectBegin', text: '{' }, { id: 'onObjectProperty', text: '"foo"', arg: 'foo' }, { id: 'onSeparator', text: ':', arg: ':' }, { id: 'onLiteralValue', text: '"bar"', arg: 'bar' }, { id: 'onObjectEnd', text: '}' }],
[{ error: ParseErrorCode.InvalidCommentToken, offset: 0, length: 7 }, { error: ParseErrorCode.InvalidCommentToken, offset: 17, length: 3 }],
assertVisit('/* g */ { "foo": //f\n"bar" }', [
{ id: 'onComment', text: '/* g */', startLine: 0, startCharacter: 0 },
{ id: 'onObjectBegin', text: '{', startLine: 0, startCharacter: 8 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 0, startCharacter: 10, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 0, startCharacter: 15, arg: ':' },
{ id: 'onComment', text: '//f', startLine: 0, startCharacter: 17 },
{ id: 'onLiteralValue', text: '"bar"', startLine: 1, startCharacter: 0, arg: 'bar' },
{ id: 'onObjectEnd', text: '}', startLine: 1, startCharacter: 6 },
]);
assertVisit('/* g\r\n */ { "foo": //f\n"bar" }', [
{ id: 'onComment', text: '/* g\r\n */', startLine: 0, startCharacter: 0 },
{ id: 'onObjectBegin', text: '{', startLine: 1, startCharacter: 4 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 1, startCharacter: 6, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 1, startCharacter: 11, arg: ':' },
{ id: 'onComment', text: '//f', startLine: 1, startCharacter: 13 },
{ id: 'onLiteralValue', text: '"bar"', startLine: 2, startCharacter: 0, arg: 'bar' },
{ id: 'onObjectEnd', text: '}', startLine: 2, startCharacter: 6 },
]);
assertVisit('/* g\n */ { "foo": //f\n"bar"\n}',
[
{ id: 'onObjectBegin', text: '{', startLine: 1, startCharacter: 4 },
{ id: 'onObjectProperty', text: '"foo"', startLine: 1, startCharacter: 6, arg: 'foo' },
{ id: 'onSeparator', text: ':', startLine: 1, startCharacter: 11, arg: ':' },
{ id: 'onLiteralValue', text: '"bar"', startLine: 2, startCharacter: 0, arg: 'bar' },
{ id: 'onObjectEnd', text: '}', startLine: 3, startCharacter: 0 },
],
[
{ error: ParseErrorCode.InvalidCommentToken, offset: 0, length: 8, startLine: 0, startCharacter: 0 },
{ error: ParseErrorCode.InvalidCommentToken, offset: 18, length: 3, startLine: 1, startCharacter: 13 },
],
true);
});

Expand Down