Skip to content

Commit

Permalink
Lex $foo as a single token, instead of using '$' IDENT
Browse files Browse the repository at this point in the history
Previously, variable/symbols were parsed as the combination of two
tokens: '$' IDENT

This meant that using a keyword as variable name (e.g. $then, $label)
did not work.
Attempts were made to allow $keyword to work by adding some '$' Keyword
rules in the parser, but this did not allow $keyword in all places:

  jq --arg label foo -n '$label'  # ok

  jq -n '"foo" as $label | .'     # error

Treating $foo as a single token is much simpler, in my opinion.

This patch also changes how LOC is lexed: "$__loc__" instead of as
"__loc__" that gets combined with '$' in the parser.

This patch also disallows having spaces after '$' when recalling a
variable  `$ foo'  since that was probably just an unintentional side
effect of the implementation, and it was not documented.

Fixes #2675
  • Loading branch information
emanuele6 committed Jul 9, 2023
1 parent 600e602 commit 3426d32
Show file tree
Hide file tree
Showing 6 changed files with 1,558 additions and 1,617 deletions.
336 changes: 171 additions & 165 deletions src/lexer.c

Large diffs are not rendered by default.

11 changes: 5 additions & 6 deletions src/lexer.h
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,6 @@ typedef int16_t flex_int16_t;
typedef uint16_t flex_uint16_t;
typedef int32_t flex_int32_t;
typedef uint32_t flex_uint32_t;
typedef uint64_t flex_uint64_t;
#else
typedef signed char flex_int8_t;
typedef short int flex_int16_t;
Expand Down Expand Up @@ -399,7 +398,7 @@ struct yy_buffer_state
/* Number of characters read into yy_ch_buf, not including EOB
* characters.
*/
yy_size_t yy_n_chars;
int yy_n_chars;

/* Whether we "own" the buffer - i.e., we know we created it,
* and can realloc() it to grow it, and should free() it to
Expand Down Expand Up @@ -443,7 +442,7 @@ void yypop_buffer_state ( yyscan_t yyscanner );

YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size , yyscan_t yyscanner );
YY_BUFFER_STATE yy_scan_string ( const char *yy_str , yyscan_t yyscanner );
YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, yy_size_t len , yyscan_t yyscanner );
YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len , yyscan_t yyscanner );

void *yyalloc ( yy_size_t , yyscan_t yyscanner );
void *yyrealloc ( void *, yy_size_t , yyscan_t yyscanner );
Expand Down Expand Up @@ -501,7 +500,7 @@ FILE *yyget_out ( yyscan_t yyscanner );

void yyset_out ( FILE * _out_str , yyscan_t yyscanner );

yy_size_t yyget_leng ( yyscan_t yyscanner );
int yyget_leng ( yyscan_t yyscanner );

char *yyget_text ( yyscan_t yyscanner );

Expand Down Expand Up @@ -732,9 +731,9 @@ extern int yylex \
#undef yyTABLES_NAME
#endif

#line 130 "src/lexer.l"
#line 131 "src/lexer.l"


#line 738 "src/lexer.h"
#line 737 "src/lexer.h"
#undef jq_yyIN_HEADER
#endif /* jq_yyHEADER_H */
3 changes: 2 additions & 1 deletion src/lexer.l
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ struct lexer_param;
"catch" { return CATCH; }
"label" { return LABEL; }
"break" { return BREAK; }
"__loc__" { return LOC; }
"$__loc__" { return LOC; }
"|=" { return SETPIPE; }
"+=" { return SETPLUS; }
"-=" { return SETMINUS; }
Expand Down Expand Up @@ -122,6 +122,7 @@ struct lexer_param;

([a-zA-Z_][a-zA-Z_0-9]*::)*[a-zA-Z_][a-zA-Z_0-9]* { yylval->literal = jv_string(yytext); return IDENT;}
\.[a-zA-Z_][a-zA-Z_0-9]* { yylval->literal = jv_string(yytext+1); return FIELD;}
\$[a-zA-Z_][a-zA-Z_0-9]* { yylval->literal = jv_string(yytext+1); return VARIABLE;}

[ \n\t]+ {}

Expand Down
Loading

0 comments on commit 3426d32

Please sign in to comment.