Assertion Failure in Parsegenapp with supplied grammar #10

utahwithak · 2014-04-09T18:17:31Z

I have a grammar that I translated from:
http://www.sqlite.org/docsrc/doc/trunk/art/syntax/all-bnf.html

It is an extremely rough draft of the grammar but is complete. I took what was in the above site and converted it into an ANTLR valid grammar and was able to generate parsing code from the grammar so I know that the syntax and rule definitions are correct and syntactically valid. Other subtle gotchas are sure to abound. But this is not a request to fix the grammar but to provide it as an example of something that will hard crash the ParseGenApp.

again this is not asking for a grammar fix or anything like that (but you're welcome to as well) but just an a data set that crashes the app hard.

Thanks for the great tool!
If there is a better way to provide these types of bugs/issues let me know

parsegen grammar

//******************************************************************************
sqlstmtlist = sqlstmt  ( ';' ( sqlstmt )? )*;

databasename =  Word;
tablename = Word;
newtablename =  tablename;

tableorindexname = tablename;   

savepointname   =Word;
columnname  =Word;
indexname   =Word;
collationname   =Word;
name        =Word;
foreigntable    =Word;
triggername =Word;  
viewname    =Word;
modulename  =Word;
moduleargument  =Word;
initialselect   =Word;
recursiveselect =Word;
errormessage=Word;  
pragmaname= Word;   
columnalias=Word;
tablealias=Word;
functionname    =Word;

explainOption  =    ( 'EXPLAIN' ( 'QUERY' 'PLAN' )? )?;

sqlstmt =   explainOption actionStatement;

actionStatement 
        =  altertablestmt 
        | analyzestmt
        | attachstmt 
        | beginstmt 
        | commitstmt 
        | createindexstmt 
        | createtablestmt 
        | createtriggerstmt 
        | createviewstmt 
        | createvirtualtablestmt
        | deletestmt 
        | detachstmt 
        | dropindexstmt 
        | droptablestmt 
        | droptriggerstmt 
        | dropviewstmt 
        | insertstmt 
        | pragmastmt 
        | reindexstmt 
        | releasestmt 
        | rollbackstmt 
        | savepointstmt 
        | selectstmt 
        | updatestmt 
        | vacuumstmt 
        ;

altertablestmt  =   'ALTER' 'TABLE' ( databasename '.' )? tablename ( 'RENAME' 'TO' newtablename | 'ADD' ( 'COLUMN' )? columndef );

analyzestmt =   'ANALYZE' ( databasename | tableorindexname | databasename '.' tableorindexname )?;


attachstmt  =   'ATTACH' ( 'DATABASE' )? expr 'AS' databasename;


beginstmt   =   'BEGIN' ( 'DEFERRED' | 'IMMEDIATE' | 'EXCLUSIVE' )? ( 'TRANSACTION' )?;


commitstmt  =   ( 'COMMIT' | 'END' ) ( 'TRANSACTION' )?;


rollbackstmt    =   'ROLLBACK' ( 'TRANSACTION' )? ( 'TO' ( 'SAVEPOINT' )? savepointname )?;


savepointstmt   =   'SAVEPOINT' savepointname;


releasestmt =   'RELEASE' ( 'SAVEPOINT' )? savepointname;


createindexstmt = 'CREATE' ( 'UNIQUE' )? 'INDEX' ( 'IF' 'NOT' 'EXISTS' )? ( databasename '.' )? indexname 'ON' tablename '(' indexedcolumn ( ',' indexedcolumn )* ')' ( 'WHERE' expr )?;


indexedcolumn   =   columnname ( 'COLLATE' collationname )? ( 'ASC' | 'DESC' )?;


createtablestmt = 'CREATE' ( 'TEMP' | 'TEMPORARY' )? 'TABLE' ( 'IF' 'NOT' 'EXISTS' )? ( databasename '.' )? tablename ( '(' columndef ( ',' columndef )* ( ',' tableconstraint )* ')' ( 'WITHOUT' 'ROWWord' )? | 'AS' selectstmt );


columndef   =   columnname ( typename )? ( columnconstraint )*;

typename    =   name ( '(' signednumber ')' | '(' signednumber ',' signednumber ')' )?;

columnconstraint    =   ( 'CONSTRAINT' name )? ( 'PRIMARY' 'KEY' ( 'ASC' | 'DESC' )? conflictclause ( 'AUTOINCREMENT' )? | 'NOT' 'NULL' conflictclause | 'UNIQUE' conflictclause | 'CHECK' '(' expr ')' | 'DEFAULT' ( signednumber | literalvalue | '(' expr ')' ) | 'COLLATE' collationname | foreignkeyclause );

signednumber    =   ( '+' | '-' )? numericliteral;

tableconstraint =   ( 'CONSTRAINT' name )? ( ( 'PRIMARY' 'KEY' | 'UNIQUE' ) '(' indexedcolumn ( ',' indexedcolumn )* ')' conflictclause | 'CHECK' '(' expr ')' | 'FOREIGN' 'KEY' '(' columnname ( ',' columnname )* ')' foreignkeyclause );

foreignkeyclause    =   'REFERENCES' foreigntable ( '(' columnname ( ',' columnname )* ')' )?
                        ( ( 'ON' ( 'DELETE' | 'UPDATE' ) ( 'SET' 'NULL' | 'SET' 'DEFAULT' | 'CASCADE' | 'RESTRICT' | 'NO' 'ACTION' ) | 'MATCH' name ) )?
                        ( ( 'NOT' )? 'DEFERRABLE' ( 'INITIALLY' 'DEFERRED' | 'INITIALLY' 'IMMEDIATE' )? )?;

conflictclause  =   ( 'ON' 'CONFLICT' ( 'ROLLBACK' | 'ABORT' | 'FAIL' | 'IGNORE' | 'REPLACE' ) )?;

createtriggerstmt   =   'CREATE' ( 'TEMP' | 'TEMPORARY' )? 'TRIGGER' ( 'IF' 'NOT' 'EXISTS' )?
                        ( databasename '.' )? triggername ( 'BEFORE' | 'AFTER' | 'INSTEAD' 'OF' )?
                        ( 'DELETE' | 'INSERT' | 'UPDATE' ( 'OF' columnname ( ',' columnname )* )? ) 'ON' tablename
                        ( 'FOR' 'EACH' 'ROW' )? ( 'WHEN' expr )?
                        'BEGIN' ( updatestmt | insertstmt | deletestmt | selectstmt ) ';' 'END';

createviewstmt  = 'CREATE' ( 'TEMP' | 'TEMPORARY' )? 'VIEW' ( 'IF' 'NOT' 'EXISTS' )? ( databasename '.' )? viewname 'AS' selectstmt;

createvirtualtablestmt  = 'CREATE' 'VIRTUAL' 'TABLE' ( 'IF' 'NOT' 'EXISTS' )? ( databasename '.' )? tablename 'USING' modulename ( '(' moduleargument ( ',' moduleargument )* ')' )?;

withclause  =   'WITH' ( 'RECURSIVE' )? ctetablename 'AS' '(' selectstmt ')' ( ',' ctetablename 'AS' '(' selectstmt ')' )*;


ctetablename    =   tablename ( '(' columnname ( ',' columnname )* ')' )?;

recursivecte    =   ctetablename 'AS' '(' initialselect ( 'UNION' | 'UNION' 'ALL' ) recursiveselect ')';

commontableexpression   =   tablename ( '(' columnname ( ',' columnname )* ')' )? 'AS' '(' selectstmt ')';


deletestmt  =   ( withclause )? 'DELETE' 'FROM' qualifiedtablename ( 'WHERE' expr )? limitedDeleteStatement?;

limitedDeleteStatement  = ( ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )? 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?;

detachstmt  =   'DETACH' ( 'DATABASE' )? databasename;

dropindexstmt   =   'DROP' 'INDEX' ( 'IF' 'EXISTS' )? ( databasename '.' )? indexname;

droptablestmt   =   'DROP' 'TABLE' ( 'IF' 'EXISTS' )? ( databasename '.' )? tablename;

droptriggerstmt =   'DROP' 'TRIGGER' ( 'IF' 'EXISTS' )? ( databasename '.' )? triggername;

dropviewstmt    =   'DROP' 'VIEW' ( 'IF' 'EXISTS' )? ( databasename '.' )? viewname;
bindparameter   = ('?'Word? | '='Word);
unaryoperator  =    '-'|    '+'  |  '~' |  ' NOT';
binaryoperator =    '||' | '*' | '/' | '%' | '+' | '-' | '<<' | '>>' | '&' | '|' | '<' | '<=' | '>' | '>=' | '=' | '==' | '!=' | '<>' | 'IS' | ( 'IS' 'NOT') | 'IN' |  'LIKE'  | 'GLOB' | 'MATCH' | 'REGEXP' | 'AND' | 'OR';

expr        = requiredExp optionalExp*;
requiredExp =   (literalvalue 
        | bindparameter 
        | ( ( databasename '.' )? tablename '.' )? columnname 
        | unaryoperator expr 
        | functionname '(' ( ( 'DISTINCT' )? expr ( ',' expr )* | '*' )? ')' 
        | '(' expr ')' 
        | 'CAST' '(' expr 'AS' typename ')'
        | ( ( 'NOT' )? 'EXISTS' )? '(' selectstmt ')' 
        | 'CASE' ( expr )? 'WHEN' expr 'THEN' expr ( 'ELSE' expr )? 'END' 
        | raisefunction);
optionalExp =
        binaryoperator expr 
        | 'COLLATE' collationname 
        | ( 'NOT' )? ( 'LIKE' | 'GLOB' | 'REGEXP' | 'MATCH' ) expr ( 'ESCAPE' expr )? 
        | ( 'ISNULL' | 'NOTNULL' | 'NOT' 'NULL' ) 
        | 'IS' ( 'NOT' )? expr 
        | ( 'NOT' )? 'BETWEEN' expr 'AND' expr 
        | ( 'NOT' )? 'IN' ( '(' ( selectstmt | expr ( ',' expr )* )? ')' 
        | ( databasename '.' )? tablename )
        ;

raisefunction   =   'RAISE' '(' ( 'IGNORE' | ( 'ROLLBACK' | 'ABORT' | 'FAIL' ) ',' errormessage ) ')';

stringliteral  =  Word;

blobliteral  =('x'|'X')'\'' Word '\'';   

literalvalue    =   numericliteral | stringliteral| blobliteral | 'NULL' |  'CURRENT_TIME' |    'CURRENT_DATE' | 'CURRENT_TIMESTAMP';

digit   =   '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9';

decimalpoint = ('.'|',');

numericliteral  =   ( digit ( decimalpoint ( digit )* )? | decimalpoint digit ) ( 'E' ( '+' | '-' )? digit )?;

insertstmt  =   ( withclause )? ( 'INSERT' | 'REPLACE' | 'INSERT' 'OR' 'REPLACE' | 'INSERT' 'OR' 'ROLLBACK' | 'INSERT' 'OR' 'ABORT' | 'INSERT' 'OR' 'FAIL' | 'INSERT' 'OR' 'IGNORE' ) 'INTO' ( databasename '.' )? tablename ( '(' columnname ( ',' columnname )* ')' )? ( 'VALUES' '(' expr ( ',' expr )* ')' ( ',' '(' expr ( ',' expr )* ')' )* | selectstmt | 'DEFAULT' 'VALUES' );

pragmastmt  =   'PRAGMA' ( databasename '.' )? pragmaname ( '=' pragmavalue | '(' pragmavalue ')' )?;

pragmavalue =   signednumber| name | stringliteral;

reindexstmt =   'REINDEX' ( collationname | ( databasename '.' )? ( tablename | indexname ) )?;

selectstmt  =   ( 'WITH' ( 'RECURSIVE' )? commontableexpression ( ',' commontableexpression )* )?
                ( 'SELECT' ( 'DISTINCT' | 'ALL' )? resultcolumn ( ',' resultcolumn )*
                ( 'FROM' ( tableorsubquery ( ',' tableorsubquery )* | joinclause ) )?
                ( 'WHERE' expr )?
                ( 'GROUP' 'BY' expr ( ',' expr )* ( 'HAVING' expr )? )? | 'VALUES' '(' expr ( ',' expr )* ')' ( ',' '(' expr ( ',' expr )* ')' )* ) ( compoundoperator ( 'SELECT' ( 'DISTINCT' | 'ALL' )? resultcolumn ( ',' resultcolumn )*
                ( 'FROM' ( tableorsubquery ( ',' tableorsubquery )* | joinclause ) )?
                ( 'WHERE' expr )?
                ( 'GROUP' 'BY' expr ( ',' expr )* ( 'HAVING' expr )? )? | 'VALUES' '(' expr ( ',' expr )* ')' ( ',' '(' expr ( ',' expr )* ')' )* ) )*
                ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )?
                ( 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?
            ;

joinclause  =   tableorsubquery ( joinoperator tableorsubquery joinconstraint )?;

selectcore  =   'SELECT' ( 'DISTINCT' | 'ALL' )? resultcolumn ( ',' resultcolumn )*
                ( 'FROM' ( tableorsubquery ( ',' tableorsubquery )* | joinclause ) )?
                ( 'WHERE' expr )?
                ( 'GROUP' 'BY' expr ( ',' expr )* ( 'HAVING' expr )? )?
            |   'VALUES' '(' expr ( ',' expr )* ')' ( ',' '(' expr ( ',' expr )* ')' )*
            ;

factoredselectstmt  =   ( 'WITH' ( 'RECURSIVE' )? commontableexpression ( ',' commontableexpression )* )?
                        selectcore ( compoundoperator selectcore )*
                        ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )?
                        ( 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?
                    ;

simpleselectstmt    =   ( 'WITH' ( 'RECURSIVE' )? commontableexpression ( ',' commontableexpression )* )?
                        selectcore ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )?
                        ( 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?
                    ;

compoundselectstmt  =   ( 'WITH' ( 'RECURSIVE' )? commontableexpression ( ',' commontableexpression )* )?
                        selectcore ( 'UNION' | 'UNION' 'ALL' | 'INTERSECT' | 'EXCEPT' ) selectcore
                        ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )?
                        ( 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?
                    ;

tableorsubquery     =   ( databasename '.' )? tablename ( ( 'AS' )? tablealias )? ( 'INDEXED' 'BY' indexname | 'NOT' 'INDEXED' )?
                    |   '(' ( tableorsubquery ( ',' tableorsubquery )* | joinclause ) ')'
                    |   '(' selectstmt ')' ( ( 'AS' )? tablealias )?
                ;

resultcolumn        =   '*'
            |   tablename '.' '*'
            |   expr ( ( 'AS' )? columnalias )?
            ;

joinoperator        =   ','
                |   ( 'NATURAL' )? ( 'LEFT' ( 'OUTER' )? | 'INNER' | 'CROSS' )? 'JOIN'
                ;

joinconstraint      =   ( 'ON' expr | 'USING' '(' columnname ( ',' columnname )* ')' )?;

orderingterm        =   expr ( 'COLLATE' collationname )? ( 'ASC' | 'DESC' )?;

compoundoperator    =   'UNION'|'UNION' 'ALL'| 'INTERSECT'| 'EXCEPT';

updatestmt      =   ( withclause )? 'UPDATE' ( 'OR' 'ROLLBACK' | 'OR' 'ABORT' | 'OR' 'REPLACE' | 'OR' 'FAIL' | 'OR' 'IGNORE' )? qualifiedtablename
                'SET' columnname '=' expr ( ',' columnname '=' expr )* ( 'WHERE' expr )?   limitedupdate?   ;

limitedupdate   =( ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )? 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?;

qualifiedtablename  =   ( databasename '.' )? tablename ( 'INDEXED' 'BY' indexname | 'NOT' 'INDEXED' )?;

vacuumstmt  =   'VACUUM';

//commentsyntax =    ( anythingexceptnewline )* ( newline | endofinput )
//commentsyntax =   /* ( anythingexcept*/ )* ( */ | endofinput )

Here is the ANTLR grammar that is successful:
(all ':' converted to '=' and ID converted to Word)

grammar sqlitecomplete;
options{backtrack=true;}
ID  :   ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
    ;


start : sqlstmtlist;

databasename
    :   ID;
tablename
    :   ID;
newtablename :  tablename;

tableorindexname : tablename;   

savepointname   :ID;
columnname  :ID;
indexname   :ID;
collationname   :ID;
name        :ID;
foreigntable    :ID;
triggername :ID;    
viewname    :ID;
modulename  :ID;
moduleargument  :ID;
initialselect   :ID;
recursiveselect :ID;
errormessage:ID;    
pragmaname: ID; 
columnalias:ID;
tablealias:ID;
functionname    :ID;

sqlstmtlist : sqlstmt  ( ';' ( sqlstmt )? )*;
explainOption 
    :   ( 'EXPLAIN' ( 'QUERY' 'PLAN' )? )?;
sqlstmt :   explainOption actionStatement;
actionStatement 
        :  altertablestmt 
        | analyzestmt
        | attachstmt 
        | beginstmt 
        | commitstmt 
        | createindexstmt 
        | createtablestmt 
        | createtriggerstmt 
        | createviewstmt 
        | createvirtualtablestmt
        | deletestmt 
        | detachstmt 
        | dropindexstmt 
        | droptablestmt 
        | droptriggerstmt 
        | dropviewstmt 
        | insertstmt 
        | pragmastmt 
        | reindexstmt 
        | releasestmt 
        | rollbackstmt 
        | savepointstmt 
        | selectstmt 
        | updatestmt 
        | vacuumstmt 
        ;

altertablestmt  :   'ALTER' 'TABLE' ( databasename '.' )? tablename ( 'RENAME' 'TO' newtablename | 'ADD' ( 'COLUMN' )? columndef );

analyzestmt :   'ANALYZE' ( databasename | tableorindexname | databasename '.' tableorindexname )?;


attachstmt  :   'ATTACH' ( 'DATABASE' )? expr 'AS' databasename;


beginstmt   :   'BEGIN' ( 'DEFERRED' | 'IMMEDIATE' | 'EXCLUSIVE' )? ( 'TRANSACTION' )?;


commitstmt  :   ( 'COMMIT' | 'END' ) ( 'TRANSACTION' )?;


rollbackstmt    :   'ROLLBACK' ( 'TRANSACTION' )? ( 'TO' ( 'SAVEPOINT' )? savepointname )?;


savepointstmt   :   'SAVEPOINT' savepointname;


releasestmt :   'RELEASE' ( 'SAVEPOINT' )? savepointname;


createindexstmt : 'CREATE' ( 'UNIQUE' )? 'INDEX' ( 'IF' 'NOT' 'EXISTS' )? ( databasename '.' )? indexname 'ON' tablename '(' indexedcolumn ( ',' indexedcolumn )* ')' ( 'WHERE' expr )?;


indexedcolumn   :   columnname ( 'COLLATE' collationname )? ( 'ASC' | 'DESC' )?;


createtablestmt : 'CREATE' ( 'TEMP' | 'TEMPORARY' )? 'TABLE' ( 'IF' 'NOT' 'EXISTS' )? ( databasename '.' )? tablename ( '(' columndef ( ',' columndef )* ( ',' tableconstraint )* ')' ( 'WITHOUT' 'ROWID' )? | 'AS' selectstmt );


columndef   :   columnname ( typename )? ( columnconstraint )*;

typename    :   name ( '(' signednumber ')' | '(' signednumber ',' signednumber ')' )?;

columnconstraint    :   ( 'CONSTRAINT' name )?
( 'PRIMARY' 'KEY' ( 'ASC' | 'DESC' )? conflictclause ( 'AUTOINCREMENT' )? | 'NOT' 'NULL' conflictclause | 'UNIQUE' conflictclause | 'CHECK' '(' expr ')' | 'DEFAULT' ( signednumber | literalvalue | '(' expr ')' ) | 'COLLATE' collationname | foreignkeyclause );

signednumber    :   ( '+' | '-' )? numericliteral;

tableconstraint :   ( 'CONSTRAINT' name )? ( ( 'PRIMARY' 'KEY' | 'UNIQUE' ) '(' indexedcolumn ( ',' indexedcolumn )* ')' conflictclause | 'CHECK' '(' expr ')' | 'FOREIGN' 'KEY' '(' columnname ( ',' columnname )* ')' foreignkeyclause );

foreignkeyclause    :   'REFERENCES' foreigntable ( '(' columnname ( ',' columnname )* ')' )?
( ( 'ON' ( 'DELETE' | 'UPDATE' ) ( 'SET' 'NULL' | 'SET' 'DEFAULT' | 'CASCADE' | 'RESTRICT' | 'NO' 'ACTION' ) | 'MATCH' name ) )?
( ( 'NOT' )? 'DEFERRABLE' ( 'INITIALLY' 'DEFERRED' | 'INITIALLY' 'IMMEDIATE' )? )?;

conflictclause  :   ( 'ON' 'CONFLICT' ( 'ROLLBACK' | 'ABORT' | 'FAIL' | 'IGNORE' | 'REPLACE' ) )?;

createtriggerstmt   : 'CREATE' ( 'TEMP' | 'TEMPORARY' )? 'TRIGGER' ( 'IF' 'NOT' 'EXISTS' )?
( databasename '.' )? triggername ( 'BEFORE' | 'AFTER' | 'INSTEAD' 'OF' )?
( 'DELETE' | 'INSERT' | 'UPDATE' ( 'OF' columnname ( ',' columnname )* )? ) 'ON' tablename
( 'FOR' 'EACH' 'ROW' )? ( 'WHEN' expr )?
'BEGIN' ( updatestmt | insertstmt | deletestmt | selectstmt ) ';' 'END';

createviewstmt  : 'CREATE' ( 'TEMP' | 'TEMPORARY' )? 'VIEW' ( 'IF' 'NOT' 'EXISTS' )? ( databasename '.' )? viewname 'AS' selectstmt;

createvirtualtablestmt  : 'CREATE' 'VIRTUAL' 'TABLE' ( 'IF' 'NOT' 'EXISTS' )? ( databasename '.' )? tablename 'USING' modulename ( '(' moduleargument ( ',' moduleargument )* ')' )?;

withclause  :   'WITH' ( 'RECURSIVE' )? ctetablename 'AS' '(' selectstmt ')' ( ',' ctetablename 'AS' '(' selectstmt ')' )*;


ctetablename    :   tablename ( '(' columnname ( ',' columnname )* ')' )?;

recursivecte    :   ctetablename 'AS' '(' initialselect ( 'UNION' | 'UNION' 'ALL' ) recursiveselect ')';

commontableexpression   :   tablename ( '(' columnname ( ',' columnname )* ')' )? 'AS' '(' selectstmt ')';


deletestmt  :   ( withclause )? 'DELETE' 'FROM' qualifiedtablename ( 'WHERE' expr )? limitedDeleteStatement?;

limitedDeleteStatement  : ( ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )? 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?;

detachstmt  :   'DETACH' ( 'DATABASE' )? databasename;

dropindexstmt   :   'DROP' 'INDEX' ( 'IF' 'EXISTS' )? ( databasename '.' )? indexname;

droptablestmt   :   'DROP' 'TABLE' ( 'IF' 'EXISTS' )? ( databasename '.' )? tablename;

droptriggerstmt :   'DROP' 'TRIGGER' ( 'IF' 'EXISTS' )? ( databasename '.' )? triggername;

dropviewstmt    :   'DROP' 'VIEW' ( 'IF' 'EXISTS' )? ( databasename '.' )? viewname;
bindparameter   : ('?'ID? | ':'ID);
unaryoperator  :    '-'|    '+'  |  '~' |  ' NOT';
binaryoperator :    '||' | '*' | '/' | '%' | '+' | '-' | '<<' | '>>' | '&' | '|' | '<' | '<=' | '>' | '>=' | '=' | '==' | '!=' | '<>' | 'IS' | ( 'IS' 'NOT') | 'IN' |  'LIKE'  | 'GLOB' | 'MATCH' | 'REGEXP' | 'AND' | 'OR';

expr        : requiredExp optionalExp*;
requiredExp :   (literalvalue 
        | bindparameter 
        | ( ( databasename '.' )? tablename '.' )? columnname 
        | unaryoperator expr 
        | functionname '(' ( ( 'DISTINCT' )? expr ( ',' expr )* | '*' )? ')' 
        | '(' expr ')' 
        | 'CAST' '(' expr 'AS' typename ')'
        | ( ( 'NOT' )? 'EXISTS' )? '(' selectstmt ')' 
        | 'CASE' ( expr )? 'WHEN' expr 'THEN' expr ( 'ELSE' expr )? 'END' 
        | raisefunction);
optionalExp :
        binaryoperator expr 
        | 'COLLATE' collationname 
        | ( 'NOT' )? ( 'LIKE' | 'GLOB' | 'REGEXP' | 'MATCH' ) expr ( 'ESCAPE' expr )? 
        | ( 'ISNULL' | 'NOTNULL' | 'NOT' 'NULL' ) 
        | 'IS' ( 'NOT' )? expr 
        | ( 'NOT' )? 'BETWEEN' expr 'AND' expr 
        | ( 'NOT' )? 'IN' ( '(' ( selectstmt | expr ( ',' expr )* )? ')' 
        | ( databasename '.' )? tablename )
        ;

raisefunction   :   'RAISE' '(' ( 'IGNORE' | ( 'ROLLBACK' | 'ABORT' | 'FAIL' ) ',' errormessage ) ')';

stringliteral  :  ID;

blobliteral  :('x'|'X')'\'' ID '\'';     

literalvalue    :   numericliteral | stringliteral| blobliteral | 'NULL' |  'CURRENT_TIME' |    'CURRENT_DATE' | 'CURRENT_TIMESTAMP';

digit   :   '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9';

decimalpoint : ('.'|',');

numericliteral  :   ( digit ( decimalpoint ( digit )* )? | decimalpoint digit ) ( 'E' ( '+' | '-' )? digit )?;

insertstmt  :   ( withclause )? ( 'INSERT' | 'REPLACE' | 'INSERT' 'OR' 'REPLACE' | 'INSERT' 'OR' 'ROLLBACK' | 'INSERT' 'OR' 'ABORT' | 'INSERT' 'OR' 'FAIL' | 'INSERT' 'OR' 'IGNORE' ) 'INTO' ( databasename '.' )? tablename ( '(' columnname ( ',' columnname )* ')' )? ( 'VALUES' '(' expr ( ',' expr )* ')' ( ',' '(' expr ( ',' expr )* ')' )* | selectstmt | 'DEFAULT' 'VALUES' );

pragmastmt  :   'PRAGMA' ( databasename '.' )? pragmaname ( '=' pragmavalue | '(' pragmavalue ')' )?;

pragmavalue :   signednumber| name | stringliteral;

reindexstmt :   'REINDEX' ( collationname | ( databasename '.' )? ( tablename | indexname ) )?;

selectstmt  :   ( 'WITH' ( 'RECURSIVE' )? commontableexpression ( ',' commontableexpression )* )?
                ( 'SELECT' ( 'DISTINCT' | 'ALL' )? resultcolumn ( ',' resultcolumn )*
                ( 'FROM' ( tableorsubquery ( ',' tableorsubquery )* | joinclause ) )?
                ( 'WHERE' expr )?
                ( 'GROUP' 'BY' expr ( ',' expr )* ( 'HAVING' expr )? )? | 'VALUES' '(' expr ( ',' expr )* ')' ( ',' '(' expr ( ',' expr )* ')' )* ) ( compoundoperator ( 'SELECT' ( 'DISTINCT' | 'ALL' )? resultcolumn ( ',' resultcolumn )*
                ( 'FROM' ( tableorsubquery ( ',' tableorsubquery )* | joinclause ) )?
                ( 'WHERE' expr )?
                ( 'GROUP' 'BY' expr ( ',' expr )* ( 'HAVING' expr )? )? | 'VALUES' '(' expr ( ',' expr )* ')' ( ',' '(' expr ( ',' expr )* ')' )* ) )*
                ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )?
                ( 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?
            ;

joinclause  :   tableorsubquery ( joinoperator tableorsubquery joinconstraint )?;

selectcore  :   'SELECT' ( 'DISTINCT' | 'ALL' )? resultcolumn ( ',' resultcolumn )*
                ( 'FROM' ( tableorsubquery ( ',' tableorsubquery )* | joinclause ) )?
                ( 'WHERE' expr )?
                ( 'GROUP' 'BY' expr ( ',' expr )* ( 'HAVING' expr )? )?
            |   'VALUES' '(' expr ( ',' expr )* ')' ( ',' '(' expr ( ',' expr )* ')' )*
            ;

factoredselectstmt  :   ( 'WITH' ( 'RECURSIVE' )? commontableexpression ( ',' commontableexpression )* )?
                        selectcore ( compoundoperator selectcore )*
                        ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )?
                        ( 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?
                    ;

simpleselectstmt    :   ( 'WITH' ( 'RECURSIVE' )? commontableexpression ( ',' commontableexpression )* )?
                        selectcore ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )?
                        ( 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?
                    ;

compoundselectstmt  :   ( 'WITH' ( 'RECURSIVE' )? commontableexpression ( ',' commontableexpression )* )?
                        selectcore ( 'UNION' | 'UNION' 'ALL' | 'INTERSECT' | 'EXCEPT' ) selectcore
                        ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )?
                        ( 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?
                    ;

tableorsubquery     :   ( databasename '.' )? tablename ( ( 'AS' )? tablealias )? ( 'INDEXED' 'BY' indexname | 'NOT' 'INDEXED' )?
                |   '(' ( tableorsubquery ( ',' tableorsubquery )* | joinclause ) ')'
                |   '(' selectstmt ')' ( ( 'AS' )? tablealias )?
                ;

resultcolumn        :   '*'
            |   tablename '.' '*'
            |   expr ( ( 'AS' )? columnalias )?
            ;

joinoperator        :   ','
                |   ( 'NATURAL' )? ( 'LEFT' ( 'OUTER' )? | 'INNER' | 'CROSS' )? 'JOIN'
                ;

joinconstraint      :   ( 'ON' expr | 'USING' '(' columnname ( ',' columnname )* ')' )?;

orderingterm        :   expr ( 'COLLATE' collationname )? ( 'ASC' | 'DESC' )?;

compoundoperator    :   'UNION'|'UNION' 'ALL'| 'INTERSECT'| 'EXCEPT';

updatestmt      :   ( withclause )? 'UPDATE' ( 'OR' 'ROLLBACK' | 'OR' 'ABORT' | 'OR' 'REPLACE' | 'OR' 'FAIL' | 'OR' 'IGNORE' )? qualifiedtablename
                'SET' columnname '=' expr ( ',' columnname '=' expr )* ( 'WHERE' expr )?   limitedupdate?   ;

limitedupdate   :( ( 'ORDER' 'BY' orderingterm ( ',' orderingterm )* )? 'LIMIT' expr ( ( 'OFFSET' | ',' ) expr )? )?;

qualifiedtablename  :   ( databasename '.' )? tablename ( 'INDEXED' 'BY' indexname | 'NOT' 'INDEXED' )?;

vacuumstmt  :   'VACUUM';

//commentsyntax :    ( anythingexceptnewline )* ( newline | endofinput )
//commentsyntax :   /* ( anythingexcept*/ )* ( */ | endofinput )

The text was updated successfully, but these errors were encountered:

itod · 2014-04-09T19:53:21Z

Thanks! Only had time to glance at this so far...

The problem is apparently only an Assertion Failure in Debug builds of the GenApp.

The GenApp is choking on the optionalExp rule. This line:

    // nested Alts should always be on the lhs. never on rhs.
    NSAssert(PGNodeTypeAlternation != [(PGBaseNode *)node.children[1] type], @"");

In -[PGDefinitionPhaseVisitor visitAlternation:]

This is the tree representation of the optionalExp AST node after parsing:

(| (| (| (. #binaryoperator #expr) (. 'COLLATE' #collationname)) (. (? 'NOT') (| (| (| 'LIKE' 'GLOB') 'REGEXP') 'MATCH') #expr (? (. 'ESCAPE' #expr)))) (| (| 'ISNULL' 'NOTNULL') (. 'NOT' 'NULL')) (. 'IS' (? 'NOT') #expr) (. (? 'NOT') 'BETWEEN' #expr 'AND' #expr) (. (? 'NOT') 'IN' (| (. '(' (? (| #selectstmt (. #expr (* (. ',' #expr))))) ')') (. (? (. #databasename '.')) #tablename))))

utahwithak · 2014-04-09T20:02:10Z

in my nearly nonexistent grammar knowledge if you could explain what that comment means that would be very helpful.

the way I believe it is set up looks like:

A = B C*
B = some stuff possibly stuff A
C = more optional stuff and possibly A

As a small background, the expr is the biggest headache i?ve had.
it was translated from :

expr ::=
expr ::=
expr ::= [ [ database-name . ] table-name . ] column-name
expr ::= unary-operator
expr ::= binary-operator
expr ::= function-name ( [ [ DISTINCT ] [ , ]* | * ] )
expr ::= ( )
expr ::= CAST ( AS )
expr ::= COLLATE collation-name
expr ::= [ NOT ] [ LIKE | GLOB | REGEXP | MATCH ]1 [ ESCAPE ]
expr ::= [ ISNULL | NOTNULL | NOT NULL ]1
expr ::= IS [ NOT ]
expr ::= [ NOT ] BETWEEN AND
expr ::= [ NOT ] IN [ ( [ | [ , ]* ] ) | [ database-name . ] table-name ]1
expr ::= [ NOT ] EXISTS
expr ::= CASE [ ] WHEN THEN [ ELSE ] END
expr ::=

which is 100% absolutely the definition of left recursion. I attempted my self to remove it and failed miserably. So I converted it to the ANTLR grammar that would (potentially) remove it for me. It did remove the left recursion but certainly made the rule much more ugly

expr        :   (literalvalue 
            | bindparameter 
            | ( ( databasename '.' )? tablename '.' )? columnname 
            | unaryoperator expr 
            | functionname '(' ( ( 'DISTINCT' )? expr ( ',' expr )* | * )? ')' 
            | '(' expr ')' 
            | 'CAST' '(' expr 'AS' typename ')'
            | ( ( 'NOT' )? 'EXISTS' )? '(' selectstmt ')' 
            | 'CASE' ( expr )? 'WHEN' expr 'THEN' expr ( 'ELSE' expr )? 'END' 
            | raisefunction) 

            (binaryoperator expr 
            | 'COLLATE' collationname 
            | ( 'NOT' )? ( 'LIKE' | 'GLOB' | 'REGEXP' | 'MATCH' ) expr ( 'ESCAPE' expr )? 
            | ( 'ISNULL' | 'NOTNULL' | 'NOT' 'NULL' ) 
            | 'IS' ( 'NOT' )? expr 
            | ( 'NOT' )? 'BETWEEN' expr 'AND' expr 
            | ( 'NOT' )? 'IN' ( '(' ( selectstmt | expr ( ',' expr )* )? ')' 
            | ( databasename '.' )? tablename )
            )* ;

Thanks again

On Apr 9, 2014, at 12:53 PM, Todd Ditchendorf notifications@github.com wrote:

// nested Alts should always be on the lhs. never on rhs.
NSAssert(PGNodeTypeAlternation != [(PGBaseNode *)node.children[1] type], @"");

In -[PGDefinitionPhaseVisitor visitAlternation:]

itod · 2014-04-09T20:10:34Z

My comment was more for my own reference when I have time to look more closely into this :).

3 things:

Your descriptions and fixes of left recursion look correct to me on quick glance. :)
I really don't think anyone but me will be able to fix this GenApp bug. It requires deep understanding of how PEGKit is implemented (undocumented).
In the meantime, you might try running your grammar thru a RELEASE build of the GenApp. It runs cleanly, but I don't know if the produced parser code is valid... Might be.

utahwithak · 2014-04-09T20:17:09Z

great. thanks for your support.

On Apr 9, 2014, at 1:10 PM, Todd Ditchendorf notifications@github.com wrote:

My comment was more for my own reference when I have time to look more closely into this :).

3 things:

Your descriptions and fixes of left recursion look correct to me on quick glance. :)
I really don't think anyone but me will be able to fix this GenApp bug. It requires deep understanding of how PEGKit is implemented (undocumented).
In the meantime, you might try running your grammar thru a RELEASE build of the GenApp. It runs cleanly, but I don't know if the produced parser code is valid... Might be.
—
Reply to this email directly or view it on GitHub.

itod · 2014-04-10T03:10:26Z

BTW, I remembered that I had tried my hand at the sqlite grammar in the ParseKit days:

https://github.com/itod/parsekit/blob/master/res/sqlite.grammar

Looks like I never finished...

utahwithak · 2014-04-10T03:15:11Z

I noticed that in your test cases and poked around a bit. I too looked at their nice diagrams that are so readily available but trying to go from diagram and nested diagrams to code was much harder for me than from their file that generates it… 95% of the work was already done with all the rules written out. I’m hope I can get it to work as it would be a very nice asset for the app i’m working on.

Really enjoying working with grammars again. Last time was a compilers course where we used the java book you recommend. quite a different grammar than static java that we worked on.

Thanks again for the great tool.

On Apr 9, 2014, at 8:10 PM, Todd Ditchendorf notifications@github.com wrote:

BTW, I remembered that I had tried my hand at the sqlite grammar in the ParseKit days:

https://github.com/itod/parsekit/blob/master/res/sqlite.grammar

Looks like I never finished...

—
Reply to this email directly or view it on GitHub.

utahwithak · 2014-04-14T02:54:20Z

Probably already got this far, i just had a chance to work on the grammar again and was able to change the expr rule to be :

expr        = requiredExp optionalExp*;

requiredExp =   (literalvalue 
        | bindparameter 
        | ( dbName tablename '.' )? columnname 
        | unaryoperator expr 
        | functionname '(' ( ( 'DISTINCT' )? expr ( ',' expr )* | '*' )? ')' 
        | '(' expr ')' 
        | 'CAST' '(' expr 'AS' typename ')'
        | ( ( 'NOT' )? 'EXISTS' )? '(' selectstmt ')' 
        | 'CASE' ( expr )? 'WHEN' expr 'THEN' expr ( 'ELSE' expr )? 'END' 
        | raisefunction);

optionalExp = binaryoperator expr  | 'COLLATE' collationname  | notExp  | nullExpr  | 'IS' ( 'NOT' )? expr ;

nullExpr = 'ISNULL' | 'NOTNULL' | ('NOT' 'NULL');

notExp = ('NOT')? likeGlobRegexpMatchExp | betweenExp | inExpr;

betweenExp = 'BETWEEN' expr 'AND' expr ;

likeGlobRegexpMatchExp = ( 'LIKE' | 'GLOB' | 'REGEXP' | 'MATCH' ) expr ( 'ESCAPE' expr )?;

inExpr = 'IN' ( '(' ( selectstmt | expr ( ',' expr )* )? ')' | dbName tablename );

this generates the parser just fine and I don't think it changed the language definition.

itod added the bug label Apr 9, 2014

itod changed the title ~~Hard crash in Parsegenapp with supplied grammar~~ Assertion Failure in Parsegenapp with supplied grammar Apr 10, 2014

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Assertion Failure in Parsegenapp with supplied grammar #10

Assertion Failure in Parsegenapp with supplied grammar #10

utahwithak commented Apr 9, 2014

itod commented Apr 9, 2014

utahwithak commented Apr 9, 2014

itod commented Apr 9, 2014

utahwithak commented Apr 9, 2014

itod commented Apr 10, 2014

utahwithak commented Apr 10, 2014

utahwithak commented Apr 14, 2014

Assertion Failure in Parsegenapp with supplied grammar #10

Assertion Failure in Parsegenapp with supplied grammar #10

Comments

utahwithak commented Apr 9, 2014

itod commented Apr 9, 2014

utahwithak commented Apr 9, 2014

itod commented Apr 9, 2014

utahwithak commented Apr 9, 2014

itod commented Apr 10, 2014

utahwithak commented Apr 10, 2014

utahwithak commented Apr 14, 2014