Skip to content

Commit

Permalink
add the JSONParser (#145)
Browse files Browse the repository at this point in the history 
add the JSONParser to convert the JSON file to `Query`
  • Loading branch information
@JavierJia
JavierJia authored Jul 28, 2016
1 parent 3c4f44d commit 744f9a1
Show file tree
Hide file tree
Showing 12 changed files with 367 additions and 81 deletions.
1 change: 1 addition & 0 deletions zion/src/main/resources/request/fullRequest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
]
}
],
"unnest" : {"hashtags": "hashtag"},
"group": {
"by": [
{
Expand Down
11 changes: 5 additions & 6 deletions zion/src/main/resources/request/hashtag.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
"value" : ["zika", "virus"]
}
],
"unnest" : [
{ "hashTags": "hashTag"}
],
"unnest" : { "hashTags": "hashTag"},
"group": {
"by": [
{
Expand All @@ -29,15 +27,16 @@
"aggregate": [
{
"field" : "*",
"apply" : "count",
"apply" : {
"name": "count"
},
"as" : "count"
}
]
},
"select" : {
"order" : [ "-count"],
"limit": 100,
"offset" : 0,
"field": ["hashTag", "count"]
"offset" : 0
}
}
17 changes: 12 additions & 5 deletions zion/src/main/resources/request/mapTimeCube.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,19 @@
"by": [
{
"field": "geo",
"apply": "state",
"apply": {
"name": "level",
"args": {
"level": "state"
}
},
"as": "state"
},
{
"field": "create_at",
"apply": {
"name": "interval",
"args" : {
"args": {
"unit": "hour"
}
},
Expand All @@ -47,15 +52,17 @@
"aggregate": [
{
"field": "*",
"apply": "count",
"apply": {
"name": "count"
},
"as": "count"
},
{
"field": "hashTags",
"apply": {
"name": "topK",
"args": {
"k" : 10
"k": 10
}
},
"as": "top10_hash"
Expand All @@ -65,7 +72,7 @@
"apply": {
"name": "topK",
"args": {
"k" : 10
"k": 10
}
},
"as": "top10_lang"
Expand Down
1 change: 1 addition & 0 deletions zion/src/main/scala/edu/uci/ics/cloudberry/zion/model/datastore/CherryException.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ case class QueryInitException(msg: String) extends CherryException(msg, null)

class QueryParsingException(msg: String) extends CherryException(msg, null)

//TODO change to BADRequest response
case class JsonRequestException(msg: String) extends CherryException(msg, null)

case class FieldNotFound(fieldName: String) extends QueryParsingException(s"cannot find field $fieldName")
38 changes: 16 additions & 22 deletions zion/src/main/scala/edu/uci/ics/cloudberry/zion/model/impl/AQLFuncVisitor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ object AQLFuncVisitor {
values: Seq[Any]
): String = {
//TODO add the function handling logic
if (!Schema.Type2Relations.get(field.dataType).get.contains(relation)) {
if (!Schema.Type2Relations(field.dataType).contains(relation)) {
throw new QueryParsingException(s"field ${field.name} of type ${field.dataType} can not apply to relation: ${relation}")
}
field.dataType match {
Expand Down Expand Up @@ -51,23 +51,18 @@ object AQLFuncVisitor {
relation: Relation,
values: Seq[AnyVal]): String = {
relation match {
case Relation.inRange => {
if (values.size != 2) throw new QueryParsingException(s"relation: ${relation} require two parameters")
return s"$aqlExpr >= ${values(0)} and $aqlExpr < ${values(1)}"
}
case Relation.in => {
case Relation.inRange =>
if (values.size != 2) throw new QueryParsingException(s"relation: $relation require two parameters")
s"$aqlExpr >= ${values(0)} and $aqlExpr < ${values(1)}"
case Relation.in =>
val setVar = s"$$set${field.name.replace('.', '_')}"
val ret =
s"""|true
|for $setVar in [ ${values.mkString(",")} ]
|where $aqlExpr = $setVar
|""".stripMargin
return ret
}
case _ => {
if (values.size != 1) throw new QueryParsingException(s"relation: ${relation} require one parameter")
return s"$aqlExpr $relation ${values.head}"
}
s"""|true
|for $setVar in [ ${values.mkString(",")} ]
|where $aqlExpr = $setVar
|""".stripMargin
case _ =>
if (values.size != 1) throw new QueryParsingException(s"relation: $relation require one parameter")
s"$aqlExpr $relation ${values.head}"
}
}

Expand All @@ -78,12 +73,12 @@ object AQLFuncVisitor {
values: Seq[String]): String = {
relation match {
case Relation.inRange => {
if (values.size != 2) throw new QueryParsingException(s"relation: ${relation} require two parameters")
if (values.size != 2) throw new QueryParsingException(s"relation: $relation require two parameters")
return s"$aqlExpr >= datetime('${values(0)}') " +
s"and $aqlExpr < datetime('${values(1)}')"
}
case _ => {
if (values.size != 1) throw new QueryParsingException(s"relation: ${relation} require one parameter")
if (values.size != 1) throw new QueryParsingException(s"relation: $relation require one parameter")
s"$aqlExpr $relation datetime('${values.head}')"
}
}
Expand Down Expand Up @@ -120,7 +115,7 @@ object AQLFuncVisitor {

private def validateNumberValue(relation: Relation, values: Seq[Any]): Unit = {
if (!values.forall(isAnyNumber)) {
throw new QueryParsingException(s"values contain non compatible data type for relation: ${relation}")
throw new QueryParsingException(s"values contain non compatible data type for relation: $relation")
}
}

Expand Down Expand Up @@ -199,10 +194,9 @@ object AQLFuncVisitor {
aqlExpr: String
): (DataType.DataType, String) = {
func match {
case Count => {
case Count =>
if (field.dataType != DataType.Record) throw new QueryParsingException("count requires to aggregate on the record bag")
(DataType.Number, s"count($aqlExpr)")
}
case Max => ???
case Min => ???
case topK: TopK => ???
Expand Down
21 changes: 9 additions & 12 deletions zion/src/main/scala/edu/uci/ics/cloudberry/zion/model/impl/AQLQueryParser.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ import scala.collection.mutable

class AQLQueryParser extends IQueryParser {

case class AQLVar(val field: Field, val aqlExpr: String)

override def parse(query: Query, schema: Schema): Seq[String] = {

validateQuery(query)
Expand Down Expand Up @@ -87,7 +85,7 @@ class AQLQueryParser extends IQueryParser {
val producedVar = mutable.Map.newBuilder[String, AQLVar]
val aql = unnest.zipWithIndex.map { case (stat, id) =>
varMap.get(stat.fieldName) match {
case Some(aqlVar) => {
case Some(aqlVar) =>
aqlVar.field match {
case field: BagField =>
val newVar = s"$$unnest$id"
Expand All @@ -99,7 +97,6 @@ class AQLQueryParser extends IQueryParser {
|""".stripMargin
case _ => throw new QueryParsingException("unnest can only apply on Bag type")
}
}
case None => throw FieldNotFound(stat.fieldName)
}
}.mkString("\n")
Expand All @@ -113,13 +110,12 @@ class AQLQueryParser extends IQueryParser {
val producedVar = mutable.Map.newBuilder[String, AQLVar]
val groupByAQLPair: Seq[(String, String)] = group.bys.zipWithIndex.map { case (by, id) =>
varMap.get(by.fieldName) match {
case Some(aqlVar) => {
case Some(aqlVar) =>
val key = by.as.getOrElse(aqlVar.field.name)
val varKey = s"$$g$id"
val (dataType, aqlGrpFunc) = AQLFuncVisitor.translateGroupFunc(aqlVar.field, by.funcOpt, aqlVar.aqlExpr)
producedVar += key -> AQLVar(new Field(key, dataType), s"$varGroupSource.$key")
(s"$varKey := ${aqlGrpFunc}", s" '$key' : $varKey")
}
(s"$varKey := $aqlGrpFunc", s" '$key' : $varKey")
case None => throw FieldNotFound(by.fieldName)
}
}
Expand All @@ -128,12 +124,11 @@ class AQLQueryParser extends IQueryParser {
val aggrRequiredVar = mutable.Seq.newBuilder[String]
val aggrNameMap = group.aggregates.map { aggr =>
varMap.get(aggr.fieldName) match {
case Some(aqlVar) => {
case Some(aqlVar) =>
aggrRequiredVar += aqlVar.aqlExpr.split('.')(0)
val (dataType, aqlAggExpr) = AQLFuncVisitor.translateAggrFunc(aqlVar.field, aggr.func, aqlVar.aqlExpr)
producedVar += aggr.as -> AQLVar(new Field(aggr.as, dataType), s"$varGroupSource.${aggr.as}")
s"'${aggr.as}' : $aqlAggExpr"
}
case None => throw FieldNotFound(aggr.fieldName)
}
}
Expand All @@ -157,7 +152,7 @@ class AQLQueryParser extends IQueryParser {
sourceVar: String = "$g"
): (String, String) = {

val (prefix, wrap) = if (isInGroup) (s"for ${sourceVar} in (", ")") else ("", "")
val (prefix, wrap) = if (isInGroup) (s"for $sourceVar in (", ")") else ("", "")
//sampling only
val orders = select.orderOn.map { fieldNameWithOrder =>
val order = if (fieldNameWithOrder.startsWith("-")) "desc" else ""
Expand All @@ -167,15 +162,15 @@ class AQLQueryParser extends IQueryParser {
case None => throw FieldNotFound(fieldName)
}
}
val ordersAQL = if (orders.size > 0) orders.mkString("order by ", ",", "") else ""
val ordersAQL = if (orders.nonEmpty) orders.mkString("order by ", ",", "") else ""

val rets = select.fields.map { fieldName =>
varMap.get(fieldName) match {
case Some(aqlVar) => s" '${aqlVar.field.name}': ${aqlVar.aqlExpr}"
case None => throw FieldNotFound(fieldName)
}
}
val retAQL = if (rets.size > 0) rets.mkString("{", ",", "}") else sourceVar
val retAQL = if (rets.nonEmpty) rets.mkString("{", ",", "}") else sourceVar

val aql =
s"""
Expand All @@ -193,4 +188,6 @@ class AQLQueryParser extends IQueryParser {
requireOrThrow(query.select.isDefined || query.groups.isDefined, "either group or select statement is required")
}

case class AQLVar(field: Field, aqlExpr: String)

}
Loading

0 comments on commit 744f9a1

Please sign in to comment.