Skip to content

Commit

Permalink
Merge pull request #1 from maditya/pg-filter
Browse files Browse the repository at this point in the history
add pg filter based on mysql filter
  • Loading branch information
maditya authored and GitHub Enterprise committed Dec 29, 2020
2 parents a55e763 + e5767e5 commit bd4801d
Show file tree
Hide file tree
Showing 6 changed files with 252 additions and 22 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/antlr/antlr4 v0.0.0-20190819145818-b43a4c3a8015
github.com/boltdb/bolt v1.3.1
github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292
github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e
github.com/fernet/fernet-go v0.0.0-20191111064656-eff2850e6001
github.com/golang/protobuf v1.4.3
github.com/google/go-cmp v0.5.2
github.com/google/logger v1.0.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e h1:P10tZmVD2XclAaT9l7OduMH1OLFzTa1wUuUqHZnEdI0=
github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e/go.mod h1:2H9hjfbpSMHwY503FclkV/lZTBh2YlOmLLSda12uL8c=
github.com/fernet/fernet-go v0.0.0-20191111064656-eff2850e6001 h1:/UMxx5lGDg30aioUL9e7xJnbJfJeX7vhcm57fa5udaI=
github.com/fernet/fernet-go v0.0.0-20191111064656-eff2850e6001/go.mod h1:2H9hjfbpSMHwY503FclkV/lZTBh2YlOmLLSda12uL8c=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
Expand Down
Binary file removed go/v1beta1/__debug_bin
Binary file not shown.
187 changes: 187 additions & 0 deletions go/v1beta1/storage/pgsqlfilter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
// Copyright 2019 The Grafeas Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
"fmt"
"log"
"strings"

syntax "github.com/grafeas/grafeas/cel"
"github.com/grafeas/grafeas/go/filtering/common"
"github.com/grafeas/grafeas/go/filtering/operators"
"github.com/grafeas/grafeas/go/filtering/parser"
)

// var fs filterSql
// fmt.Println("\nsql:", fs.makeSql(expr))

type PgsqlFilterSql struct {
selects int
}

func (fs *PgsqlFilterSql) sqlFromCall(func_name string, args []*syntax.Expr) string {

var sql_op string
switch func_name {
case operators.Equals:
sql_op = "="
case operators.Greater:
sql_op = ">"
case operators.GreaterEquals:
sql_op = ">="
case operators.Less:
sql_op = "<"
case operators.LessEquals:
sql_op = "<="
case operators.NotEquals:
sql_op = "!="
case operators.LogicalAnd:
sql_op = "AND"
case operators.LogicalOr:
sql_op = "OR"
case operators.Index:
sql_op = "["
default:
sql_op = ""
}
var arg_names []string
for _, arg := range args {
arg_names = append(arg_names, fs.makeSql(arg))
}
if sql_op == "[" {
return fmt.Sprintf("%s[%s]", arg_names[0], arg_names[1])
} else if sql_op != "" {
return fmt.Sprintf("(%s %s %s)", arg_names[0], sql_op, arg_names[1])
} else {
return fmt.Sprintf("%s(%s)", func_name, strings.Join(arg_names, ", "))
}
}

func (fs *PgsqlFilterSql) sqlFromSelect(select_node syntax.Expr_Select) string {
operand := fs.makeSql(select_node.GetOperand())
field := select_node.GetField()
return fmt.Sprintf("%s.%s", operand, field)
}

func (fs *PgsqlFilterSql) getConstantValue(const_expr syntax.Constant) string {

switch const_expr.GetConstantKind().(type) {
case *syntax.Constant_Int64Value:
return fmt.Sprintf("%d", const_expr.GetInt64Value())
case *syntax.Constant_Uint64Value:
return fmt.Sprintf("%d", const_expr.GetUint64Value())
//case *syntax.Constant_DoubleValue: // TODO: fix
// return fmt.Sprintf("%d", const_expr.GetDoubleValue())
case *syntax.Constant_StringValue:
//return fmt.Sprintf("\"%s\"", const_expr.GetStringValue())
return fmt.Sprintf("'%s'", const_expr.GetStringValue())
}
return "NO CONST"
}

func (fs *PgsqlFilterSql) makeSql(node *syntax.Expr) string {
//log.Println("depth: ", fs.depth, "node:", node)
switch node.GetExprKind().(type) {
case *syntax.Expr_CallExpr:
func_node := *node.GetCallExpr()
return fs.sqlFromCall(func_node.Function, func_node.Args)
case *syntax.Expr_SelectExpr:
select_node := *node.GetSelectExpr()
fs.selects++
ret_str := fs.sqlFromSelect(select_node)
fs.selects--
if fs.selects == 0 {
spl := strings.Split(ret_str, ".")
ret_val := "data"
sep := "->'"
sep2 := "->>'"
for i := 0; i < len(spl); i++ {
if i != len(spl)-1 {
ret_val = ret_val + sep + spl[i] + "'"
} else {
ret_val = ret_val + sep2 + spl[i] + "'"
}
}

//return "data->'$." + ret_str + "'"
//return "data->'" + ret_str + "'"
return ret_val
} else {
return ret_str
}
case *syntax.Expr_IdentExpr:
i_expr := *node.GetIdentExpr()
// I'm not entirely sure this is the right thing here.
// We'll see though.
if fs.selects > 0 {
return i_expr.Name
} else {
//return "data->'$." + i_expr.Name + "'"
return "data->>'" + i_expr.Name + "'"
}
case *syntax.Expr_ConstExpr:
c_expr := *node.GetConstExpr()
return fs.getConstantValue(c_expr)
}

return "NO SQL"

}

func (fs *PgsqlFilterSql) ParseFilter(filter string) string {

log.Println(filter)
// replace string value for kind with integer
/*
kindPattern := `kind=".*?"`
re := regexp.MustCompile(kindPattern)
indices := re.FindStringIndex(filter)
if indices != nil {
int_val := 0
switch filter[indices[0]+6 : indices[1]-1] {
case "VULNERABILITY":
int_val = 1
case "BUILD":
int_val = 2
case "IMAGE":
int_val = 3
case "PACKAGE":
int_val = 4
case "DEPLOYMENT":
int_val = 5
case "DISCOVERY":
int_val = 6
case "ATTESTATION":
int_val = 7
case "INTOTO":
int_val = 8
}
filter = filter[:indices[0]+5] + fmt.Sprintf("%d", int_val) + filter[indices[1]:]
// log.Println(filter)
}
*/
s := common.NewStringSource(filter, "urlParam") // function
result, err := parser.Parse(s)
if err != nil {
log.Println(err)
return ""
}
// log.Println(result.Expr.GetCallExpr().Args[1].GetCallExpr().Args[0].GetIdentExpr().Name)
// log.Println(result.Expr)
sql := fs.makeSql(result.Expr)
//log.Println(sql)
return sql
}
71 changes: 56 additions & 15 deletions go/v1beta1/storage/pgsqlstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
fieldmaskpb "google.golang.org/genproto/protobuf/field_mask"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/encoding/protojson"
)

type PgSQLStore struct {
Expand Down Expand Up @@ -157,8 +158,17 @@ func (pg *PgSQLStore) ListProjects(ctx context.Context, filter string, pageSize
if err != nil {
return nil, "", status.Error(codes.Internal, "Failed to count Projects from database")
}

var filter_query, query string
if filter != "" {
var fs PgsqlFilterSql
filter_query = fs.ParseFilter(filter) + " AND "
} else {
filter_query = ""
}
query = fmt.Sprintf(listProjects, filter_query)
id := decryptInt64(pageToken, pg.paginationKey, 0)
rows, err := pg.DB.QueryContext(ctx, listProjects, id, pageSize)
rows, err := pg.DB.QueryContext(ctx, query, id, pageSize)
if err != nil {
return nil, "", status.Error(codes.Internal, "Failed to list Projects from database")
}
Expand Down Expand Up @@ -200,7 +210,12 @@ func (pg *PgSQLStore) CreateOccurrence(ctx context.Context, pID, uID string, o *
log.Printf("Invalid note name: %v", o.NoteName)
return nil, status.Error(codes.InvalidArgument, "Invalid note name")
}
_, err = pg.DB.ExecContext(ctx, insertOccurrence, pID, id, nPID, nID, proto.MarshalTextString(o))
jo, err := protojson.Marshal(o)
if err != nil {
return nil, status.Errorf(codes.Internal, "Failed to marshal proto to json: %v", err)
}

_, err = pg.DB.ExecContext(ctx, insertOccurrence, pID, id, nPID, nID, string(jo))
if err, ok := err.(*pq.Error); ok {
// Check for unique_violation
if err.Code == "23505" {
Expand Down Expand Up @@ -258,7 +273,11 @@ func (pg *PgSQLStore) UpdateOccurrence(ctx context.Context, pID, oID string, o *
// TODO(#312): implement the update operation
o.UpdateTime = ptypes.TimestampNow()

result, err := pg.DB.ExecContext(ctx, updateOccurrence, proto.MarshalTextString(o), pID, oID)
jo, err := protojson.Marshal(o)
if err != nil {
return nil, status.Errorf(codes.Internal, "Failed to marshal proto to json: %v", err)
}
result, err := pg.DB.ExecContext(ctx, updateOccurrence, string(jo), pID, oID)
if err != nil {
return nil, status.Error(codes.Internal, "Failed to update Occurrence")
}
Expand Down Expand Up @@ -299,8 +318,17 @@ func (pg *PgSQLStore) ListOccurrences(ctx context.Context, pID, filter, pageToke
if err != nil {
return nil, "", status.Error(codes.Internal, "Failed to count Occurrences from database")
}

var filter_query, query string
if filter != "" {
var fs PgsqlFilterSql
filter_query = " AND " + fs.ParseFilter(filter)
} else {
filter_query = ""
}
query = fmt.Sprintf(listOccurrences, filter_query)
id := decryptInt64(pageToken, pg.paginationKey, 0)
rows, err := pg.DB.QueryContext(ctx, listOccurrences, pID, id, pageSize)
rows, err := pg.DB.QueryContext(ctx, query, pID, id, pageSize)
if err != nil {
return nil, "", status.Error(codes.Internal, "Failed to list Occurrences from database")
}
Expand All @@ -314,8 +342,7 @@ func (pg *PgSQLStore) ListOccurrences(ctx context.Context, pID, filter, pageToke
return nil, "", status.Error(codes.Internal, "Failed to scan Occurrences row")
}
var o pb.Occurrence
proto.UnmarshalText(data, &o)
if err != nil {
if err := protojson.Unmarshal([]byte(data), &o); err != nil {
return nil, "", status.Error(codes.Internal, "Failed to unmarshal Occurrence from database")
}
os = append(os, &o)
Expand All @@ -337,7 +364,11 @@ func (pg *PgSQLStore) CreateNote(ctx context.Context, pID, nID, uID string, n *p
n.Name = nName
n.CreateTime = ptypes.TimestampNow()

_, err := pg.DB.ExecContext(ctx, insertNote, pID, nID, proto.MarshalTextString(n))
jn, err := protojson.Marshal(n)
if err != nil {
return nil, status.Errorf(codes.Internal, "Failed to marshal proto to json: %v", err)
}
_, err = pg.DB.ExecContext(ctx, insertNote, pID, nID, string(jn))
if err, ok := err.(*pq.Error); ok {
// Check for unique_violation
if err.Code == "23505" {
Expand Down Expand Up @@ -398,7 +429,11 @@ func (pg *PgSQLStore) UpdateNote(ctx context.Context, pID, nID string, n *pb.Not
// TODO(#312): implement the update operation
n.UpdateTime = ptypes.TimestampNow()

result, err := pg.DB.ExecContext(ctx, updateNote, proto.MarshalTextString(n), pID, nID)
jn, err := protojson.Marshal(n)
if err != nil {
return nil, status.Errorf(codes.Internal, "Failed to marshal proto to json: %v", err)
}
result, err := pg.DB.ExecContext(ctx, updateNote, string(jn), pID, nID)
if err != nil {
return nil, status.Error(codes.Internal, "Failed to update Note")
}
Expand All @@ -423,8 +458,7 @@ func (pg *PgSQLStore) GetNote(ctx context.Context, pID, nID string) (*pb.Note, e
return nil, status.Error(codes.Internal, "Failed to query Note from database")
}
var note pb.Note
proto.UnmarshalText(data, &note)
if err != nil {
if err := protojson.Unmarshal([]byte(data), &note); err != nil {
return nil, status.Error(codes.Internal, "Failed to unmarshal Note from database")
}
// Set the output-only field before returning
Expand Down Expand Up @@ -459,8 +493,17 @@ func (pg *PgSQLStore) ListNotes(ctx context.Context, pID, filter, pageToken stri
if err != nil {
return nil, "", status.Error(codes.Internal, "Failed to count Notes from database")
}

var filter_query, query string
if filter != "" {
var fs PgsqlFilterSql
filter_query = " AND " + fs.ParseFilter(filter)
} else {
filter_query = ""
}
query = fmt.Sprintf(listNotes, filter_query)
id := decryptInt64(pageToken, pg.paginationKey, 0)
rows, err := pg.DB.QueryContext(ctx, listNotes, pID, id, pageSize)
rows, err := pg.DB.QueryContext(ctx, query, pID, id, pageSize)
if err != nil {
return nil, "", status.Error(codes.Internal, "Failed to list Notes from database")
}
Expand All @@ -474,8 +517,7 @@ func (pg *PgSQLStore) ListNotes(ctx context.Context, pID, filter, pageToken stri
return nil, "", status.Error(codes.Internal, "Failed to scan Notes row")
}
var n pb.Note
proto.UnmarshalText(data, &n)
if err != nil {
if err := protojson.Unmarshal([]byte(data), &n); err != nil {
return nil, "", status.Error(codes.Internal, "Failed to unmarshal Note from database")
}
ns = append(ns, &n)
Expand Down Expand Up @@ -516,8 +558,7 @@ func (pg *PgSQLStore) ListNoteOccurrences(ctx context.Context, pID, nID, filter,
return nil, "", status.Error(codes.Internal, "Failed to scan Occurrences row")
}
var o pb.Occurrence
proto.UnmarshalText(data, &o)
if err != nil {
if err := protojson.Unmarshal([]byte(data), &o); err != nil {
return nil, "", status.Error(codes.Internal, "Failed to unmarshal Occurrence from database")
}
os = append(os, &o)
Expand Down
Loading

0 comments on commit bd4801d

Please sign in to comment.