Skip to content

Commit

Permalink
mark int type explicitly as int64 (#332)
Browse files Browse the repository at this point in the history 
* mark int type explicitly as int64

fixes #330

Signed-off-by: Thomas Jungblut <tjungblu@redhat.com>

* use 32bit truncation for parquet

---------

Signed-off-by: Thomas Jungblut <tjungblu@redhat.com>
  • Loading branch information
@tjungblu
tjungblu authored May 11, 2024
1 parent ca4ac3f commit 4a66fe0
Show file tree
Hide file tree
Showing 14 changed files with 38 additions and 38 deletions.
2 changes: 1 addition & 1 deletion aggregates/count.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ var CountOverloads = []physical.AggregateDescriptor{
}

type Count struct {
count int
count int64
}

func NewCountPrototype() func() nodes.Aggregate {
Expand Down
2 changes: 1 addition & 1 deletion aggregates/sum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ var SumOverloads = []physical.AggregateDescriptor{
}

type SumInt struct {
sum int
sum int64
}

func NewSumIntPrototype() func() nodes.Aggregate {
Expand Down
2 changes: 1 addition & 1 deletion cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ octosql "SELECT * FROM plugins.plugins"`,

switch output {
case "live_table", "batch_table":
var limit *int
var limit *int64
if limitExpression != nil {
val, err := (*limitExpression).Evaluate(execCtx)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion datasources/csv/execution.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (d *DatasourceExecuting) Run(ctx ExecutionContext, produce ProduceFn, metaS
if octosql.Int.Is(d.fields[i].Type) == octosql.TypeRelationIs {
integer, err := fastfloat.ParseInt64(str)
if err == nil {
values[i] = octosql.NewInt(int(integer))
values[i] = octosql.NewInt(integer)
continue
}
}
Expand Down
2 changes: 1 addition & 1 deletion datasources/lines/execution.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func (d *DatasourceExecuting) Run(ctx ExecutionContext, produce ProduceFn, metaS
})
}

line := 0
line := int64(0)
for sc.Scan() {
values := make([]octosql.Value, len(d.fields))
for i := range d.fields {
Expand Down
4 changes: 2 additions & 2 deletions datasources/parquet/reconstruct.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,9 +293,9 @@ func assignValue(dst *octosql.Value, src parquet.Value) error {
case parquet.Boolean:
*dst = octosql.NewBoolean(src.Boolean())
case parquet.Int32:
*dst = octosql.NewInt(int(src.Int32()))
*dst = octosql.NewInt(int64(src.Int32()))
case parquet.Int64:
*dst = octosql.NewInt(int(src.Int64()))
*dst = octosql.NewInt(src.Int64())
case parquet.Int96:
*dst = octosql.NewString(src.Int96().String())
case parquet.Float:
Expand Down
2 changes: 1 addition & 1 deletion execution/nodes/limit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func (m *Limit) Run(ctx ExecutionContext, produce ProduceFn, metaSend MetaSendFn

limitNodeID := ulid.MustNew(ulid.Now(), rand.Reader).String()

i := 0
i := int64(0)
if err := m.source.Run(ctx, func(produceCtx ProduceContext, record Record) error {
if err := produce(produceCtx, record); err != nil {
return fmt.Errorf("couldn't produce: %w", err)
Expand Down
8 changes: 4 additions & 4 deletions execution/nodes/order_sensitive_transform.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (item *orderByItem) Less(than btree.Item) bool {
}

func (o *OrderSensitiveTransform) Run(execCtx ExecutionContext, produce ProduceFn, metaSend MetaSendFn) error {
var limit *int
var limit *int64
if o.limit != nil {
val, err := (*o.limit).Evaluate(execCtx)
if err != nil {
Expand Down Expand Up @@ -112,7 +112,7 @@ func (o *OrderSensitiveTransform) Run(execCtx ExecutionContext, produce ProduceF
} else {
recordCounts.Delete(itemTyped)
}
if limit != nil && o.noRetractionsPossible && recordCounts.Len() > *limit {
if limit != nil && o.noRetractionsPossible && int64(recordCounts.Len()) > *limit {
// This doesn't mean we'll always keep just the records that are needed, because tree nodes might have count > 1.
// That said, it's a good approximation, and we'll definitely not lose something that we need to have.
recordCounts.DeleteMax()
Expand All @@ -130,8 +130,8 @@ func (o *OrderSensitiveTransform) Run(execCtx ExecutionContext, produce ProduceF
return nil
}

func produceOrderByItems(ctx ProduceContext, recordCounts *btree.BTree, limit *int, produce ProduceFn) error {
i := 0
func produceOrderByItems(ctx ProduceContext, recordCounts *btree.BTree, limit *int64, produce ProduceFn) error {
i := int64(0)
var outErr error
recordCounts.Ascend(func(item btree.Item) bool {
if limit != nil && i >= *limit {
Expand Down
32 changes: 16 additions & 16 deletions functions/functions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,15 +304,15 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.String,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewString(strings.Repeat(values[0].Str, values[1].Int)), nil
return octosql.NewString(strings.Repeat(values[0].Str, int(values[1].Int))), nil
},
},
{
ArgumentTypes: []octosql.Type{octosql.Int, octosql.String},
OutputType: octosql.String,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewString(strings.Repeat(values[1].Str, values[0].Int)), nil
return octosql.NewString(strings.Repeat(values[1].Str, int(values[0].Int))), nil
},
},
},
Expand Down Expand Up @@ -718,7 +718,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.String,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
if len(values[0].Str) <= values[1].Int {
if int64(len(values[0].Str)) <= values[1].Int {
return octosql.NewString(""), nil
}
return octosql.NewString(values[0].Str[values[1].Int:]), nil
Expand All @@ -729,12 +729,12 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.String,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
if len(values[0].Str) <= values[1].Int {
if int64(len(values[0].Str)) <= values[1].Int {
return octosql.NewString(""), nil
}
end := values[1].Int + values[2].Int
if end > len(values[0].Str) {
end = len(values[0].Str)
if end > int64(len(values[0].Str)) {
end = int64(len(values[0].Str))
}
return octosql.NewString(values[0].Str[values[1].Int:end]), nil
},
Expand Down Expand Up @@ -766,7 +766,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
if i == -1 {
return octosql.NewNull(), nil
}
return octosql.NewInt(i), nil
return octosql.NewInt(int64(i)), nil
},
},
},
Expand All @@ -779,7 +779,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(len(values[0].Str)), nil
return octosql.NewInt(int64(len(values[0].Str))), nil
},
},
{
Expand All @@ -794,7 +794,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
},
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(len(values[0].List)), nil
return octosql.NewInt(int64(len(values[0].List))), nil
},
},
{
Expand All @@ -809,7 +809,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
},
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(len(values[0].Struct)), nil
return octosql.NewInt(int64(len(values[0].Struct))), nil
},
},
{
Expand All @@ -824,7 +824,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
},
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(len(values[0].Tuple)), nil
return octosql.NewInt(int64(len(values[0].Tuple))), nil
},
},
},
Expand Down Expand Up @@ -891,7 +891,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(int(values[0].Time.Unix())), nil
return octosql.NewInt(values[0].Time.Unix()), nil
},
},
},
Expand Down Expand Up @@ -927,15 +927,15 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(int(values[0].Float)), nil
return octosql.NewInt(int64(values[0].Float)), nil
},
},
{
ArgumentTypes: []octosql.Type{octosql.String},
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
n, err := strconv.Atoi(values[0].Str)
n, err := strconv.ParseInt(values[0].Str, 10, 64)
if err != nil {
log.Printf("couldn't parse string '%s' as int: %s", values[0].Str, err)
return octosql.NewNull(), nil
Expand All @@ -948,7 +948,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(int(values[0].Duration)), nil
return octosql.NewInt(int64(values[0].Duration)), nil
},
},
},
Expand Down Expand Up @@ -1033,7 +1033,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
},
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
if values[1].Int >= len(values[0].List) {
if values[1].Int >= int64(len(values[0].List)) {
return octosql.NewNull(), nil
}
return values[0].List[values[1].Int], nil
Expand Down
4 changes: 2 additions & 2 deletions octosql/values.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ var ZeroValue = Value{}
// Value represents a single row value. The zero value of it is conveniently NULL.
type Value struct {
TypeID TypeID
Int int
Int int64
Float float64
Boolean bool
Str string
Expand All @@ -31,7 +31,7 @@ func NewNull() Value {
}
}

func NewInt(value int) Value {
func NewInt(value int64) Value {
return Value{
TypeID: TypeIDInt,
Int: value,
Expand Down
10 changes: 5 additions & 5 deletions outputs/batch/live_output.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ type OutputPrinter struct {
source Node
keyExprs []Expression
directionMultipliers []int
limit *int
limit *int64
noRetractionsPossible bool

schema physical.Schema
format func(io.Writer) Format
live bool
}

func NewOutputPrinter(source Node, keyExprs []Expression, directionMultipliers []int, limit *int, noRetractionsPossible bool, schema physical.Schema, format func(io.Writer) Format, live bool) *OutputPrinter {
func NewOutputPrinter(source Node, keyExprs []Expression, directionMultipliers []int, limit *int64, noRetractionsPossible bool, schema physical.Schema, format func(io.Writer) Format, live bool) *OutputPrinter {
return &OutputPrinter{
source: source,
keyExprs: keyExprs,
Expand Down Expand Up @@ -89,7 +89,7 @@ func (o *OutputPrinter) Run(execCtx ExecutionContext) error {
format := o.format(&buf)
format.SetSchema(o.schema)

i := 0
i := int64(0)
recordCounts.Ascend(func(item btree.Item) bool {
itemTyped := item.(*outputItem)
for j := 0; j < itemTyped.Count; j++ {
Expand Down Expand Up @@ -157,7 +157,7 @@ func (o *OutputPrinter) Run(execCtx ExecutionContext) error {
if onlyZeroEventTimesSeen && !record.EventTime.IsZero() {
onlyZeroEventTimesSeen = false
}
if o.limit != nil && o.noRetractionsPossible && recordCounts.Len() > *o.limit {
if o.limit != nil && o.noRetractionsPossible && int64(recordCounts.Len()) > *o.limit {
// This doesn't mean we'll always keep just the records that are needed, because tree nodes might have count > 1.
// That said, it's a good approximation, and we'll definitely not lose something that we need to have.
recordCounts.DeleteMax()
Expand All @@ -183,7 +183,7 @@ func (o *OutputPrinter) Run(execCtx ExecutionContext) error {
var buf bytes.Buffer
format := o.format(&buf)
format.SetSchema(o.schema)
i := 0
i := int64(0)
recordCounts.Ascend(func(item btree.Item) bool {
itemTyped := item.(*outputItem)
for j := 0; j < itemTyped.Count; j++ {
Expand Down
2 changes: 1 addition & 1 deletion outputs/formats/json_format.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func ValueToJson(arena *fastjson.Arena, t octosql.Type, value octosql.Value) *fa
case octosql.TypeIDNull:
return arena.NewNull()
case octosql.TypeIDInt:
return arena.NewNumberInt(value.Int)
return arena.NewNumberInt(int(value.Int))
case octosql.TypeIDFloat:
return arena.NewNumberFloat64(value.Float)
case octosql.TypeIDBoolean:
Expand Down
2 changes: 1 addition & 1 deletion parser/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,7 @@ func ParseExpression(expr sqlparser.Expr) (logical.Expression, error) {
case sqlparser.IntVal:
var i int64
i, err = strconv.ParseInt(string(expr.Val), 10, 64)
value = octosql.NewInt(int(i))
value = octosql.NewInt(i)
case sqlparser.FloatVal:
var val float64
val, err = strconv.ParseFloat(string(expr.Val), 64)
Expand Down
2 changes: 1 addition & 1 deletion plugins/internal/plugins/plugins.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (x *Value) ToNativeValue() octosql.Value {
switch octosql.TypeID(x.TypeId) {
case octosql.TypeIDNull:
case octosql.TypeIDInt:
out.Int = int(x.Int)
out.Int = x.Int
case octosql.TypeIDFloat:
out.Float = x.Float
case octosql.TypeIDBoolean:
Expand Down

0 comments on commit 4a66fe0

Please sign in to comment.