From cf991a08e8fe0ef95917c4521eea1bd7e2e37b6f Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Wed, 12 Apr 2023 18:31:44 -0700 Subject: [PATCH 1/5] Add EXTRACT builtin function --- partiql-conformance-tests/partiql-tests | 2 +- partiql-conformance-tests/tests/test_value.rs | 4 +- partiql-eval/src/eval/expr/mod.rs | 213 +++++++++++++++++- partiql-eval/src/plan.rs | 60 ++++- partiql-logical-planner/src/call_defs.rs | 113 ++++++++++ partiql-logical/src/lib.rs | 8 + partiql-value/src/datetime.rs | 13 +- partiql-value/src/ion.rs | 4 +- partiql-value/src/lib.rs | 21 ++ 9 files changed, 426 insertions(+), 12 deletions(-) diff --git a/partiql-conformance-tests/partiql-tests b/partiql-conformance-tests/partiql-tests index 5380f373..e24d8ef8 160000 --- a/partiql-conformance-tests/partiql-tests +++ b/partiql-conformance-tests/partiql-tests @@ -1 +1 @@ -Subproject commit 5380f373de12d8adfa5eaae86efd12dedf7342ff +Subproject commit e24d8ef837303e675bc7c29b8a41f3cc81d6b96e diff --git a/partiql-conformance-tests/tests/test_value.rs b/partiql-conformance-tests/tests/test_value.rs index 05b11062..ff39e2cf 100644 --- a/partiql-conformance-tests/tests/test_value.rs +++ b/partiql-conformance-tests/tests/test_value.rs @@ -205,14 +205,16 @@ fn parse_test_value_time(reader: &mut Reader) -> DateTime { fn parse_test_value_datetime(reader: &mut Reader) -> DateTime { let ts = reader.read_timestamp().unwrap(); + let offset = ts.offset(); // TODO: fractional seconds Cf. https://github.com/amazon-ion/ion-rust/pull/482#issuecomment-1470615286 - DateTime::from_ymdhms( + DateTime::from_ymdhms_offset_minutes( ts.year(), NonZeroU8::new(ts.month() as u8).unwrap(), ts.day() as u8, ts.hour() as u8, ts.minute() as u8, ts.second() as f64, + offset, ) } diff --git a/partiql-eval/src/eval/expr/mod.rs b/partiql-eval/src/eval/expr/mod.rs index 4951f518..e7a37135 100644 --- a/partiql-eval/src/eval/expr/mod.rs +++ b/partiql-eval/src/eval/expr/mod.rs @@ -5,7 +5,8 @@ use itertools::Itertools; use partiql_logical::Type; use partiql_value::Value::{Boolean, Missing, Null}; use partiql_value::{ - Bag, BinaryAnd, BinaryOr, BindingsName, List, NullableEq, NullableOrd, Tuple, UnaryPlus, Value, + Bag, BinaryAnd, BinaryOr, BindingsName, DateTime, List, NullableEq, NullableOrd, Tuple, + UnaryPlus, Value, }; use regex::{Regex, RegexBuilder}; use std::borrow::{Borrow, Cow}; @@ -935,3 +936,213 @@ impl EvalExpr for EvalFnCardinality { Cow::Owned(result) } } + +/// Represents a year `EXTRACT` function, e.g. `extract(YEAR FROM t)`. +#[derive(Debug)] +pub struct EvalFnExtractYear { + pub value: Box, +} + +impl EvalExpr for EvalFnExtractYear { + #[inline] + fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { + let value = self.value.evaluate(bindings, ctx); + let result = match value.borrow() { + Null => Null, + Missing => Missing, + Value::DateTime(dt) => match dt.as_ref() { + DateTime::Date(d) => Value::from(d.year()), + DateTime::Timestamp(tstamp) => Value::from(tstamp.year()), + DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.year()), + DateTime::Time(_) => Missing, + DateTime::TimeWithTz(_, _) => Missing, + }, + _ => Missing, + }; + Cow::Owned(result) + } +} + +/// Represents a month `EXTRACT` function, e.g. `extract(MONTH FROM t)`. +#[derive(Debug)] +pub struct EvalFnExtractMonth { + pub value: Box, +} + +impl EvalExpr for EvalFnExtractMonth { + #[inline] + fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { + let value = self.value.evaluate(bindings, ctx); + let result = match value.borrow() { + Null => Null, + Missing => Missing, + Value::DateTime(dt) => match dt.as_ref() { + DateTime::Date(d) => Value::from(d.month() as u8), + DateTime::Timestamp(tstamp) => Value::from(tstamp.month() as u8), + DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.month() as u8), + DateTime::Time(_) => Missing, + DateTime::TimeWithTz(_, _) => Missing, + }, + _ => Missing, + }; + Cow::Owned(result) + } +} + +/// Represents a day `EXTRACT` function, e.g. `extract(DAT FROM t)`. +#[derive(Debug)] +pub struct EvalFnExtractDay { + pub value: Box, +} + +impl EvalExpr for EvalFnExtractDay { + #[inline] + fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { + let value = self.value.evaluate(bindings, ctx); + let result = match value.borrow() { + Null => Null, + Missing => Missing, + Value::DateTime(dt) => match dt.as_ref() { + DateTime::Date(d) => Value::from(d.day()), + DateTime::Timestamp(tstamp) => Value::from(tstamp.day()), + DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.day()), + DateTime::Time(_) => Missing, + DateTime::TimeWithTz(_, _) => Missing, + }, + _ => Missing, + }; + Cow::Owned(result) + } +} + +/// Represents an hour `EXTRACT` function, e.g. `extract(HOUR FROM t)`. +#[derive(Debug)] +pub struct EvalFnExtractHour { + pub value: Box, +} + +impl EvalExpr for EvalFnExtractHour { + #[inline] + fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { + let value = self.value.evaluate(bindings, ctx); + let result = match value.borrow() { + Null => Null, + Missing => Missing, + Value::DateTime(dt) => match dt.as_ref() { + DateTime::Time(t) => Value::from(t.hour()), + DateTime::TimeWithTz(t, _) => Value::from(t.hour()), + DateTime::Timestamp(tstamp) => Value::from(tstamp.hour()), + DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.hour()), + DateTime::Date(_) => Missing, + }, + _ => Missing, + }; + Cow::Owned(result) + } +} + +/// Represents a minute `EXTRACT` function, e.g. `extract(MINUTE FROM t)`. +#[derive(Debug)] +pub struct EvalFnExtractMinute { + pub value: Box, +} + +impl EvalExpr for EvalFnExtractMinute { + #[inline] + fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { + let value = self.value.evaluate(bindings, ctx); + let result = match value.borrow() { + Null => Null, + Missing => Missing, + Value::DateTime(dt) => match dt.as_ref() { + DateTime::Time(t) => Value::from(t.minute()), + DateTime::TimeWithTz(t, _) => Value::from(t.minute()), + DateTime::Timestamp(tstamp) => Value::from(tstamp.minute()), + DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.minute()), + DateTime::Date(_) => Missing, + }, + _ => Missing, + }; + Cow::Owned(result) + } +} + +/// Represents a second `EXTRACT` function, e.g. `extract(SECOND FROM t)`. +#[derive(Debug)] +pub struct EvalFnExtractSecond { + pub value: Box, +} + +impl EvalExpr for EvalFnExtractSecond { + #[inline] + fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { + let value = self.value.evaluate(bindings, ctx); + let result = match value.borrow() { + Null => Null, + Missing => Missing, + Value::DateTime(dt) => match dt.as_ref() { + DateTime::Time(t) => Value::from(t.second()), + DateTime::TimeWithTz(t, _) => Value::from(t.second()), + DateTime::Timestamp(tstamp) => Value::from(tstamp.second()), + DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.second()), + DateTime::Date(_) => Missing, + }, + _ => Missing, + }; + Cow::Owned(result) + } +} + +/// Represents a timezone hour `EXTRACT` function, e.g. `extract(TIMEZONE_HOUR FROM t)`. +#[derive(Debug)] +pub struct EvalFnExtractTimezoneHour { + pub value: Box, +} + +impl EvalExpr for EvalFnExtractTimezoneHour { + #[inline] + fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { + let value = self.value.evaluate(bindings, ctx); + let result = match value.borrow() { + Null => Null, + Missing => Missing, + Value::DateTime(dt) => match dt.as_ref() { + DateTime::TimeWithTz(_, tz) => Value::from(tz.whole_hours()), + DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.offset().whole_hours()), + DateTime::Date(_) => Missing, + DateTime::Time(_) => Missing, + DateTime::Timestamp(_) => Missing, + }, + _ => Missing, + }; + Cow::Owned(result) + } +} + +/// Represents a timezone minute `EXTRACT` function, e.g. `extract(TIMEZONE_MINUTE FROM t)`. +#[derive(Debug)] +pub struct EvalFnExtractTimezoneMinute { + pub value: Box, +} + +impl EvalExpr for EvalFnExtractTimezoneMinute { + #[inline] + fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { + let value = self.value.evaluate(bindings, ctx); + let result = match value.borrow() { + Null => Null, + Missing => Missing, + Value::DateTime(dt) => match dt.as_ref() { + DateTime::TimeWithTz(_, tz) => Value::from(tz.minutes_past_hour()), + DateTime::TimestampWithTz(tstamp) => { + Value::from(tstamp.offset().minutes_past_hour() % 60) + } + DateTime::Date(_) => Missing, + DateTime::Time(_) => Missing, + DateTime::Timestamp(_) => Missing, + }, + _ => Missing, + }; + Cow::Owned(result) + } +} diff --git a/partiql-eval/src/plan.rs b/partiql-eval/src/plan.rs index 62e2ad3e..b6725b19 100644 --- a/partiql-eval/src/plan.rs +++ b/partiql-eval/src/plan.rs @@ -18,11 +18,13 @@ use crate::eval::evaluable::{ use crate::eval::expr::pattern_match::like_to_re_pattern; use crate::eval::expr::{ EvalBagExpr, EvalBetweenExpr, EvalBinOp, EvalBinOpExpr, EvalDynamicLookup, EvalExpr, EvalFnAbs, - EvalFnBitLength, EvalFnBtrim, EvalFnCardinality, EvalFnCharLength, EvalFnExists, EvalFnLower, - EvalFnLtrim, EvalFnModulus, EvalFnOctetLength, EvalFnOverlay, EvalFnPosition, EvalFnRtrim, - EvalFnSubstring, EvalFnUpper, EvalIsTypeExpr, EvalLikeMatch, EvalLikeNonStringNonLiteralMatch, - EvalListExpr, EvalLitExpr, EvalPath, EvalSearchedCaseExpr, EvalTupleExpr, EvalUnaryOp, - EvalUnaryOpExpr, EvalVarRef, + EvalFnBitLength, EvalFnBtrim, EvalFnCardinality, EvalFnCharLength, EvalFnExists, + EvalFnExtractDay, EvalFnExtractHour, EvalFnExtractMinute, EvalFnExtractMonth, + EvalFnExtractSecond, EvalFnExtractTimezoneHour, EvalFnExtractTimezoneMinute, EvalFnExtractYear, + EvalFnLower, EvalFnLtrim, EvalFnModulus, EvalFnOctetLength, EvalFnOverlay, EvalFnPosition, + EvalFnRtrim, EvalFnSubstring, EvalFnUpper, EvalIsTypeExpr, EvalLikeMatch, + EvalLikeNonStringNonLiteralMatch, EvalListExpr, EvalLitExpr, EvalPath, EvalSearchedCaseExpr, + EvalTupleExpr, EvalUnaryOp, EvalUnaryOpExpr, EvalVarRef, }; use crate::eval::EvalPlan; use partiql_value::Value::Null; @@ -585,6 +587,54 @@ impl EvaluatorPlanner { value: args.pop().unwrap(), }) } + CallName::ExtractYear => { + assert_eq!(args.len(), 1); + Box::new(EvalFnExtractYear { + value: args.pop().unwrap(), + }) + } + CallName::ExtractMonth => { + assert_eq!(args.len(), 1); + Box::new(EvalFnExtractMonth { + value: args.pop().unwrap(), + }) + } + CallName::ExtractDay => { + assert_eq!(args.len(), 1); + Box::new(EvalFnExtractDay { + value: args.pop().unwrap(), + }) + } + CallName::ExtractHour => { + assert_eq!(args.len(), 1); + Box::new(EvalFnExtractHour { + value: args.pop().unwrap(), + }) + } + CallName::ExtractMinute => { + assert_eq!(args.len(), 1); + Box::new(EvalFnExtractMinute { + value: args.pop().unwrap(), + }) + } + CallName::ExtractSecond => { + assert_eq!(args.len(), 1); + Box::new(EvalFnExtractSecond { + value: args.pop().unwrap(), + }) + } + CallName::ExtractTimezoneHour => { + assert_eq!(args.len(), 1); + Box::new(EvalFnExtractTimezoneHour { + value: args.pop().unwrap(), + }) + } + CallName::ExtractTimezoneMinute => { + assert_eq!(args.len(), 1); + Box::new(EvalFnExtractTimezoneMinute { + value: args.pop().unwrap(), + }) + } } } } diff --git a/partiql-logical-planner/src/call_defs.rs b/partiql-logical-planner/src/call_defs.rs index 50a629e0..ae9f12ec 100644 --- a/partiql-logical-planner/src/call_defs.rs +++ b/partiql-logical-planner/src/call_defs.rs @@ -441,6 +441,118 @@ fn function_call_def_cardinality() -> CallDef { } } +fn function_call_def_extract() -> CallDef { + CallDef { + names: vec!["extract"], + overloads: vec![ + CallSpec { + input: vec![ + CallSpecArg::Named("year".into()), + CallSpecArg::Named("from".into()), + ], + output: Box::new(|mut args| { + args.remove(0); // remove first default synthesized argument + logical::ValueExpr::Call(logical::CallExpr { + name: logical::CallName::ExtractYear, + arguments: args, + }) + }), + }, + CallSpec { + input: vec![ + CallSpecArg::Named("month".into()), + CallSpecArg::Named("from".into()), + ], + output: Box::new(|mut args| { + args.remove(0); // remove first default synthesized argument + logical::ValueExpr::Call(logical::CallExpr { + name: logical::CallName::ExtractMonth, + arguments: args, + }) + }), + }, + CallSpec { + input: vec![ + CallSpecArg::Named("day".into()), + CallSpecArg::Named("from".into()), + ], + output: Box::new(|mut args| { + args.remove(0); // remove first default synthesized argument + logical::ValueExpr::Call(logical::CallExpr { + name: logical::CallName::ExtractDay, + arguments: args, + }) + }), + }, + CallSpec { + input: vec![ + CallSpecArg::Named("hour".into()), + CallSpecArg::Named("from".into()), + ], + output: Box::new(|mut args| { + args.remove(0); // remove first default synthesized argument + logical::ValueExpr::Call(logical::CallExpr { + name: logical::CallName::ExtractHour, + arguments: args, + }) + }), + }, + CallSpec { + input: vec![ + CallSpecArg::Named("minute".into()), + CallSpecArg::Named("from".into()), + ], + output: Box::new(|mut args| { + args.remove(0); // remove first default synthesized argument + logical::ValueExpr::Call(logical::CallExpr { + name: logical::CallName::ExtractMinute, + arguments: args, + }) + }), + }, + CallSpec { + input: vec![ + CallSpecArg::Named("second".into()), + CallSpecArg::Named("from".into()), + ], + output: Box::new(|mut args| { + args.remove(0); // remove first default synthesized argument + logical::ValueExpr::Call(logical::CallExpr { + name: logical::CallName::ExtractSecond, + arguments: args, + }) + }), + }, + CallSpec { + input: vec![ + CallSpecArg::Named("timezone_hour".into()), + CallSpecArg::Named("from".into()), + ], + output: Box::new(|mut args| { + args.remove(0); // remove first default synthesized argument + logical::ValueExpr::Call(logical::CallExpr { + name: logical::CallName::ExtractTimezoneHour, + arguments: args, + }) + }), + }, + CallSpec { + input: vec![ + CallSpecArg::Named("timezone_minute".into()), + CallSpecArg::Named("from".into()), + ], + output: Box::new(|mut args| { + args.remove(0); // remove first default synthesized argument + logical::ValueExpr::Call(logical::CallExpr { + name: logical::CallName::ExtractTimezoneMinute, + arguments: args, + }) + }), + }, + ], + } +} + pub(crate) static FN_SYM_TAB: Lazy = Lazy::new(function_call_def); /// Function symbol table @@ -478,6 +590,7 @@ pub fn function_call_def() -> FnSymTab { function_call_def_abs(), function_call_def_mod(), function_call_def_cardinality(), + function_call_def_extract(), ] { assert!(!def.names.is_empty()); let primary = def.names[0]; diff --git a/partiql-logical/src/lib.rs b/partiql-logical/src/lib.rs index 81435f3a..ed49172e 100644 --- a/partiql-logical/src/lib.rs +++ b/partiql-logical/src/lib.rs @@ -651,6 +651,14 @@ pub enum CallName { Abs, Mod, Cardinality, + ExtractYear, + ExtractMonth, + ExtractDay, + ExtractHour, + ExtractMinute, + ExtractSecond, + ExtractTimezoneHour, + ExtractTimezoneMinute, } /// Indicates if a set should be reduced to its distinct elements or not. diff --git a/partiql-value/src/datetime.rs b/partiql-value/src/datetime.rs index 124aa2b0..cab141ae 100644 --- a/partiql-value/src/datetime.rs +++ b/partiql-value/src/datetime.rs @@ -48,19 +48,26 @@ impl DateTime { DateTime::Date(date) } - pub fn from_ymdhms( + pub fn from_ymdhms_offset_minutes( year: i32, month: NonZeroU8, day: u8, hour: u8, minute: u8, second: f64, + offset: Option, ) -> Self { let month: time::Month = month.get().try_into().expect("valid month"); let date = time::Date::from_calendar_date(year, month, day).expect("valid ymd"); let time = time_from_hmfs(hour, minute, second); - let date = date.with_time(time); - DateTime::Timestamp(date) + match offset { + None => DateTime::Timestamp(date.with_time(time)), + Some(o) => { + let offset = UtcOffset::from_whole_seconds(o * 60).expect("offset in range"); + let date = date.with_time(time).assume_offset(offset); + DateTime::TimestampWithTz(date) + } + } } fn from_hmfs_offset(hour: u8, minute: u8, second: f64, offset: Option) -> Self { diff --git a/partiql-value/src/ion.rs b/partiql-value/src/ion.rs index 27bf3ab4..a5184563 100644 --- a/partiql-value/src/ion.rs +++ b/partiql-value/src/ion.rs @@ -199,14 +199,16 @@ fn parse_time(reader: &mut Reader) -> DateTime { fn parse_datetime(reader: &mut Reader) -> DateTime { let ts = reader.read_timestamp().unwrap(); + let offset = ts.offset(); // TODO: fractional seconds Cf. https://github.com/amazon-ion/ion-rust/pull/482#issuecomment-1470615286 - DateTime::from_ymdhms( + DateTime::from_ymdhms_offset_minutes( ts.year(), NonZeroU8::new(ts.month() as u8).unwrap(), ts.day() as u8, ts.hour() as u8, ts.minute() as u8, ts.second() as f64, + offset, ) } diff --git a/partiql-value/src/lib.rs b/partiql-value/src/lib.rs index 4b2501a6..88df13ad 100644 --- a/partiql-value/src/lib.rs +++ b/partiql-value/src/lib.rs @@ -839,6 +839,20 @@ impl From for Value { } } +impl From for Value { + #[inline] + fn from(n: i16) -> Self { + (n as i64).into() + } +} + +impl From for Value { + #[inline] + fn from(n: i8) -> Self { + (n as i64).into() + } +} + impl From for Value { #[inline] fn from(n: usize) -> Self { @@ -847,6 +861,13 @@ impl From for Value { } } +impl From for Value { + #[inline] + fn from(n: u8) -> Self { + (n as usize).into() + } +} + impl From for Value { #[inline] fn from(f: f64) -> Self { From 5c8f971a94837d8503c9c6869f0474077f64ab11 Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Fri, 14 Apr 2023 17:04:02 -0700 Subject: [PATCH 2/5] Add parsing of nanoseconds --- partiql-conformance-tests/tests/test_value.rs | 11 +++--- partiql-eval/src/eval/expr/mod.rs | 25 ++++++++------ partiql-parser/src/preprocessor.rs | 2 +- partiql-value/src/datetime.rs | 34 +++++++++++-------- partiql-value/src/ion.rs | 11 +++--- 5 files changed, 48 insertions(+), 35 deletions(-) diff --git a/partiql-conformance-tests/tests/test_value.rs b/partiql-conformance-tests/tests/test_value.rs index ff39e2cf..e4d517fb 100644 --- a/partiql-conformance-tests/tests/test_value.rs +++ b/partiql-conformance-tests/tests/test_value.rs @@ -194,10 +194,11 @@ fn parse_test_value_time(reader: &mut Reader) -> DateTime { } reader.step_out().expect("step out of struct"); - DateTime::from_hmfs_tz( + DateTime::from_hms_nano_tz( time.hour.expect("hour"), time.minute.expect("minute"), - time.second.expect("second"), + time.second.expect("second").trunc() as u8, + time.second.expect("second").fract() as u32, time.tz_hour, time.tz_minute, ) @@ -206,14 +207,14 @@ fn parse_test_value_time(reader: &mut Reader) -> DateTime { fn parse_test_value_datetime(reader: &mut Reader) -> DateTime { let ts = reader.read_timestamp().unwrap(); let offset = ts.offset(); - // TODO: fractional seconds Cf. https://github.com/amazon-ion/ion-rust/pull/482#issuecomment-1470615286 - DateTime::from_ymdhms_offset_minutes( + DateTime::from_ymdhms_nano_offset_minutes( ts.year(), NonZeroU8::new(ts.month() as u8).unwrap(), ts.day() as u8, ts.hour() as u8, ts.minute() as u8, - ts.second() as f64, + ts.second() as u8, + ts.nanoseconds() as u32, offset, ) } diff --git a/partiql-eval/src/eval/expr/mod.rs b/partiql-eval/src/eval/expr/mod.rs index e7a37135..1f32acad 100644 --- a/partiql-eval/src/eval/expr/mod.rs +++ b/partiql-eval/src/eval/expr/mod.rs @@ -9,6 +9,7 @@ use partiql_value::{ UnaryPlus, Value, }; use regex::{Regex, RegexBuilder}; +use rust_decimal::prelude::FromPrimitive; use std::borrow::{Borrow, Cow}; use std::fmt::Debug; @@ -1074,22 +1075,26 @@ pub struct EvalFnExtractSecond { } impl EvalExpr for EvalFnExtractSecond { + // TODO: doesn't currently extract fractional seconds #[inline] fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { let value = self.value.evaluate(bindings, ctx); - let result = match value.borrow() { - Null => Null, - Missing => Missing, + let (second, nanosecond) = match value.borrow() { + Null => return Cow::Owned(Null), + Missing => return Cow::Owned(Missing), Value::DateTime(dt) => match dt.as_ref() { - DateTime::Time(t) => Value::from(t.second()), - DateTime::TimeWithTz(t, _) => Value::from(t.second()), - DateTime::Timestamp(tstamp) => Value::from(tstamp.second()), - DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.second()), - DateTime::Date(_) => Missing, + DateTime::Time(t) => (t.second(), t.nanosecond()), + DateTime::TimeWithTz(t, _) => (t.second(), t.nanosecond()), + DateTime::Timestamp(tstamp) => (tstamp.second(), tstamp.nanosecond()), + DateTime::TimestampWithTz(tstamp) => (tstamp.second(), tstamp.nanosecond()), + DateTime::Date(_) => return Cow::Owned(Missing), }, - _ => Missing, + _ => return Cow::Owned(Missing), }; - Cow::Owned(result) + let result = + rust_decimal::Decimal::from_f64(((second as f64 * 1e9) + nanosecond as f64) / 1e9) + .expect("time as decimal"); + Cow::Owned(Value::from(result)) } } diff --git a/partiql-parser/src/preprocessor.rs b/partiql-parser/src/preprocessor.rs index 86c7c8c3..77688083 100644 --- a/partiql-parser/src/preprocessor.rs +++ b/partiql-parser/src/preprocessor.rs @@ -87,7 +87,7 @@ mod built_ins { } const EXTRACT_SPECIFIER: &str = - "(?i:second)|(?i:minute)|(?i:hour)|(?i:day)|(?i:month)|(?:year)|(?:timezone_hour)|(?:timezone_minute)"; + "(?i:second)|(?i:minute)|(?i:hour)|(?i:day)|(?i:month)|(?i:year)|(?i:timezone_hour)|(?i:timezone_minute)"; pub(crate) fn built_in_extract() -> FnExpr<'static> { let re = Regex::new(EXTRACT_SPECIFIER).unwrap(); diff --git a/partiql-value/src/datetime.rs b/partiql-value/src/datetime.rs index cab141ae..67fd2877 100644 --- a/partiql-value/src/datetime.rs +++ b/partiql-value/src/datetime.rs @@ -21,14 +21,15 @@ impl DateTime { DateTime::Time(time::Time::from_hms(hour, minute, second).expect("valid time value")) } - pub fn from_hmfs(hour: u8, minute: u8, second: f64) -> Self { - Self::from_hmfs_offset(hour, minute, second, None) + pub fn from_hms_nano(hour: u8, minute: u8, second: u8, nanosecond: u32) -> Self { + Self::from_hms_nano_offset(hour, minute, second, nanosecond, None) } - pub fn from_hmfs_tz( + pub fn from_hms_nano_tz( hour: u8, minute: u8, - second: f64, + second: u8, + nanosecond: u32, tz_hours: Option, tz_minutes: Option, ) -> Self { @@ -39,7 +40,7 @@ impl DateTime { _ => None, }; - Self::from_hmfs_offset(hour, minute, second, offset) + Self::from_hms_nano_offset(hour, minute, second, nanosecond, offset) } pub fn from_ymd(year: i32, month: NonZeroU8, day: u8) -> Self { @@ -48,18 +49,19 @@ impl DateTime { DateTime::Date(date) } - pub fn from_ymdhms_offset_minutes( + pub fn from_ymdhms_nano_offset_minutes( year: i32, month: NonZeroU8, day: u8, hour: u8, minute: u8, - second: f64, + second: u8, + nanosecond: u32, offset: Option, ) -> Self { let month: time::Month = month.get().try_into().expect("valid month"); let date = time::Date::from_calendar_date(year, month, day).expect("valid ymd"); - let time = time_from_hmfs(hour, minute, second); + let time = time_from_hms_nano(hour, minute, second, nanosecond); match offset { None => DateTime::Timestamp(date.with_time(time)), Some(o) => { @@ -70,8 +72,14 @@ impl DateTime { } } - fn from_hmfs_offset(hour: u8, minute: u8, second: f64, offset: Option) -> Self { - let time = time_from_hmfs(hour, minute, second); + fn from_hms_nano_offset( + hour: u8, + minute: u8, + second: u8, + nanosecond: u32, + offset: Option, + ) -> Self { + let time = time_from_hms_nano(hour, minute, second, nanosecond); match offset { Some(offset) => DateTime::TimeWithTz(time, offset), None => DateTime::Time(time), @@ -79,10 +87,8 @@ impl DateTime { } } -fn time_from_hmfs(hour: u8, minute: u8, second: f64) -> time::Time { - let millis = (second.fract() * 1e9) as u32; - let second = second.trunc() as u8; - time::Time::from_hms_nano(hour, minute, second, millis).expect("valid time value") +fn time_from_hms_nano(hour: u8, minute: u8, second: u8, nanosecond: u32) -> time::Time { + time::Time::from_hms_nano(hour, minute, second, nanosecond).expect("valid time value") } impl Debug for DateTime { diff --git a/partiql-value/src/ion.rs b/partiql-value/src/ion.rs index a5184563..6fe6b447 100644 --- a/partiql-value/src/ion.rs +++ b/partiql-value/src/ion.rs @@ -188,10 +188,11 @@ fn parse_time(reader: &mut Reader) -> DateTime { } reader.step_out().expect("step out of struct"); - DateTime::from_hmfs_tz( + DateTime::from_hms_nano_tz( time.hour.expect("hour"), time.minute.expect("minute"), - time.second.expect("second"), + time.second.expect("second").trunc() as u8, + time.second.expect("second").fract() as u32, time.tz_hour, time.tz_minute, ) @@ -200,14 +201,14 @@ fn parse_time(reader: &mut Reader) -> DateTime { fn parse_datetime(reader: &mut Reader) -> DateTime { let ts = reader.read_timestamp().unwrap(); let offset = ts.offset(); - // TODO: fractional seconds Cf. https://github.com/amazon-ion/ion-rust/pull/482#issuecomment-1470615286 - DateTime::from_ymdhms_offset_minutes( + DateTime::from_ymdhms_nano_offset_minutes( ts.year(), NonZeroU8::new(ts.month() as u8).unwrap(), ts.day() as u8, ts.hour() as u8, ts.minute() as u8, - ts.second() as f64, + ts.second() as u8, + ts.nanoseconds(), offset, ) } From 3effa2636b74a61062a7b68a162fe1f55492bf35 Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Mon, 17 Apr 2023 10:53:26 -0700 Subject: [PATCH 3/5] Update to latest partiql-tests after extract test updates --- partiql-conformance-tests/partiql-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/partiql-conformance-tests/partiql-tests b/partiql-conformance-tests/partiql-tests index e24d8ef8..fdf35b4d 160000 --- a/partiql-conformance-tests/partiql-tests +++ b/partiql-conformance-tests/partiql-tests @@ -1 +1 @@ -Subproject commit e24d8ef837303e675bc7c29b8a41f3cc81d6b96e +Subproject commit fdf35b4d09a134aa08ec15606cfa72d3ecfc33e1 From 22379b9599d5551d6ec564052adc4ab4e77c624c Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Mon, 17 Apr 2023 12:02:57 -0700 Subject: [PATCH 4/5] Some minor cleanup --- CHANGELOG.md | 2 ++ partiql-eval/src/eval/expr/mod.rs | 40 ++++++++++++++----------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aada69a7..cc36266f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed ### Added +- Implements built-in function `EXTRACT` ### Fixes +- Fix parsing of `EXTRACT` datetime parts `YEAR`, `TIMEZONE_HOUR`, and `TIMEZONE_MINUTE` ## [0.3.0] - 2023-04-11 ### Changed diff --git a/partiql-eval/src/eval/expr/mod.rs b/partiql-eval/src/eval/expr/mod.rs index 1f32acad..001b1f3f 100644 --- a/partiql-eval/src/eval/expr/mod.rs +++ b/partiql-eval/src/eval/expr/mod.rs @@ -950,7 +950,6 @@ impl EvalExpr for EvalFnExtractYear { let value = self.value.evaluate(bindings, ctx); let result = match value.borrow() { Null => Null, - Missing => Missing, Value::DateTime(dt) => match dt.as_ref() { DateTime::Date(d) => Value::from(d.year()), DateTime::Timestamp(tstamp) => Value::from(tstamp.year()), @@ -976,7 +975,6 @@ impl EvalExpr for EvalFnExtractMonth { let value = self.value.evaluate(bindings, ctx); let result = match value.borrow() { Null => Null, - Missing => Missing, Value::DateTime(dt) => match dt.as_ref() { DateTime::Date(d) => Value::from(d.month() as u8), DateTime::Timestamp(tstamp) => Value::from(tstamp.month() as u8), @@ -990,7 +988,7 @@ impl EvalExpr for EvalFnExtractMonth { } } -/// Represents a day `EXTRACT` function, e.g. `extract(DAT FROM t)`. +/// Represents a day `EXTRACT` function, e.g. `extract(DAY FROM t)`. #[derive(Debug)] pub struct EvalFnExtractDay { pub value: Box, @@ -1002,7 +1000,6 @@ impl EvalExpr for EvalFnExtractDay { let value = self.value.evaluate(bindings, ctx); let result = match value.borrow() { Null => Null, - Missing => Missing, Value::DateTime(dt) => match dt.as_ref() { DateTime::Date(d) => Value::from(d.day()), DateTime::Timestamp(tstamp) => Value::from(tstamp.day()), @@ -1028,7 +1025,6 @@ impl EvalExpr for EvalFnExtractHour { let value = self.value.evaluate(bindings, ctx); let result = match value.borrow() { Null => Null, - Missing => Missing, Value::DateTime(dt) => match dt.as_ref() { DateTime::Time(t) => Value::from(t.hour()), DateTime::TimeWithTz(t, _) => Value::from(t.hour()), @@ -1054,7 +1050,6 @@ impl EvalExpr for EvalFnExtractMinute { let value = self.value.evaluate(bindings, ctx); let result = match value.borrow() { Null => Null, - Missing => Missing, Value::DateTime(dt) => match dt.as_ref() { DateTime::Time(t) => Value::from(t.minute()), DateTime::TimeWithTz(t, _) => Value::from(t.minute()), @@ -1074,27 +1069,30 @@ pub struct EvalFnExtractSecond { pub value: Box, } +fn total_seconds(second: u8, nanosecond: u32) -> Value { + let result = rust_decimal::Decimal::from_f64(((second as f64 * 1e9) + nanosecond as f64) / 1e9) + .expect("time as decimal"); + Value::from(result) +} + impl EvalExpr for EvalFnExtractSecond { - // TODO: doesn't currently extract fractional seconds #[inline] fn evaluate<'a>(&'a self, bindings: &'a Tuple, ctx: &'a dyn EvalContext) -> Cow<'a, Value> { let value = self.value.evaluate(bindings, ctx); - let (second, nanosecond) = match value.borrow() { - Null => return Cow::Owned(Null), - Missing => return Cow::Owned(Missing), + let result = match value.borrow() { + Null => Null, Value::DateTime(dt) => match dt.as_ref() { - DateTime::Time(t) => (t.second(), t.nanosecond()), - DateTime::TimeWithTz(t, _) => (t.second(), t.nanosecond()), - DateTime::Timestamp(tstamp) => (tstamp.second(), tstamp.nanosecond()), - DateTime::TimestampWithTz(tstamp) => (tstamp.second(), tstamp.nanosecond()), - DateTime::Date(_) => return Cow::Owned(Missing), + DateTime::Time(t) => total_seconds(t.second(), t.nanosecond()), + DateTime::TimeWithTz(t, _) => total_seconds(t.second(), t.nanosecond()), + DateTime::Timestamp(tstamp) => total_seconds(tstamp.second(), tstamp.nanosecond()), + DateTime::TimestampWithTz(tstamp) => { + total_seconds(tstamp.second(), tstamp.nanosecond()) + } + DateTime::Date(_) => Missing, }, - _ => return Cow::Owned(Missing), + _ => Missing, }; - let result = - rust_decimal::Decimal::from_f64(((second as f64 * 1e9) + nanosecond as f64) / 1e9) - .expect("time as decimal"); - Cow::Owned(Value::from(result)) + Cow::Owned(result) } } @@ -1110,7 +1108,6 @@ impl EvalExpr for EvalFnExtractTimezoneHour { let value = self.value.evaluate(bindings, ctx); let result = match value.borrow() { Null => Null, - Missing => Missing, Value::DateTime(dt) => match dt.as_ref() { DateTime::TimeWithTz(_, tz) => Value::from(tz.whole_hours()), DateTime::TimestampWithTz(tstamp) => Value::from(tstamp.offset().whole_hours()), @@ -1136,7 +1133,6 @@ impl EvalExpr for EvalFnExtractTimezoneMinute { let value = self.value.evaluate(bindings, ctx); let result = match value.borrow() { Null => Null, - Missing => Missing, Value::DateTime(dt) => match dt.as_ref() { DateTime::TimeWithTz(_, tz) => Value::from(tz.minutes_past_hour()), DateTime::TimestampWithTz(tstamp) => { From 73a47be01da176ae257089f479e0bb0bf0dcd58a Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Mon, 17 Apr 2023 15:27:00 -0700 Subject: [PATCH 5/5] Fix timezone_minute calculation; add more details to calldef comment --- partiql-conformance-tests/tests/test_value.rs | 2 +- partiql-eval/src/eval/expr/mod.rs | 2 +- partiql-logical-planner/src/call_defs.rs | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/partiql-conformance-tests/tests/test_value.rs b/partiql-conformance-tests/tests/test_value.rs index e4d517fb..179f57d0 100644 --- a/partiql-conformance-tests/tests/test_value.rs +++ b/partiql-conformance-tests/tests/test_value.rs @@ -214,7 +214,7 @@ fn parse_test_value_datetime(reader: &mut Reader) -> DateTime { ts.hour() as u8, ts.minute() as u8, ts.second() as u8, - ts.nanoseconds() as u32, + ts.nanoseconds(), offset, ) } diff --git a/partiql-eval/src/eval/expr/mod.rs b/partiql-eval/src/eval/expr/mod.rs index 001b1f3f..a6df7f7f 100644 --- a/partiql-eval/src/eval/expr/mod.rs +++ b/partiql-eval/src/eval/expr/mod.rs @@ -1136,7 +1136,7 @@ impl EvalExpr for EvalFnExtractTimezoneMinute { Value::DateTime(dt) => match dt.as_ref() { DateTime::TimeWithTz(_, tz) => Value::from(tz.minutes_past_hour()), DateTime::TimestampWithTz(tstamp) => { - Value::from(tstamp.offset().minutes_past_hour() % 60) + Value::from(tstamp.offset().minutes_past_hour()) } DateTime::Date(_) => Missing, DateTime::Time(_) => Missing, diff --git a/partiql-logical-planner/src/call_defs.rs b/partiql-logical-planner/src/call_defs.rs index ae9f12ec..fc459049 100644 --- a/partiql-logical-planner/src/call_defs.rs +++ b/partiql-logical-planner/src/call_defs.rs @@ -451,7 +451,7 @@ fn function_call_def_extract() -> CallDef { CallSpecArg::Named("from".into()), ], output: Box::new(|mut args| { - args.remove(0); // remove first default synthesized argument + args.remove(0); // remove first default synthesized argument from parser preprocessor logical::ValueExpr::Call(logical::CallExpr { name: logical::CallName::ExtractYear, arguments: args, @@ -464,7 +464,7 @@ fn function_call_def_extract() -> CallDef { CallSpecArg::Named("from".into()), ], output: Box::new(|mut args| { - args.remove(0); // remove first default synthesized argument + args.remove(0); // remove first default synthesized argument from parser preprocessor logical::ValueExpr::Call(logical::CallExpr { name: logical::CallName::ExtractMonth, arguments: args, @@ -477,7 +477,7 @@ fn function_call_def_extract() -> CallDef { CallSpecArg::Named("from".into()), ], output: Box::new(|mut args| { - args.remove(0); // remove first default synthesized argument + args.remove(0); // remove first default synthesized argument from parser preprocessor logical::ValueExpr::Call(logical::CallExpr { name: logical::CallName::ExtractDay, arguments: args, @@ -490,7 +490,7 @@ fn function_call_def_extract() -> CallDef { CallSpecArg::Named("from".into()), ], output: Box::new(|mut args| { - args.remove(0); // remove first default synthesized argument + args.remove(0); // remove first default synthesized argument from parser preprocessor logical::ValueExpr::Call(logical::CallExpr { name: logical::CallName::ExtractHour, arguments: args, @@ -503,7 +503,7 @@ fn function_call_def_extract() -> CallDef { CallSpecArg::Named("from".into()), ], output: Box::new(|mut args| { - args.remove(0); // remove first default synthesized argument + args.remove(0); // remove first default synthesized argument from parser preprocessor logical::ValueExpr::Call(logical::CallExpr { name: logical::CallName::ExtractMinute, arguments: args, @@ -516,7 +516,7 @@ fn function_call_def_extract() -> CallDef { CallSpecArg::Named("from".into()), ], output: Box::new(|mut args| { - args.remove(0); // remove first default synthesized argument + args.remove(0); // remove first default synthesized argument from parser preprocessor logical::ValueExpr::Call(logical::CallExpr { name: logical::CallName::ExtractSecond, arguments: args, @@ -529,7 +529,7 @@ fn function_call_def_extract() -> CallDef { CallSpecArg::Named("from".into()), ], output: Box::new(|mut args| { - args.remove(0); // remove first default synthesized argument + args.remove(0); // remove first default synthesized argument from parser preprocessor logical::ValueExpr::Call(logical::CallExpr { name: logical::CallName::ExtractTimezoneHour, arguments: args, @@ -542,7 +542,7 @@ fn function_call_def_extract() -> CallDef { CallSpecArg::Named("from".into()), ], output: Box::new(|mut args| { - args.remove(0); // remove first default synthesized argument + args.remove(0); // remove first default synthesized argument from parser preprocessor logical::ValueExpr::Call(logical::CallExpr { name: logical::CallName::ExtractTimezoneMinute, arguments: args,