Lift tech.v3.datatype.functional operations (#90)

* Add ->general-types function * Add a general type :logical * Use type hierarchy in tablecloth.api.utils for `typeof` functions * Add column dev branch to pr workflow * Add tests for typeof * Fix tests for typeof * Return the concrete type from `typeof` * Simplify `concrete-types` fn * Optimize ->general-types by using static lookup * Adjust fns listing types * We decided that the default meaning of type points to the "concrete" type, and not the general type. * So `types` now returns the set of concrete types and `general-types` returns the general types. * Revert "Adjust fns listing types" This reverts commit d93e34f. * Fix `typeof` test to test for concerete types * Reorganize `typeof?` tests * Reword docstring for `typeof?` slightly * Update column api template and add missing `typeof?` * Add commment to `general-types-lookup` * Improve `->general-types` docstring * Add `general-types` fn that returns sets of general types * Adjust util `types` fn to return concrete types * Save changes to column api.clj * Save ongoing experiments with lifting * Save ongoing work on lifting * Adjust lift-ops-1 to handle any number of args with rest arg * Working `rearrange-args` fn * Save work actually writing lifted fns * Saving first attempt to writer operators * Add `percentiiles test * Adjust `rearrange-args to take new-args in option map * Unify two lift functions * Add in docstrings when present * Move lift utils into utils ns * Rename lifting namespaces * Lift some more fns * Make exclusions for ns header helper an arg * Add new operators and tests * Add ops with lhs rhs arg pattern * Lift '* * Add require to operators ns for utils * Update test to make it more complete * Lift `equals * Make test more accurate * Reorganize tests * Fix grammar * Lift 'shift * Uncomment 'or test * Lift 'normalize op * Life 'magnitude * Lifting bit manipulation ops * lift ieee-remainder * Lifting more functions * Add excludes * Lift a bunch of new functions * Alphebetize some lists * More alphebitization * Clean up * Instead of using `col` as arg conform to using `x & and `y * Temporarily disable failing test fix in 7.000-beta23 * Disable the correct test * Just some minor cleanup in op tests * Some more cleanup/reorg in op tests * Update generated operators namespace with switch from col -> x etc * Lift 'descriptive-statistics * Fix messed up test layout * Lift 'quartiles * Lift 'fill-range and a bunch of reduce operations * Lift 'mean-fast 'sum-fast 'magnitude-squared * Lift correlation fns kendalls, pearsons, and spearmans * Lift cumulative ops * cleanup
scicloj · Feb 10, 2023 · 5dc5065 · 5dc5065
1 parent 704156a
commit 5dc5065
Show file tree

Hide file tree

Showing 5 changed files with 2,197 additions and 3 deletions.
diff --git a/src/tablecloth/column/api/lift_operators.clj b/src/tablecloth/column/api/lift_operators.clj
@@ -0,0 +1,195 @@
+(ns tablecloth.column.api.lift-operators
+  (:require [tablecloth.column.api.utils :refer [do-lift lift-op]]))
+
+(def serialized-lift-fn-lookup
+  {['*
+    '+
+    '-
+    '/
+    '<
+    '<=
+    '>
+    '>=
+    'abs
+    'acos
+    'and
+    'asin
+    'atan
+    'atan2
+    'bit-and
+    'bit-and-not
+    'bit-clear
+    'bit-flip
+    'bit-not
+    'bit-or
+    'bit-set
+    'bit-shift-left
+    'bit-shift-right
+    #_bit-test ;; can't get this to work yet.
+    'bit-xor
+    'cbrt
+    'ceil
+    'cos
+    'cosh
+    'distance
+    'distance-squared
+    'dot-product
+    'eq
+    'equals
+    'exp
+    'expm1
+    'floor
+    'get-significand
+    'hypot
+    'identity
+    'ieee-remainder
+    'log
+    'log10
+    'log1p
+    'logistic
+    'magnitude
+    'max
+    'min
+    'next-down
+    'next-up
+    'normalize
+    'not-eq
+    'or
+    'pow
+    'quot
+    'rem
+    'rint
+    'signum
+    'sin
+    'sinh
+    'sq
+    'sqrt
+    'tan
+    'tanh
+    'to-degrees
+    'to-radians
+    'ulp
+    'unsigned-bit-shift-right] lift-op
+   ['kurtosis
+    'sum
+    'mean
+    'skew
+    'variance
+    'standard-deviation
+    'quartile-3
+    'quartile-1
+    'median] (fn [fn-sym fn-meta]
+              (lift-op
+               fn-sym fn-meta
+               {:new-args '([col] [col options])
+                :new-args-lookup {'data 'col
+                                  'options 'options}}))
+   ['even?
+    'finite?
+    'infinite?
+    'mathematical-integer?
+    'nan?
+    'neg?
+    'not
+    'odd?
+    'pos?
+    'round
+    'zero?]
+   (fn [fn-sym fn-meta]
+             (lift-op
+              fn-sym fn-meta
+              {:new-args '([x] [x options])
+               :new-args-lookup {'arg 'x
+                                 'options 'options}}))
+   ['percentiles] (fn [fn-sym fn-meta]
+                    (lift-op
+                     fn-sym fn-meta
+                     {:new-args '([x percentiles] [x percentiles options])
+                      :new-args-lookup {'data 'x,
+                                        'percentages 'percentiles,
+                                        'options 'options}}))
+   ['shift] (fn [fn-sym fn-meta]
+              (lift-op
+               fn-sym fn-meta
+               {:new-args '([x n])
+                :new-args-lookup {'rdr 'x
+                                  'n 'n}}))
+   ['descriptive-statistics] (fn [fn-sym fn-meta]
+                              (lift-op
+                               fn-sym fn-meta
+                               {:new-args '([x stats-names stats-data options]
+                                            [x stats-names options]
+                                            [x stats-names]
+                                            [x])
+                                :new-args-lookup {'rdr 'x
+                                                  'src-rdr 'x
+                                                  'stats-names 'stats-names
+                                                  'stats-data 'stats-data
+                                                  'options 'options}}))
+   ['quartiles] (fn [fn-sym fn-meta]
+                  (lift-op
+                   fn-sym fn-meta
+                   {:new-args '([x options] [x])
+                    :new-args-lookup {'item 'x
+                                      'options 'options}}))
+   ['fill-range] (fn [fn-sym fn-meta]
+                  (lift-op
+                   fn-sym fn-meta
+                   {:new-args '([x max-span])
+                    :new-args-lookup {'numeric-data 'x
+                                      'max-span 'max-span}}))
+   ['reduce-min
+    'reduce-max
+    'reduce-*
+    'reduce-+] (fn [fn-sym fn-meta]
+                 (lift-op
+                  fn-sym fn-meta
+                  {:new-args '([x])
+                   :new-args-lookup {'rdr 'x}}))
+  ['mean-fast
+   'sum-fast
+   'magnitude-squared] (fn [fn-sym fn-meta]
+                         (lift-op
+                          fn-sym fn-meta
+                          {:new-args '([x])
+                           :new-args-lookup {'data 'x}}))
+   ['kendalls-correlation
+    'pearsons-correlation
+    'spearmans-correlation] (fn [fn-sym fn-meta]
+                              (lift-op
+                               fn-sym fn-meta
+                               {:new-args '([x y] [x y options])
+                                :new-args-lookup {'lhs 'x
+                                                  'rhs 'y
+                                                  'options 'options}}))
+   ['cumprod
+    'cumsum
+    'cummax
+    'cummin] (fn [fn-sym fn-meta]
+                (lift-op
+                 fn-sym fn-meta
+                 {:new-args '([x] [x options])
+                  :new-args-lookup {'data 'x
+                                    'options 'options}}))})
+
+
+(defn deserialize-lift-fn-lookup []
+  (reduce (fn [m [symlist liftfn]]
+            (loop [syms symlist
+                   result m]
+              (if (empty? syms)
+                result
+                (recur (rest syms) (assoc result (first syms) liftfn)))))
+          {}
+          serialized-lift-fn-lookup))
+
+(comment
+  (do-lift (deserialize-lift-fn-lookup)
+           'tablecloth.column.api.operators
+           'tech.v3.datatype.functional
+           '[* + - / < <= > >= abs and bit-and bit-and-not bit-clear bit-flip
+             bit-not bit-or bit-set bit-shift-left bit-shift-right bit-test bit-xor
+             even? identity infinite? max min neg? not odd? odd? or pos? quot rem
+             unsigned-bit-shift-right zero?]
+           "src/tablecloth/column/api/operators.clj")
+  ,)