Skip to content

Commit

Permalink
feat: tidy up defhierarchy linting, add hash-expr-fn support,
Browse files Browse the repository at this point in the history 
remove defdata, other options cleanup and minor refactor
  • Loading branch information
@k13gomez
k13gomez committed Aug 29, 2024
1 parent 991ec78 commit 06a8b53
Show file tree
Hide file tree
Showing 9 changed files with 89 additions and 151 deletions.
3 changes: 1 addition & 2 deletions clj-kondo/clj-kondo.exports/clara/rules/config.edn
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
{:lint-as {clara.rules/defsession clojure.core/def
clara.rules/defhierarchy clojure.core/def
clara.rules/defdata clojure.core/def
clara.rules.platform/eager-for clojure.core/for
clara.rules.platform/compute-for clojure.core/for}
:hooks {:analyze-call {clara.rules/defquery hooks.clara-rules/analyze-defquery-macro
clara.rules/defrule hooks.clara-rules/analyze-defrule-macro
clara.rules/defhierarchy hooks.clara-rules/analyze-defhierarchy-macro
clara.rules.dsl/parse-query hooks.clara-rules/analyze-parse-query-macro
clara.rules.dsl/parse-rule hooks.clara-rules/analyze-parse-rule-macro
clara.tools.testing-utils/def-rules-test hooks.clara-rules/analyze-def-rules-test-macro}}}
21 changes: 21 additions & 0 deletions clj-kondo/clj-kondo.exports/clara/rules/hooks/clara_rules.clj_kondo
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,27 @@
merge {:clj-kondo/ignore [:clojure-lsp/unused-public-var]})]
{:node new-node}))

(defn analyze-defhierarchy-macro
"analyze clara-rules defhierarchy macro"
[{:keys [:node]}]
(let [[var-name & children] (rest (:children node))
doc-str (when (= :token (node-type (first children)))
(first children))
children (if doc-str (rest children) children)
new-node (vary-meta
(api/list-node
(list
(api/token-node 'def)
(if doc-str
(vary-meta var-name merge {:doc doc-str})
var-name)
(api/list-node
(list*
(api/token-node 'do)
children))))
merge {:clj-kondo/ignore [:clojure-lsp/unused-public-var]})]
{:node new-node}))

(defn analyze-def-rules-test-macro
[{:keys [:node]}]
(let [[test-name test-params & test-body] (rest (:children node))
Expand Down
21 changes: 10 additions & 11 deletions dev/user.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
quick-benchmark] :as crit]
[clara.rules.platform :refer [compute-for]]
[clojure.core.async :refer [go timeout <!]]
[clara.rules :refer [defrule defquery defdata defhierarchy
insert! insert insert-all fire-rules query
[clara.rules :refer [defrule defquery defhierarchy
insert! insert-all! insert insert-all fire-rules query
mk-session clear-ns-vars!
derive! underive!]]
[clara.rules.compiler :as com]
Expand Down Expand Up @@ -37,15 +37,14 @@
[]
[?output <- :thing/result])

(defdata foo
{:type :thing/foo
:value 1})

(defdata bar
[{:type :thing/bar
:value 2}
{:type :thing/bar
:value 3}])
(defrule default-data
(insert-all!
[{:type :thing/foo
:value 1}
{:type :thing/bar
:value 2}
{:type :thing/bar
:value 3}]))

(time
(-> (mk-session 'user :fact-type-fn :type)
Expand Down
44 changes: 0 additions & 44 deletions src/main/clojure/clara/rules.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,39 +141,6 @@
[(afn)]))

(extend-type clojure.lang.Symbol
com/IFactSource
(load-facts [sym]
;; Find the facts in the namespace, shred them,
;; and compile them into a rule base.
(if (namespace sym)
;; The symbol is qualified, so load hierarchies in the qualified symbol.
(let [resolved (resolve sym)]
(when (nil? resolved)
(throw (ex-info (str "Unable to resolve fact source: " sym) {:sym sym})))

(cond
;; The symbol references a fact, so just return it
(:hierarchy (meta resolved))
(com/load-facts-from-source @resolved)

;; The symbol references a sequence, so ensure we load all sources.
(sequential? @resolved)
(mapcat com/load-facts-from-source @resolved)

:else
[]))

;; The symbol is not qualified, so treat it as a namespace.
(->> (ns-interns sym)
(vals) ; Get the references in the namespace.
(filter var?)
(filter (comp (some-fn :fact :fact-seq) meta)) ; Filter down to fact and fact-seq, and seqs of both.
;; If definitions are created dynamically (i.e. are not reflected in an actual code file)
;; it is possible that they won't have :line metadata, so we have a default of 0.
(sort (fn [v1 v2]
(compare (or (:line (meta v1)) 0)
(or (:line (meta v2)) 0))))
(mapcat com/load-facts-from-source))))
com/IHierarchySource
(load-hierarchies [sym]
;; Find the hierarchies in the namespace, shred them,
Expand Down Expand Up @@ -372,17 +339,6 @@
(binding [hierarchy/*hierarchy* (atom (hierarchy/make-hierarchy))]
~@body))))

(defmacro defdata
"Defines a data fact which is stored in the given var. For instance, the following fact is simply a
map which is then inserted into the session when the namespace is loaded.
(defdata default-temperature
(Cold. 32))"
[name & body]
(let [doc (if (string? (first body)) (first body) nil)]
`(def ~(vary-meta name assoc :fact true :doc doc)
~@body)))

(defmacro clear-ns-vars!
"Ensures that any rule/query definitions which have been cached will be cleared from the associated namespace.
Rule and query definitions can be cached such that if their definitions are not explicitly overwritten with the same
Expand Down
134 changes: 49 additions & 85 deletions src/main/clojure/clara/rules/compiler.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,6 @@
(defprotocol IRuleSource
(load-rules [source]))

(defprotocol IFactSource
(load-facts [source]))

(defprotocol IHierarchySource
(load-hierarchies [source]))

Expand Down Expand Up @@ -113,11 +110,6 @@
;; A map of [node-id field-name] to function.
node-expr-fn-lookup :- schema/NodeFnLookup])

(defn- md5-hash
"Returns the md5 digest of the given data after converting it to a string"
[x]
(digest/md5 ^String (pr-str x)))

(defn- is-variable?
"Returns true if the given expression is a variable (a symbol prefixed by ?)"
[expr]
Expand Down Expand Up @@ -1520,17 +1512,54 @@
(:id-to-condition-node beta-graph))]
(persistent! id->expr)))

(def forms-per-eval-default
"The default max number of forms that will be evaluated together as a single batch.
5000 is chosen here due to the way that clojure will evaluate the vector of forms extracted from the nodes.
The limiting factor here is the max java method size (64KiB), clojure will compile each form in the vector down into
its own class file and generate another class file that will reference each of the other functions and wrap them in
a vector inside a static method. For example,
(eval [(fn one [_] ...) (fn two [_] ...)])
would generate 3 classes.
some_namespace$eval1234
some_namespace$eval1234$one_1234
some_namespace$eval1234$two_1235
some_namespace$eval1234$one_1234 and some_namespace$eval1234$two_1235 contian the implementation of the functions,
where some_namespace$eval1234 will contain two methods, invoke and invokeStatic.
The invokeStatic method breaks down into something similar to a single create array call followed by 2 array set calls
with new invocations on the 2 classes the method then returns a new vector created from the array.
5000 is lower than the absolute max to allow for modifications to how clojure compiles without needing to modify this.
The current limit should be 5471, this is derived from the following opcode investigation:
Array creation: 5B
Creating and populating the first 6 elements of the array: 60B
Creating and populating the next 122 elements of the array: 1,342B
Creating and populating the next 5343 elements of the array: 64,116B
Creating the vector and the return statement: 4B
This sums to 65,527B just shy of the 65,536B method size limit."
5000)

(sc/defn compile-exprs :- schema/NodeFnLookup
"Takes a map in the form produced by extract-exprs and evaluates the values(expressions) of the map in a batched manner.
This allows the eval calls to be more effecient, rather than evaluating each expression on its own.
See #381 for more details."
[key->expr :- schema/NodeExprLookup
expr-cache :- (sc/maybe sc/Any)
partition-size :- sc/Int]
(let [prepare-expr (fn do-prepare-expr
options :- {sc/Keyword sc/Any}]
(let [expr-cache (or (:compiler-cache options)
default-compiler-cache)
forms-per-eval (or (:forms-per-eval options)
forms-per-eval-default)
hash-expr-fn (or (:hash-expr-fn options)
hash)
prepare-expr (fn do-prepare-expr
[[expr-key [expr compilation-ctx]]]
(if expr-cache
(let [cache-key (str (md5-hash expr) (md5-hash compilation-ctx))
(let [cache-key [(hash-expr-fn expr)
(hash-expr-fn compilation-ctx)]
compilation-ctx (assoc compilation-ctx :cache-key cache-key)
compiled-handler (some-> compilation-ctx :compile-ctx :production :handler resolve)
compiled-expr (or compiled-handler
Expand Down Expand Up @@ -1578,7 +1607,7 @@
(for [[nspace ns-expr-group] (sort-by key (group-by (comp :ns second val) key->expr))
;; Partitioning the number of forms to be evaluated, Java has a limit to the size of methods if we were
;; evaluate all expressions at once it would likely exceed this limit and throw an exception.
expr-batch (partition-all partition-size ns-expr-group) ;;;; [<node-expr-keys> [<expr> <ctx>]]
expr-batch (partition-all forms-per-eval ns-expr-group) ;;;; [<node-expr-keys> [<expr> <ctx>]]
:let [grouped-exprs (->> (mapv prepare-expr expr-batch)
(group-by first))
prepared-exprs (map second (:prepared grouped-exprs))
Expand Down Expand Up @@ -2016,37 +2045,6 @@
productions
(throw (ex-info (str "Non-unique production names: " non-unique) {:names non-unique})))))

(def forms-per-eval-default
"The default max number of forms that will be evaluated together as a single batch.
5000 is chosen here due to the way that clojure will evaluate the vector of forms extracted from the nodes.
The limiting factor here is the max java method size (64KiB), clojure will compile each form in the vector down into
its own class file and generate another class file that will reference each of the other functions and wrap them in
a vector inside a static method. For example,
(eval [(fn one [_] ...) (fn two [_] ...)])
would generate 3 classes.
some_namespace$eval1234
some_namespace$eval1234$one_1234
some_namespace$eval1234$two_1235
some_namespace$eval1234$one_1234 and some_namespace$eval1234$two_1235 contian the implementation of the functions,
where some_namespace$eval1234 will contain two methods, invoke and invokeStatic.
The invokeStatic method breaks down into something similar to a single create array call followed by 2 array set calls
with new invocations on the 2 classes the method then returns a new vector created from the array.
5000 is lower than the absolute max to allow for modifications to how clojure compiles without needing to modify this.
The current limit should be 5471, this is derived from the following opcode investigation:
Array creation: 5B
Creating and populating the first 6 elements of the array: 60B
Creating and populating the next 122 elements of the array: 1,342B
Creating and populating the next 5343 elements of the array: 64,116B
Creating the vector and the return statement: 4B
This sums to 65,527B just shy of the 65,536B method size limit."
5000)

(def omit-compile-ctx-default
"During construction of the Session there is data maintained such that if the underlying expressions fail to compile
then this data can be used to explain the failure and the constraints of the rule who's expression is being evaluated.
Expand All @@ -2059,22 +2057,18 @@
(sc/defn mk-session*
"Compile the rules into a rete network and return the given session."
[productions :- #{schema/Production}
facts :- [sc/Any]
options :- {sc/Keyword sc/Any}]
(validate-names-unique productions)
(let [;; A stateful counter used for unique ids of the nodes of the graph.
id-counter (atom 0)
create-id-fn (fn [] (swap! id-counter inc))

compiler-cache (:compiler-cache options default-compiler-cache)
forms-per-eval (:forms-per-eval options forms-per-eval-default)

beta-graph (to-beta-graph productions create-id-fn)
alpha-graph (to-alpha-graph beta-graph create-id-fn)

;; Extract the expressions from the graphs and evaluate them in a batch manner.
;; This is a performance optimization, see Issue 381 for more information.
exprs (compile-exprs (extract-exprs beta-graph alpha-graph) compiler-cache forms-per-eval)
exprs (compile-exprs (extract-exprs beta-graph alpha-graph) options)

;; If we have made it to this point, it means that we have succeeded in compiling all expressions
;; thus we can free the :compile-ctx used for troubleshooting compilation failures.
Expand Down Expand Up @@ -2122,7 +2116,7 @@
:transport transport
:listeners (get options :listeners [])
:get-alphas-fn get-alphas-fn})]
(eng/insert session facts)))
session))

(defn add-production-load-order
"Adds ::rule-load-order to metadata of productions. Custom DSL's may need to use this if
Expand Down Expand Up @@ -2154,34 +2148,6 @@

:else []))

(defn load-facts-from-source
"loads the hierarchies from a source if it implements `IRuleSource`, or navigates inside
collections to load from vectors, lists, sets, seqs."
[source]
(cond
(u/instance-satisfies? IFactSource source)
(load-facts source)

(or (vector? source)
(list? source)
(set? source)
(seq? source))
(mapcat load-facts-from-source source)

(var? source)
(load-facts-from-source @source)

(fn? source) ;;; source is a rule fn so it can't also be a fact unless explicitly inserted
[]

(:hierarchy-data source) ;;; source is a hierarchy so it can't also be a fact unless explicitly inserted
[]

(:lhs source) ;;; source is a production so it can't also be a fact unless explicitly inserted
[]

:else [source]))

(defn load-hierarchies-from-source
"loads the hierarchies from a source if it implements `IRuleSource`, or navigates inside
collections to load from vectors, lists, sets, seqs."
Expand Down Expand Up @@ -2230,8 +2196,6 @@
(:hierarchy options) (cons (:hierarchy options)))
hierarchy (when (seq hierarchies-loaded)
(reduce reduce-hierarchy (hierarchy/make-hierarchy) hierarchies-loaded))
facts (->> (mapcat load-facts-from-source sources)
(vec))
options (cond-> options
(some? hierarchy)
(assoc :hierarchy hierarchy))
Expand All @@ -2242,12 +2206,12 @@
(nil? options-cache)
default-session-cache
:else options-cache)
hash-expr-fn (or (:hash-expr-fn options) hash)
;;; this is simpler than storing all the productions and options in the cache
session-key (str (md5-hash productions-sorted)
(md5-hash (dissoc options :cache :compiler-cache))
(hash facts))]
session-key [(hash-expr-fn productions-sorted)
(hash-expr-fn (dissoc options :cache :compiler-cache))]]
(if session-cache
(cache/lookup-or-miss session-cache session-key
(fn do-mk-session [_]
(mk-session* productions-sorted facts options)))
(mk-session* productions-sorted facts options)))))
(mk-session* productions-sorted options)))
(mk-session* productions-sorted options)))))
6 changes: 4 additions & 2 deletions src/main/clojure/clara/rules/dsl.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
(defn split-lhs-rhs
"Given a rule with the =>, splits the left- and right-hand sides."
[rule-body]
(let [[lhs [sep & rhs]] (split-with #(not (separator? %)) rule-body)]

(let [[pre [sep & post]] (split-with (complement separator?) rule-body)
[lhs rhs] (if sep
[pre post]
['() pre])]
{:lhs lhs
:rhs (when-not (empty? rhs)
(conj rhs 'do))}))
Expand Down
1 change: 1 addition & 0 deletions src/main/clojure/clara/rules/durability.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,7 @@
Options for the rulebase semantics that are documented at clara.rules/mk-session include:
* :fact-type-fn
* :hash-expr-fn
* :ancestors-fn
* :activation-group-sort-fn
* :activation-group-fn
Expand Down
Loading

0 comments on commit 06a8b53

Please sign in to comment.