From c633fa6d9de3938be0f99f09a6db68053c432ba3 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Thu, 23 Mar 2017 00:14:07 +0100 Subject: [PATCH 01/34] Add a konserve backend implementation. --- project.clj | 6 +- src/hitchhiker/konserve.clj | 159 ++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/hitchhiker/konserve.clj diff --git a/project.clj b/project.clj index c3b1d2c..7ad6cb3 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,11 @@ [org.clojure/core.memoize "0.5.8"] [com.taoensso/carmine "2.12.2"] [org.clojure/core.rrb-vector "0.0.11"] - [org.clojure/core.cache "0.6.5"]] + [org.clojure/core.cache "0.6.5"] + + [io.replikativ/incognito "0.2.2-SNAPSHOT"] + [io.replikativ/konserve "0.4.8"] + [io.replikativ/superv.async "0.2.5"]] :aliases {"bench" ["with-profile" "profiling" "run" "-m" "hitchhiker.bench"]} :jvm-opts ["-server" "-Xmx3700m" "-Xms3700m"] :profiles {:test diff --git a/src/hitchhiker/konserve.clj b/src/hitchhiker/konserve.clj new file mode 100644 index 0000000..5498a3a --- /dev/null +++ b/src/hitchhiker/konserve.clj @@ -0,0 +1,159 @@ +(ns hitchhiker.konserve + (:require [clojure.core.rrb-vector :refer [catvec subvec]] + [clojure.core.async :refer [chan ( (assoc node :storage-addr nil) + (update :children (fn [cs] (mapv #(assoc % :store nil + :storage-addr nil) cs)))) + (assoc node :storage-addr nil))] + (try + (let [id (uuid pnode)] + (KonserveAddr store (core/last-key node) id (synthesize-storage-addr id))) + (catch Exception e + (prn e) + (prn pnode) + (throw e)))) + #_(let [key (str (java.util.UUID/randomUUID)) + addr (redis-addr (core/last-key node) key)] + ;(.submit service #(wcar {} (car/set key node))) + (when (some #(not (satisfies? msg/IOperation %)) (:op-buf node)) + (println (str "Found a broken node, has " (count (:op-buf node)) " ops")) + (println (str "The node data is " node)) + (println (str "and " (:op-buf node)))) + (wcar {} + (car/set key node) + (when (core/index-node? node) + (add-refs key + (for [child (:children node) + :let [child-key @(:storage-addr child)]] + child-key)))) + (seed-cache! key (doto (promise) (deliver node))) + addr)) + (delete-addr [_ addr session] + #_(wcar {} (car/del addr)) + (swap! session update-in :deletes inc))) + +(defn get-root-key + [tree] + (-> tree :storage-addr (deref 10 nil))) + +(defn create-tree-from-root-key + [store root-key] + (let [val (KonserveAddr store last-key root-key (synthesize-storage-addr root-key))))) + +(comment + + + (def store + (let [store ( % map->KonserveAddr (assoc :store store + :storage-addr (synthesize-storage-addr (:konserve-key %)))) + 'hitchhiker.tree.core.DataNode + (fn [{:keys [children cfg]}] + (core/->DataNode (into (sorted-map-by + compare) children) + (promise) + cfg)) + 'hitchhiker.tree.core.IndexNode + (fn [{:keys [children cfg op-buf]}] + (core/->IndexNode (->> children + #_((fn [e] (prn "reading" e) e)) + catvec) + (promise) + (catvec op-buf) + cfg)) + 'hitchhiker.tree.messaging.InsertOp + msg/map->InsertOp + 'hitchhiker.tree.messaging.DeleteOp + msg/map->DeleteOp + 'hitchhiker.tree.core.Config + core/map->Config}) + store)) + + +(def my-tree (core/flush-tree + (time (reduce (fn [t i] +; ^{:break/when (> i 49)} + (msg/insert t i i)) + (core/b-tree (core/->Config 17 300 (- 300 17))) + (range 50000))) + (->KonserveBackend store) + )) + + + +(def my-tree-updated (core/flush-tree + (msg/delete (create-tree-from-root-key store (get-root-key (:tree my-tree))) 10) + (->KonserveBackend store) + )) + + + + +(time (msg/lookup (create-tree-from-root-key store (get-root-key (:tree my-tree))) 1001)) + +(reduce (fn [t i] + (let [st (System/currentTimeMillis) + tree (:tree (core/flush-tree (msg/insert t i i) + (->KonserveBackend store)))] + (when (= (mod i 100) 0) + (let [delta (- (System/currentTimeMillis) + st)] + (println "Op for" i " took " delta " ms"))) + tree)) + (core/b-tree (core/->Config 17 300 (- 300 17))) + (range 20000)) + + + + + + +) From a81c3aaf515016de16bab51058cc5fae8aed28e9 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 26 Mar 2017 04:28:14 +0200 Subject: [PATCH 02/34] Wrap by core.async, add konserve test. --- src/hitchhiker/konserve.clj | 161 ++++++------ src/hitchhiker/outboard.clj | 23 +- src/hitchhiker/redis.clj | 47 ++-- src/hitchhiker/tree/core.clj | 314 +++++++++++++----------- src/hitchhiker/tree/messaging.clj | 14 +- test/hitchhiker/konserve_test.clj | 70 ++++++ test/hitchhiker/tree/core_test.clj | 48 ++-- test/hitchhiker/tree/messaging_test.clj | 27 +- 8 files changed, 410 insertions(+), 294 deletions(-) create mode 100644 test/hitchhiker/konserve_test.clj diff --git a/src/hitchhiker/konserve.clj b/src/hitchhiker/konserve.clj index 5498a3a..7cc4793 100644 --- a/src/hitchhiker/konserve.clj +++ b/src/hitchhiker/konserve.clj @@ -1,7 +1,8 @@ (ns hitchhiker.konserve + (:refer-clojure :exclude [resolve subvec]) (:require [clojure.core.rrb-vector :refer [catvec subvec]] [clojure.core.async :refer [chan ( ( (assoc node :storage-addr nil) - (update :children (fn [cs] (mapv #(assoc % :store nil - :storage-addr nil) cs)))) - (assoc node :storage-addr nil))] - (try + (go-try S + (swap! session update-in [:writes] inc) + (let [pnode (if (core/index-node? node) + (-> (assoc node :storage-addr nil) + (update :children (fn [cs] (mapv #(assoc % :store nil + :storage-addr nil) cs)))) + (assoc node :storage-addr nil))] (let [id (uuid pnode)] - (KonserveAddr store (core/last-key node) id (synthesize-storage-addr id))) - (catch Exception e - (prn e) - (prn pnode) - (throw e)))) + (KonserveAddr store (core/last-key node) id (synthesize-storage-addr id))))) #_(let [key (str (java.util.UUID/randomUUID)) addr (redis-addr (core/last-key node) key)] ;(.submit service #(wcar {} (car/set key node))) @@ -77,81 +75,86 @@ (defn create-tree-from-root-key [store root-key] - (let [val (KonserveAddr store last-key root-key (synthesize-storage-addr root-key))))) + (go-try S + (let [val (KonserveAddr store last-key root-key (synthesize-storage-addr root-key)) + S))))) + +(defn add-read-handlers [store] + (swap! (:read-handlers store) merge + {'hitchhiker.konserve.KonserveAddr + #(-> % map->KonserveAddr + (assoc :store store + :storage-addr (synthesize-storage-addr (:konserve-key %)))) + 'hitchhiker.tree.core.DataNode + (fn [{:keys [children cfg]}] + (core/->DataNode (into (sorted-map-by + compare) children) + (promise) + cfg)) + 'hitchhiker.tree.core.IndexNode + (fn [{:keys [children cfg op-buf]}] + (core/->IndexNode (->> children + #_((fn [e] (prn "reading" e) e)) + catvec) + (promise) + (catvec op-buf) + cfg)) + 'hitchhiker.tree.messaging.InsertOp + msg/map->InsertOp + 'hitchhiker.tree.messaging.DeleteOp + msg/map->DeleteOp + 'hitchhiker.tree.core.Config + core/map->Config}) + store) (comment - (def store - (let [store ( % map->KonserveAddr (assoc :store store - :storage-addr (synthesize-storage-addr (:konserve-key %)))) - 'hitchhiker.tree.core.DataNode - (fn [{:keys [children cfg]}] - (core/->DataNode (into (sorted-map-by - compare) children) - (promise) - cfg)) - 'hitchhiker.tree.core.IndexNode - (fn [{:keys [children cfg op-buf]}] - (core/->IndexNode (->> children - #_((fn [e] (prn "reading" e) e)) - catvec) - (promise) - (catvec op-buf) - cfg)) - 'hitchhiker.tree.messaging.InsertOp - msg/map->InsertOp - 'hitchhiker.tree.messaging.DeleteOp - msg/map->DeleteOp - 'hitchhiker.tree.core.Config - core/map->Config}) - store)) - - -(def my-tree (core/flush-tree - (time (reduce (fn [t i] -; ^{:break/when (> i 49)} - (msg/insert t i i)) - (core/b-tree (core/->Config 17 300 (- 300 17))) - (range 50000))) - (->KonserveBackend store) - )) - - - -(def my-tree-updated (core/flush-tree - (msg/delete (create-tree-from-root-key store (get-root-key (:tree my-tree))) 10) - (->KonserveBackend store) - )) - - - - -(time (msg/lookup (create-tree-from-root-key store (get-root-key (:tree my-tree))) 1001)) + (def store (add-read-handlers ( i 49)} + (Config 17 300 (- 300 17)))) + (range 20000))) + (->KonserveBackend store) + ))) + + + + (def my-tree-updated (KonserveBackend store) + ))) + + + + + (time (KonserveBackend store)))] + tree (:tree (KonserveBackend store))))] (when (= (mod i 100) 0) (let [delta (- (System/currentTimeMillis) st)] (println "Op for" i " took " delta " ms"))) tree)) - (core/b-tree (core/->Config 17 300 (- 300 17))) + (Config 17 300 (- 300 17)))) (range 20000)) +(msg/lookup-fwd-iter (RedisBackend))))) + (:tree (RedisBackend)))))) (let [new-root (redis/get-root-key @tree-atom)] (wcar {} @@ -79,7 +80,7 @@ ", its already in use") {:used-name new-name}))) ;;TODO race condition where additional calls to create could all succeed ;;we should guard against this - (let [conn (->OutboardConnection (LinkedBlockingQueue.) (atom (core/b-tree (core/->Config 30 600 870))) (atom :running) (atom nil) new-name)] + (let [conn (->OutboardConnection (LinkedBlockingQueue.) (atom (Config 30 600 870)))) (atom :running) (atom nil) new-name)] (launch-outboard-processer! conn new-name) (swap! connection-registry assoc new-name conn) conn)) @@ -137,27 +138,27 @@ [snapshot k v & kvs] (let [tree snapshot] (if (and (seq kvs) (even? (count kvs))) - (loop [tree (msg/insert tree k v) + (loop [tree ( (totally-fetch redis-key) - (assoc :storage-addr (synthesize-storage-addr redis-key))))) + (resolve [_ S] + (go-try S + (-> (totally-fetch redis-key) + (assoc :storage-addr (synthesize-storage-addr redis-key)))))) (comment (:cfg (wcar {} (car/get "b89bb965-e584-45a2-9232-5b76bf47a21c"))) @@ -201,23 +204,24 @@ (wcar {} (add-to-expiry redis-key (+ 5000 (System/currentTimeMillis)))) node) (write-node [_ node session] - (swap! session update-in [:writes] inc) - (let [key (str (java.util.UUID/randomUUID)) - addr (redis-addr (core/last-key node) key)] - ;(.submit service #(wcar {} (car/set key node))) - (when (some #(not (satisfies? msg/IOperation %)) (:op-buf node)) - (println (str "Found a broken node, has " (count (:op-buf node)) " ops")) - (println (str "The node data is " node)) - (println (str "and " (:op-buf node)))) - (wcar {} - (car/set key node) - (when (core/index-node? node) - (add-refs key - (for [child (:children node) - :let [child-key @(:storage-addr child)]] - child-key)))) - (seed-cache! key (doto (promise) (deliver node))) - addr)) + (go-try S + (swap! session update-in [:writes] inc) + (let [key (str (java.util.UUID/randomUUID)) + addr (redis-addr (core/last-key node) key)] + ;(.submit service #(wcar {} (car/set key node))) + (when (some #(not (satisfies? msg/IOperation %)) (:op-buf node)) + (println (str "Found a broken node, has " (count (:op-buf node)) " ops")) + (println (str "The node data is " node)) + (println (str "and " (:op-buf node)))) + (wcar {} + (car/set key node) + (when (core/index-node? node) + (add-refs key + (for [child (:children node) + :let [child-key @(:storage-addr child)]] + child-key)))) + (seed-cache! key (doto (promise) (deliver node))) + addr))) (delete-addr [_ addr session] (wcar {} (car/del addr)) (swap! session update-in :deletes inc))) @@ -229,8 +233,9 @@ (defn create-tree-from-root-key [root-key] (let [last-key (core/last-key (wcar {} (car/get root-key)))] ; need last key to bootstrap - (core/resolve - (->RedisAddr last-key root-key (synthesize-storage-addr root-key))))) + (RedisAddr last-key root-key (synthesize-storage-addr root-key)) + S)))) (comment (wcar {} (car/ping) (car/set "foo" "bar") (car/get "foo")) diff --git a/src/hitchhiker/tree/core.clj b/src/hitchhiker/tree/core.clj index a9ee782..1e3179f 100644 --- a/src/hitchhiker/tree/core.clj +++ b/src/hitchhiker/tree/core.clj @@ -2,6 +2,8 @@ (:refer-clojure :exclude [compare resolve subvec]) (:require [clojure.core.rrb-vector :refer [catvec subvec]] [clojure.pprint :as pp] + [superv.async :refer [go-try S common-parent-path peek inc) - parent (-> common-parent-path pop peek) - new-sibling (resolve (nth (:children parent) next-index)) - ;; We must get back down to the data node - sibling-lineage (into [] - (take-while #(or (index-node? %) - (data-node? %))) - (iterate #(let [c (-> % :children first)] - (if (tree-node? c) - (resolve c) - c)) - new-sibling)) - path-suffix (-> (interleave sibling-lineage - (repeat 0)) - (butlast)) ; butlast ensures we end w/ node - ] - (-> (pop common-parent-path) - (conj next-index) - (into path-suffix))))) + (go-try S + (when-let [common-parent-path + (backtrack-up-path-until + path + (fn [parent index] + (< (inc index) (count (:children parent)))))] + (let [next-index (-> common-parent-path peek inc) + parent (-> common-parent-path pop peek) + new-sibling ( s :children first) + c (if (tree-node? c) + ( % :children first)] + (if (tree-node? c) + ( (interleave sibling-lineage + (repeat 0)) + (butlast)) ; butlast ensures we end w/ node + ] + (-> (pop common-parent-path) + (conj next-index) + (into path-suffix)))))) + + (defn forward-iterator "Takes the result of a search and returns an iterator going @@ -342,133 +359,144 @@ :children ; Get the indices of it (subseq >= start-key)) ; skip to the start-index next-elements (lazy-seq - (when-let [succ (right-successor (pop path))] - (forward-iterator succ start-key)))] + (when-let [succ ( (:children cur) - ;;TODO what are the semantics for exceeding on the right? currently it's trunc to the last element - (nth index (peek (:children cur))) - (resolve))) - path' (conj path index child)] - (recur path' child))) - nil))) + (go-try S + (loop [path [tree] ;alternating node/index/node/index/node... of the search taken + cur tree ;current search node + ] + (if (seq (:children cur)) + (if (data-node? cur) + path + (let [index (lookup cur key) + child (if (data-node? cur) + nil #_(nth-of-set (:children cur) index) + ( (:children cur) + ;;TODO what are the semantics for exceeding on the right? currently it's trunc to the last element + (nth index (peek (:children cur))) + (resolve S)))) + path' (conj path index child)] + (recur path' child))) + nil)))) (defn lookup-key "Given a B-tree and a key, gets an iterator into the tree" ([tree key] (lookup-key tree key nil)) ([tree key not-found] - (-> (lookup-path tree key) - (peek) - (resolve) - :children - (get key not-found)))) + (go-try S + (-> + ( (IndexNode [left right] (promise) [] cfg)) - node) - (let [index (peek path) - {:keys [children keys] :as parent} (peek (pop path))] - (if (overflow? node) ; splice the split into the parent - ;;TODO refactor paths to be node/index pairs or 2 vectors or something - (let [{:keys [left right median]} (split-node node) - new-children (catvec (conj (subvec children 0 index) - left right) - (subvec children (inc index)))] + (go-try S + (let [path (IndexNode [left right] (promise) [] cfg)) + node) + (let [index (peek path) + {:keys [children keys] :as parent} (peek (pop path))] + (if (overflow? node) ; splice the split into the parent + ;;TODO refactor paths to be node/index pairs or 2 vectors or something + (let [{:keys [left right median]} (split-node node) + new-children (catvec (conj (subvec children 0 index) + left right) + (subvec children (inc index)))] + (recur (-> parent + (assoc :children new-children) + (dirty!)) + (pop (pop path)))) (recur (-> parent - (assoc :children new-children) + ;;TODO this assoc-in seems to be a bottleneck + (assoc-in [:children index] node) (dirty!)) - (pop (pop path)))) - (recur (-> parent - ;;TODO this assoc-in seems to be a bottleneck - (assoc-in [:children index] node) - (dirty!)) - (pop (pop path))))))))) + (pop (pop path)))))))))) ;;TODO: cool optimization: when merging children, push as many operations as you can ;;into them to opportunisitcally minimize overall IO costs (defn delete [{:keys [cfg] :as tree} key] - (let [path (lookup-path tree key) ; don't care about the found key or its index - {:keys [children] :or {children (sorted-map-by compare)}} (peek path) - updated-data-node (data-node cfg (dissoc children key))] - (loop [node updated-data-node - path (pop path)] - (if (empty? path) - ;; Check for special root underflow case - (if (and (index-node? node) (= 1 (count (:children node)))) - (first (:children node)) - node) - (let [index (peek path) - {:keys [children keys op-buf] :as parent} (peek (pop path))] - (if (underflow? node) ; splice the split into the parent - ;;TODO this needs to use a polymorphic sibling-count to work on serialized nodes - (let [bigger-sibling-idx - (cond - (= (dec (count children)) index) (dec index) ; only have left sib - (zero? index) 1 ;only have right sib - (> (count (:children (nth children (dec index)))) - (count (:children (nth children (inc index))))) - (dec index) ; right sib bigger - :else (inc index)) - node-first? (> bigger-sibling-idx index) ; if true, `node` is left - merged (if node-first? - (merge-node node (resolve (nth children bigger-sibling-idx))) - (merge-node (resolve (nth children bigger-sibling-idx)) node)) - old-left-children (subvec children 0 (min index bigger-sibling-idx)) - old-right-children (subvec children (inc (max index bigger-sibling-idx)))] - (if (overflow? merged) - (let [{:keys [left right median]} (split-node merged)] - (recur (->IndexNode (catvec (conj old-left-children left right) + (go-try S + (let [path ( (count (:children (nth children (dec index)))) + (count (:children (nth children (inc index))))) + (dec index) ; right sib bigger + :else (inc index)) + node-first? (> bigger-sibling-idx index) ; if true, `node` is left + merged (if node-first? + (merge-node node (IndexNode (catvec (conj old-left-children left right) + old-right-children) + (promise) + op-buf + cfg) + (pop (pop path)))) + (recur (->IndexNode (catvec (conj old-left-children merged) old-right-children) (promise) op-buf cfg) - (pop (pop path)))) - (recur (->IndexNode (catvec (conj old-left-children merged) - old-right-children) - (promise) - op-buf - cfg) - (pop (pop path))))) - (recur (->IndexNode (assoc children index node) - (promise) - op-buf - cfg) - (pop (pop path))))))))) + (pop (pop path))))) + (recur (->IndexNode (assoc children index node) + (promise) + op-buf + cfg) + (pop (pop path)))))))))) (defn b-tree [cfg & kvs] - (reduce (fn [t [k v]] + (go-try S + (loop [[[k v] & r] (partition 2 kvs) + t (data-node cfg (sorted-map-by compare))] + (if k + (recur r (TestingAddr (last-key node) node)) + (go-try S + (swap! session update-in [:writes] inc) + (->TestingAddr (last-key node) node))) (delete-addr [_ addr session ])) (defn flush-tree @@ -534,22 +565,25 @@ Every dirty node also gets replaced with its TestingAddr. These form a GC cycle, have fun with the unmanaged memory port :)" ([tree backend] - (let [session (new-session backend) - flushed (flush-tree tree backend session)] - {:tree (resolve (anchor-root backend flushed)) ; root should never be unresolved for API - :stats session})) + (go-try S + (let [session (new-session backend) + flushed ( % kons/map->KonserveAddr + (assoc :store store + :storage-addr (kons/synthesize-storage-addr (:konserve-key %)))) + 'hitchhiker.tree.core.DataNode + (fn [{:keys [children cfg]}] + (core/->DataNode (into (sorted-map-by + compare) children) + (promise) + cfg)) + 'hitchhiker.tree.core.IndexNode + (fn [{:keys [children cfg op-buf]}] + (core/->IndexNode (->> children + #_((fn [e] (prn "reading" e) e)) + catvec) + (promise) + (catvec op-buf) + cfg)) + 'hitchhiker.tree.messaging.InsertOp + msg/map->InsertOp + 'hitchhiker.tree.messaging.DeleteOp + msg/map->DeleteOp + 'hitchhiker.tree.core.Config + core/map->Config}) + store)) + +(deftest simple-konserve-test + (testing "Insert and lookup" + (let [folder "/tmp/async-hitchhiker-tree-test" + store (setup-store folder) + backend (kons/->KonserveBackend store) + flushed (Config 1 3 (- 3 1)))) + (range 1 11))) + backend)) + root-key (kons/get-root-key (:tree flushed)) + tree (Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) data2 (data-node (->Config 3 3 2) (sorted-map 6 6 7 7 8 8 9 9 10 10)) root (->IndexNode [data1 data2] (promise) [] (->Config 3 3 2))] - (is (= (lookup-key root -10) nil) "not found key") - (is (= (lookup-key root 100) nil) "not found key") + (is (= (Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) data2 (data-node (->Config 3 3 2) (sorted-map 6 6 7 7 8 8 9 9 10 10)) root (->IndexNode [data1 data2] (promise) [] (->Config 3 3 2))] (is (= (map first (lookup-fwd-iter root 4)) (range 4 11))) (is (= (map first (lookup-fwd-iter root 0)) (range 1 11))))) - (testing "index nodes identified as such" (let [data (data-node (->Config 3 3 2) (sorted-map 1 1)) root (->IndexNode [data] (promise) [] (->Config 3 3 2))] @@ -31,13 +31,13 @@ (defn insert-helper [t k] - (insert t k k)) + (Config 3 3 2)) + (Config 3 3 2))) v) b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) (seq (map first b-tree-order)))))) @@ -50,7 +50,7 @@ 1000 (prop/for-all [v (gen/vector gen/int)] (let [sorted-set-order (into (sorted-set) v) - b-tree (reduce insert-helper (b-tree (->Config 3 3 2)) v) + b-tree (reduce insert-helper (Config 3 3 2))) v) b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) (seq (map first b-tree-order)))))) @@ -60,22 +60,22 @@ num gen/nat] (let [set-a (sort the-set) set-b (take num the-set) - b-tree (reduce insert-helper (b-tree (->Config 3 3 2)) set-a) - b-tree-without (reduce delete b-tree set-b) + b-tree (reduce insert-helper (Config 3 3 2))) set-a) + b-tree-without (reduce #(Config 3 3 2) (sorted-map 1 "1" 2 "2" 3 "3" 4 "4")) root (->IndexNode [data1] (promise) [] (->Config 3 3 2))] - (is (= (map second (lookup-fwd-iter (insert root 3 "3") -10)) ["1" "2" "3" "4"])) - (are [x] (= (map second (lookup-fwd-iter (insert root x (str x)) -10)) (sort (map str (conj [1 2 3 4] x)))) + (is (= (map second (lookup-fwd-iter (Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) root (->IndexNode [data1] (promise) [] (->Config 3 3 2))] - (are [x y] (= (map first (lookup-fwd-iter (insert root x x) y)) + (are [x y] (= (map first (lookup-fwd-iter (Config 3 3 2)) (range 5))] - (is (= (map first (lookup-fwd-iter (delete tree 3) 0)) [0 1 2 4]))) - (let [tree (reduce insert-helper (b-tree (->Config 3 3 2)) (range 10))] - (is (= (map first (lookup-fwd-iter (delete tree 0) 0)) (map inc (range 9))))) - (let [tree (reduce insert-helper (b-tree (->Config 3 3 2)) (range 6))] - (is (= (map first (lookup-fwd-iter (delete tree 0) 0)) (map inc (range 5)))))) + (let [tree (reduce insert-helper (Config 3 3 2))) (range 5))] + (is (= (map first (lookup-fwd-iter (Config 3 3 2))) (range 10))] + (is (= (map first (lookup-fwd-iter (Config 3 3 2))) (range 6))] + (is (= (map first (lookup-fwd-iter (Config 3 3 2)) the-set)] + (let [b-tree (reduce insert-helper (Config 3 3 2))) the-set)] (check-node-is-balanced b-tree)))) (defspec test-wider-balanced-after-many-inserts 1000 (prop/for-all [the-set (gen/vector (gen/no-shrink gen/int))] - (let [b-tree (reduce insert-helper (b-tree (->Config 200 250 17)) the-set)] + (let [b-tree (reduce insert-helper (Config 200 250 17))) the-set)] (check-node-is-balanced b-tree)))) #_(require '[criterium.core :refer (quick-bench)]) @@ -149,8 +149,8 @@ (let [x-reduced (mod x universe-size)] (condp = op :add (insert-helper t x-reduced) - :del (delete t x-reduced)))) - (b-tree (->Config 3 3 2)) + :del (Config 3 3 2))) ops)] ; (println ops) (check-node-is-balanced b-tree))))) @@ -174,9 +174,9 @@ 1000 (prop/for-all [v (gen/vector gen/int)] (let [sorted-set-order (into (sorted-set) v) - b-tree (reduce insert-helper (b-tree (->Config 3 3 2)) v) + b-tree (reduce insert-helper (Config 3 3 2))) v) b-tree-order (map first (lookup-fwd-iter b-tree Integer/MIN_VALUE)) - flushed-tree (:tree (flush-tree b-tree (->TestingBackend))) + flushed-tree (:tree (TestingBackend)))) flushed-tree-order (map first (lookup-fwd-iter flushed-tree Integer/MIN_VALUE))] (= (seq sorted-set-order) (seq b-tree-order) diff --git a/test/hitchhiker/tree/messaging_test.clj b/test/hitchhiker/tree/messaging_test.clj index ab2e96d..f566fe8 100644 --- a/test/hitchhiker/tree/messaging_test.clj +++ b/test/hitchhiker/tree/messaging_test.clj @@ -1,5 +1,6 @@ (ns hitchhiker.tree.messaging-test (:require [clojure.test.check.clojure-test :refer [defspec]] + [superv.async :refer [Config 3 3 2)) v) + b-tree (reduce insert (Config 3 3 2))) v) b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) b-tree-order)))) @@ -26,7 +27,7 @@ 1000 (prop/for-all [v (gen/vector gen/int)] (let [sorted-set-order (into (sorted-set) v) - b-tree (reduce insert (core/b-tree (core/->Config 3 3 2)) v) + b-tree (reduce insert (Config 3 3 2))) v) b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) b-tree-order)))) @@ -36,8 +37,8 @@ num gen/nat] (let [set-a (sort the-set) set-b (take num the-set) - b-tree (reduce insert (core/b-tree (core/->Config 3 3 2)) set-a) - b-tree-without (reduce msg/delete b-tree set-b) + b-tree (reduce insert (Config 3 3 2))) set-a) + b-tree-without (reduce #(Config 3 3 2)) the-set)] + (let [b-tree (reduce insert (Config 3 3 2))) the-set)] (hitchhiker.tree.core-test/check-node-is-balanced b-tree)))) (defspec test-wider-balanced-after-many-inserts 1000 (prop/for-all [the-set (gen/vector (gen/no-shrink gen/int))] - (let [b-tree (reduce insert (core/b-tree (core/->Config 200 220 17)) the-set)] + (let [b-tree (reduce insert (Config 200 220 17))) the-set)] (hitchhiker.tree.core-test/check-node-is-balanced b-tree)))) ;; This test will show how if you apply killerop to the b-tree result, it corrupts @@ -78,7 +79,7 @@ (conj s x-reduced)] :del [(msg/delete t x-reduced) (disj s x-reduced)]))) - [(core/b-tree (core/->Config 3 3 2)) #{}] + [(Config 3 3 2))) #{}] ops) f #(case (first %1) :add (insert %2 (second %1)) :del (msg/delete %2 (second %1))) @@ -120,9 +121,9 @@ (condp = op :add [(insert t x-reduced) (conj s x-reduced)] - :del [(msg/delete t x-reduced) + :del [(Config 3 3 2)) #{}] + [(Config 3 3 2))) #{}] ops)] ; (println ops) (and (= (lookup-fwd-iter b-tree -1) (seq (sort s))) @@ -139,7 +140,7 @@ (conj s x-reduced)] :del [(msg/delete t x-reduced) (disj s x-reduced)]))) - [(core/b-tree (core/->Config 3 3 2)) #{}] + [(Config 3 3 2))) #{}] data)] ; (println ops) (println (lookup-fwd-iter b-tree -1)) @@ -153,9 +154,9 @@ (mixed-op-seq 0.5 250 5000)) (defspec test-many-keys-bigger-trees - 1000 + 100 (mixed-op-seq 0.8 1000 1000)) (defspec test-sparse-ops - 1000 + 100 (mixed-op-seq 0.7 100000 1000)) From 7863b14f9bc703aee4213a212c1c3cbb6a164ad7 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 26 Mar 2017 04:43:07 +0200 Subject: [PATCH 03/34] Fix stash mismatch after re-adding of print statements. --- src/hitchhiker/tree/core.clj | 2 +- src/hitchhiker/tree/messaging.clj | 130 ++++++++++++++++-------------- 2 files changed, 69 insertions(+), 63 deletions(-) diff --git a/src/hitchhiker/tree/core.clj b/src/hitchhiker/tree/core.clj index 1e3179f..6460166 100644 --- a/src/hitchhiker/tree/core.clj +++ b/src/hitchhiker/tree/core.clj @@ -505,7 +505,7 @@ IResolve (dirty? [this] false) (last-key [_] last-key) - (resolve [_] node)) + (resolve [_ S] (go node))) (defn print-testing-addr [node ^Writer writer fully-qualified?] diff --git a/src/hitchhiker/tree/messaging.clj b/src/hitchhiker/tree/messaging.clj index 1945893..1ed8110 100644 --- a/src/hitchhiker/tree/messaging.clj +++ b/src/hitchhiker/tree/messaging.clj @@ -63,69 +63,75 @@ (defn enqueue ([tree msgs] - (let [deferred-ops (atom []) - msg-buffers-propagated (enqueue tree msgs deferred-ops)] - ;(when (seq @deferred-ops) (println "appyling deferred ops" @deferred-ops)) - (reduce (fn [tree op] - (apply-op-to-tree op tree)) - msg-buffers-propagated - @deferred-ops))) + (go-try S + (let [deferred-ops (atom []) + msg-buffers-propagated ( tree - (core/dirty!) - (update-in [:op-buf] into msgs)) - :else ; overflow, should be IndexNode - (do (assert (core/index-node? tree)) - ;(println "overflowing node" (:keys tree) "with buf" (:op-buf tree) - ; "with new msgs" msgs - ; ) - (loop [[child & children] (:children tree) - rebuilt-children [] - msgs (vec (sort-by affects-key ;must be a stable sort - (concat (:op-buf tree) msgs)))] - (let [took-msgs (into [] - (take-while #(>= 0 (core/compare - (affects-key %) - (core/last-key child)))) - msgs) - extra-msgs (into [] - (drop-while #(>= 0 (core/compare - (affects-key %) - (core/last-key child)))) - msgs) - ;_ (println "last-key:" (core/last-key child)) - ;_ (println "goes left:" took-msgs) - ;_ (println "goes right:" extra-msgs) - on-the-last-child? (empty? children) - - ;; Any changes to the current child? - new-child - (cond - (and on-the-last-child? (seq extra-msgs)) - (enqueue (core/resolve child) - (catvec took-msgs extra-msgs) - deferred-ops) - (seq took-msgs) ; save a write - (enqueue (core/resolve child) - took-msgs - deferred-ops) - :else - child)] - - (if on-the-last-child? - (-> tree - (assoc :children (conj rebuilt-children new-child)) - (assoc :op-buf []) - (core/dirty!)) - (recur children (conj rebuilt-children new-child) extra-msgs))))))))) + (go-try S + (let [tree ( tree + (core/dirty!) + (update-in [:op-buf] into msgs)) + :else ; overflow, should be IndexNode + (do (assert (core/index-node? tree)) + ;(println "overflowing node" (:keys tree) "with buf" (:op-buf tree) + ; "with new msgs" msgs + ; ) + (loop [[child & children] (:children tree) + rebuilt-children [] + msgs (vec (sort-by affects-key ;must be a stable sort + (concat (:op-buf tree) msgs)))] + (let [took-msgs (into [] + (take-while #(>= 0 (core/compare + (affects-key %) + (core/last-key child)))) + msgs) + extra-msgs (into [] + (drop-while #(>= 0 (core/compare + (affects-key %) + (core/last-key child)))) + msgs) + ;_ (println "last-key:" (core/last-key child)) + ;_ (println "goes left:" took-msgs) + ;_ (println "goes right:" extra-msgs) + on-the-last-child? (empty? children) + + ;; Any changes to the current child? + new-child + (cond + (and on-the-last-child? (seq extra-msgs)) + ( tree + (assoc :children (conj rebuilt-children new-child)) + (assoc :op-buf []) + (core/dirty!)) + (recur children (conj rebuilt-children new-child) extra-msgs)))))))))) ;;TODO delete in core needs to stop using the index-node constructor to be more From 1668488fcac98574b6965d4e2722dc8eb38433e1 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 2 Apr 2017 00:27:34 +0200 Subject: [PATCH 04/34] Use plain core.async + helpers, port promise, fixes. --- project.clj | 32 +- .../{konserve.clj => konserve.cljc} | 90 ++-- src/hitchhiker/outboard.clj | 25 +- src/hitchhiker/redis.clj | 62 +-- src/hitchhiker/tree/{core.clj => core.cljc} | 479 +++++++++++------- .../tree/{messaging.clj => messaging.cljc} | 144 +++--- test/hitchhiker/konserve_test.clj | 13 +- test/hitchhiker/tree/core_test.clj | 51 +- test/hitchhiker/tree/messaging_test.clj | 25 +- 9 files changed, 526 insertions(+), 395 deletions(-) rename src/hitchhiker/{konserve.clj => konserve.cljc} (69%) rename src/hitchhiker/tree/{core.clj => core.cljc} (60%) rename src/hitchhiker/tree/{messaging.clj => messaging.cljc} (74%) diff --git a/project.clj b/project.clj index 7ad6cb3..e230b70 100644 --- a/project.clj +++ b/project.clj @@ -3,15 +3,14 @@ :url "https://github.com/dgrnbrg/hitchhiker-tree" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :dependencies [[org.clojure/clojure "1.7.0"] + :dependencies [[org.clojure/clojure "1.8.0"] [org.clojure/core.memoize "0.5.8"] [com.taoensso/carmine "2.12.2"] [org.clojure/core.rrb-vector "0.0.11"] [org.clojure/core.cache "0.6.5"] [io.replikativ/incognito "0.2.2-SNAPSHOT"] - [io.replikativ/konserve "0.4.8"] - [io.replikativ/superv.async "0.2.5"]] + [io.replikativ/konserve "0.4.8"]] :aliases {"bench" ["with-profile" "profiling" "run" "-m" "hitchhiker.bench"]} :jvm-opts ["-server" "-Xmx3700m" "-Xms3700m"] :profiles {:test @@ -22,4 +21,29 @@ :dependencies [[criterium "0.4.4"] [org.clojure/tools.cli "0.3.3"] [org.clojure/test.check "0.9.0"] - [com.infolace/excel-templates "0.3.3"]]}}) + [com.infolace/excel-templates "0.3.3"]]} + :dev {:dependencies [[binaryage/devtools "0.8.2"] + [figwheel-sidecar "0.5.8"] + [com.cemerick/piggieback "0.2.1"] + [org.clojure/test.check "0.9.0"]] + ;; need to add dev source path here to get user.clj loaded + :source-paths ["src" "dev"] + ;; for CIDER + ;; :plugins [[cider/cider-nrepl "0.12.0"]] + :repl-options {; for nREPL dev you really need to limit output + :init (set! *print-length* 50) + :nrepl-middleware [cemerick.piggieback/wrap-cljs-repl]}}} + :clean-targets ^{:protect false} ["resources/public/js/compiled" "target"] + + + :cljsbuild {:builds + [{:id "dev" + :figwheel true + :source-paths ["src"] + :compiler {:main hitchhiker.tree.core + :asset-path "js/out" + :output-to "resources/public/js/core.js" + :output-dir "resources/public/js/out" }}]} + + :plugins [[lein-figwheel "0.5.8"] + [lein-cljsbuild "1.1.4" :exclusions [[org.clojure/clojure]]]]) diff --git a/src/hitchhiker/konserve.clj b/src/hitchhiker/konserve.cljc similarity index 69% rename from src/hitchhiker/konserve.clj rename to src/hitchhiker/konserve.cljc index 7cc4793..740aafc 100644 --- a/src/hitchhiker/konserve.clj +++ b/src/hitchhiker/konserve.cljc @@ -1,34 +1,32 @@ (ns hitchhiker.konserve (:refer-clojure :exclude [resolve subvec]) (:require [clojure.core.rrb-vector :refer [catvec subvec]] - [clojure.core.async :refer [chan ( ( (assoc node :storage-addr nil) @@ -47,40 +44,22 @@ :storage-addr nil) cs)))) (assoc node :storage-addr nil))] (let [id (uuid pnode)] - (KonserveAddr store (core/last-key node) id (synthesize-storage-addr id))))) - #_(let [key (str (java.util.UUID/randomUUID)) - addr (redis-addr (core/last-key node) key)] - ;(.submit service #(wcar {} (car/set key node))) - (when (some #(not (satisfies? msg/IOperation %)) (:op-buf node)) - (println (str "Found a broken node, has " (count (:op-buf node)) " ops")) - (println (str "The node data is " node)) - (println (str "and " (:op-buf node)))) - (wcar {} - (car/set key node) - (when (core/index-node? node) - (add-refs key - (for [child (:children node) - :let [child-key @(:storage-addr child)]] - child-key)))) - (seed-cache! key (doto (promise) (deliver node))) - addr)) + (KonserveAddr store (core/last-key node) id (synthesize-storage-addr id)))))) (delete-addr [_ addr session] - #_(wcar {} (car/del addr)) (swap! session update-in :deletes inc))) (defn get-root-key [tree] - (-> tree :storage-addr (deref 10 nil))) + (-> tree :storage-addr (async/poll!))) (defn create-tree-from-root-key [store root-key] - (go-try S - (let [val (KonserveAddr store last-key root-key (synthesize-storage-addr root-key)) - S))))) + (KonserveAddr store last-key root-key (synthesize-storage-addr root-key))))))) (defn add-read-handlers [store] (swap! (:read-handlers store) merge @@ -92,14 +71,13 @@ (fn [{:keys [children cfg]}] (core/->DataNode (into (sorted-map-by compare) children) - (promise) + (promise-chan) cfg)) 'hitchhiker.tree.core.IndexNode (fn [{:keys [children cfg op-buf]}] (core/->IndexNode (->> children - #_((fn [e] (prn "reading" e) e)) catvec) - (promise) + (promise-chan) (catvec op-buf) cfg)) 'hitchhiker.tree.messaging.InsertOp @@ -116,7 +94,7 @@ (def store (add-read-handlers (KonserveBackend store) ))) + (enable-console-print!) + + (go-try (def foo (Config 17 300 (- 300 17)))) + 1 1)))) + + (go-try (prn (Config 17 300 (- 300 17)))) + 1 1)) + 2 2)))) + + (go-try + (prn "foo" + (loop [i 0 + t (Config 17 300 (- 300 17))))] + (if (= i 2) + t + (recur (inc i) (Config 17 300 (- 300 17)))) + (range 20000))) + (def my-tree-updated (RedisBackend)))))) + (:tree (RedisBackend)))))) (let [new-root (redis/get-root-key @tree-atom)] (wcar {} @@ -80,7 +79,7 @@ ", its already in use") {:used-name new-name}))) ;;TODO race condition where additional calls to create could all succeed ;;we should guard against this - (let [conn (->OutboardConnection (LinkedBlockingQueue.) (atom (Config 30 600 870)))) (atom :running) (atom nil) new-name)] + (let [conn (->OutboardConnection (LinkedBlockingQueue.) (atom (Config 30 600 870)))) (atom :running) (atom nil) new-name)] (launch-outboard-processer! conn new-name) (swap! connection-registry assoc new-name conn) conn)) @@ -138,27 +137,27 @@ [snapshot k v & kvs] (let [tree snapshot] (if (and (seq kvs) (even? (count kvs))) - (loop [tree ( (totally-fetch redis-key) + (resolve [_] + (go-try (-> (totally-fetch redis-key) (assoc :storage-addr (synthesize-storage-addr redis-key)))))) (comment @@ -204,38 +204,38 @@ (wcar {} (add-to-expiry redis-key (+ 5000 (System/currentTimeMillis)))) node) (write-node [_ node session] - (go-try S - (swap! session update-in [:writes] inc) - (let [key (str (java.util.UUID/randomUUID)) - addr (redis-addr (core/last-key node) key)] + (go-try + (swap! session update-in [:writes] inc) + (let [key (str (java.util.UUID/randomUUID)) + addr (redis-addr (core/last-key node) key)] ;(.submit service #(wcar {} (car/set key node))) - (when (some #(not (satisfies? msg/IOperation %)) (:op-buf node)) - (println (str "Found a broken node, has " (count (:op-buf node)) " ops")) - (println (str "The node data is " node)) - (println (str "and " (:op-buf node)))) - (wcar {} - (car/set key node) - (when (core/index-node? node) - (add-refs key - (for [child (:children node) - :let [child-key @(:storage-addr child)]] - child-key)))) - (seed-cache! key (doto (promise) (deliver node))) - addr))) + (when (some #(not (satisfies? msg/IOperation %)) (:op-buf node)) + (println (str "Found a broken node, has " (count (:op-buf node)) " ops")) + (println (str "The node data is " node)) + (println (str "and " (:op-buf node)))) + (wcar {} + (car/set key node) + (when (core/index-node? node) + (add-refs key + (for [child (:children node) + :let [child-key ( tree :storage-addr (deref 10 nil))) + (-> tree :storage-addr (async/poll!))) + (defn create-tree-from-root-key [root-key] (let [last-key (core/last-key (wcar {} (car/get root-key)))] ; need last key to bootstrap - (RedisAddr last-key root-key (synthesize-storage-addr root-key)) - S)))) + (RedisAddr last-key root-key (synthesize-storage-addr root-key)))))) (comment (wcar {} (car/ping) (car/set "foo" "bar") (car/get "foo")) @@ -286,8 +286,8 @@ (wcar {} (car/flushall)) (count (wcar {} (car/keys "*"))) - (count (msg/lookup-fwd-iter (create-tree-from-root-key @(:storage-addr (:tree my-tree))) -1)) - (count (msg/lookup-fwd-iter (create-tree-from-root-key @(:storage-addr (:tree my-tree-updated))) -1)) + (count (msg/lookup-fwd-iter (create-tree-from-root-key (Config 17 300 (- 300 17))) diff --git a/src/hitchhiker/tree/core.clj b/src/hitchhiker/tree/core.cljc similarity index 60% rename from src/hitchhiker/tree/core.clj rename to src/hitchhiker/tree/core.cljc index 6460166..c65bcb5 100644 --- a/src/hitchhiker/tree/core.clj +++ b/src/hitchhiker/tree/core.cljc @@ -1,12 +1,76 @@ (ns hitchhiker.tree.core (:refer-clojure :exclude [compare resolve subvec]) (:require [clojure.core.rrb-vector :refer [catvec subvec]] - [clojure.pprint :as pp] - [superv.async :refer [go-try S IndexNode) + (defrecord IndexNode [children storage-addr op-buf cfg] IResolve (index? [this] true) - (dirty? [this] (not (realized? storage-addr))) - (resolve [this S] (go this)) ;;TODO this is a hack for testing + (dirty? [this] (not (async/poll! storage-addr))) + (resolve [this] (go this)) ;;TODO this is a hack for testing (last-key [this] ;;TODO should optimize by caching to reduce IOps (can use monad) (last-key (peek children))) @@ -124,17 +194,17 @@ ;;TODO this should use msg/affects-key (sort-by :key op-buf))] (->Split (->IndexNode (subvec children 0 b) - (promise) + (promise-chan) (vec left-buf) cfg) (->IndexNode (subvec children b) - (promise) + (promise-chan) (vec right-buf) cfg) median))) (merge-node [this other] (->IndexNode (catvec children (:children other)) - (promise) + (promise-chan) (catvec op-buf (:op-buf other)) cfg)) (lookup [root key] @@ -143,72 +213,80 @@ a (object-array l) _ (dotimes [i l] (aset a i (last-key (nth children i)))) - x (Arrays/binarySearch a 0 l key compare)] + x #?(:clj (Arrays/binarySearch a 0 l key compare) + :cljs (goog.array/binarySearch a key compare))] (if (neg? x) (- (inc x)) x)))) -(nippy/extend-freeze IndexNode :b-tree/index-node - [{:keys [storage-addr cfg children op-buf]} data-output] - (nippy/freeze-to-out! data-output cfg) - (nippy/freeze-to-out! data-output children) - ;;TODO apparently RRB-vectors don't freeze correctly; - ;;we'll force it to a normal vector as a workaround - (nippy/freeze-to-out! data-output (into [] op-buf))) - -(nippy/extend-thaw :b-tree/index-node - [data-input] - (let [cfg (nippy/thaw-from-in! data-input) - children (nippy/thaw-from-in! data-input) - op-buf (nippy/thaw-from-in! data-input)] - (->IndexNode children nil op-buf cfg))) +#?(:clj + (nippy/extend-freeze IndexNode :b-tree/index-node + [{:keys [storage-addr cfg children op-buf]} data-output] + (nippy/freeze-to-out! data-output cfg) + (nippy/freeze-to-out! data-output children) + ;;TODO apparently RRB-vectors don't freeze correctly; + ;;we'll force it to a normal vector as a workaround + (nippy/freeze-to-out! data-output (into [] op-buf)))) + +#?(:clj + (nippy/extend-thaw :b-tree/index-node + [data-input] + (let [cfg (nippy/thaw-from-in! data-input) + children (nippy/thaw-from-in! data-input) + op-buf (nippy/thaw-from-in! data-input)] + (->IndexNode children nil op-buf cfg)))) (defn index-node? [node] (instance? IndexNode node)) -(defn print-index-node - "Optionally include" - [node ^Writer writer fully-qualified?] - (.write writer (if fully-qualified? - (pr-str IndexNode) - "IndexNode")) - (.write writer (str {:keys (index-node-keys (:children node)) - :children (:children node)}))) - -(defmethod print-method IndexNode - [node writer] - (print-index-node node writer false)) - -(defmethod print-dup IndexNode - [node writer] - (print-index-node node writer true)) - -(defn node-status-bits - [node] - (str "[" - (if (dirty? node) "D" " ") - "]")) - -(defmethod pp/simple-dispatch IndexNode - [node] - (let [out ^Writer *out*] - (.write out "IndexNode") - (.write out (node-status-bits node)) - (pp/pprint-logical-block - :prefix "{" :suffix "}" - (pp/pprint-logical-block - (.write out ":keys ") - (pp/write-out (index-node-keys (:children node))) - (pp/pprint-newline :linear)) - (pp/pprint-logical-block - (.write out ":op-buf ") - (pp/write-out (:op-buf node)) - (pp/pprint-newline :linear)) - (pp/pprint-logical-block - (.write out ":children ") - (pp/pprint-newline :mandatory) - (pp/write-out (:children node)))))) +#?(:clj + (defn print-index-node + "Optionally include" + [node ^Writer writer fully-qualified?] + (.write writer (if fully-qualified? + (pr-str IndexNode) + "IndexNode")) + (.write writer (str {:keys (index-node-keys (:children node)) + :children (:children node)})))) + +#?(:clj + (defmethod print-method IndexNode + [node writer] + (print-index-node node writer false))) + +#?(:clj + (defmethod print-dup IndexNode + [node writer] + (print-index-node node writer true))) + +#?(:clj + (defn node-status-bits + [node] + (str "[" + (if (dirty? node) "D" " ") + "]"))) + +#?(:clj + (defmethod pp/simple-dispatch IndexNode + [node] + (let [out ^Writer *out*] + (.write out "IndexNode") + (.write out (node-status-bits node)) + (pp/pprint-logical-block + :prefix "{" :suffix "}" + (pp/pprint-logical-block + (.write out ":keys ") + (pp/write-out (index-node-keys (:children node))) + (pp/pprint-newline :linear)) + (pp/pprint-logical-block + (.write out ":op-buf ") + (pp/write-out (:op-buf node)) + (pp/pprint-newline :linear)) + (pp/pprint-logical-block + (.write out ":children ") + (pp/pprint-newline :mandatory) + (pp/write-out (:children node))))))) (defn nth-of-set "Like nth, but for sorted sets. O(n)" @@ -218,8 +296,8 @@ (defrecord DataNode [children storage-addr cfg] IResolve (index? [this] false) - (resolve [this S] (go this)) ;;TODO this is a hack for testing - (dirty? [this] (not (realized? storage-addr))) + (resolve [this] (go this)) ;;TODO this is a hack for testing + (dirty? [this] (not (async/poll! storage-addr))) (last-key [this] (when (seq children) (-> children @@ -239,7 +317,8 @@ (merge-node [this other] (data-node cfg (into children (:children other)))) (lookup [root key] - (let [x (Collections/binarySearch (vec (keys children)) key compare)] + (let [x #?(:clj (Collections/binarySearch (vec (keys children)) key compare) + :cljs (goog.array/binarySearch (into-array (keys children)) key compare))] (if (neg? x) (- (inc x)) x)))) @@ -247,46 +326,52 @@ (defn data-node "Creates a new data node" [cfg children] - (->DataNode children (promise) cfg)) + (->DataNode children (promise-chan) cfg)) (defn data-node? [node] (instance? DataNode node)) -(nippy/extend-freeze DataNode :b-tree/data-node - [{:keys [cfg children]} data-output] - (nippy/freeze-to-out! data-output cfg) - (nippy/freeze-to-out! data-output children)) +#?(:clj + (nippy/extend-freeze DataNode :b-tree/data-node + [{:keys [cfg children]} data-output] + (nippy/freeze-to-out! data-output cfg) + (nippy/freeze-to-out! data-output children))) -(nippy/extend-thaw :b-tree/data-node - [data-input] - (let [cfg (nippy/thaw-from-in! data-input) - children (nippy/thaw-from-in! data-input)] - (->DataNode children nil cfg))) +#?(:clj + (nippy/extend-thaw :b-tree/data-node + [data-input] + (let [cfg (nippy/thaw-from-in! data-input) + children (nippy/thaw-from-in! data-input)] + (->DataNode children nil cfg)))) ;(println (b-tree :foo :bar :baz)) ;(pp/pprint (apply b-tree (range 100))) -(defn print-data-node - [node ^Writer writer fully-qualified?] - (.write writer (if fully-qualified? - (pr-str DataNode) - "DataNode")) - (.write writer (str {:children (:children node)}))) - -(defmethod print-method DataNode - [node writer] - (print-data-node node writer false)) - -(defmethod print-dup DataNode - [node writer] - (print-data-node node writer true)) - -(defmethod pp/simple-dispatch DataNode - [node] - (let [out ^Writer *out*] - (.write out (str "DataNode" - (node-status-bits node))) - (.write out (str {:children (:children node)})))) +#?(:clj + (defn print-data-node + [node ^Writer writer fully-qualified?] + (.write writer (if fully-qualified? + (pr-str DataNode) + "DataNode")) + (.write writer (str {:children (:children node)})))) + +#?(:clj + (defmethod print-method DataNode + [node writer] + (print-data-node node writer false))) + +#?(:clj + (defmethod print-dup DataNode + [node writer] + (print-data-node node writer true))) + +#?(:clj + (defmethod pp/simple-dispatch DataNode + [node] + (let [out ^Writer *out*] + (.write out (str "DataNode" + (node-status-bits node))) + (.write out (str {:children (:children node)}))))) (defn backtrack-up-path-until "Given a path (starting with root and ending with an index), searches backwards, @@ -310,7 +395,7 @@ ;(clojure.pprint/pprint path) ;TODO this function would benefit from a prefetching hint ; to keep the next several sibs in mem - (go-try S + (go-try (when-let [common-parent-path (backtrack-up-path-until path @@ -318,13 +403,13 @@ (< (inc index) (count (:children parent)))))] (let [next-index (-> common-parent-path peek inc) parent (-> common-parent-path pop peek) - new-sibling ( s :children first) c (if (tree-node? c) - ( start-node - :children ; Get the indices of it - (subseq >= start-key)) ; skip to the start-index - next-elements (lazy-seq - (when-let [succ ( start-node + :children ; Get the indices of it + (subseq >= start-key)) ; skip to the start-index + next-elements (lazy-seq + (when-let [succ ( (:children cur) + ( (:children cur) ;;TODO what are the semantics for exceeding on the right? currently it's trunc to the last element (nth index (peek (:children cur))) - (resolve S)))) + (resolve)))) path' (conj path index child)] (recur path' child))) nil)))) @@ -389,24 +475,25 @@ ([tree key] (lookup-key tree key nil)) ([tree key not-found] - (go-try S + (go-try (-> - ( ( (IndexNode [left right] (promise) [] cfg)) + (->IndexNode [left right] (promise-chan) [] cfg)) node) (let [index (peek path) {:keys [children keys] :as parent} (peek (pop path))] @@ -439,8 +526,8 @@ (defn delete [{:keys [cfg] :as tree} key] - (go-try S - (let [path ( bigger-sibling-idx index) ; if true, `node` is left merged (if node-first? - (merge-node node (IndexNode (catvec (conj old-left-children left right) old-right-children) - (promise) + (promise-chan) op-buf cfg) (pop (pop path)))) (recur (->IndexNode (catvec (conj old-left-children merged) old-right-children) - (promise) + (promise-chan) op-buf cfg) (pop (pop path))))) (recur (->IndexNode (assoc children index node) - (promise) + (promise-chan) op-buf cfg) (pop (pop path)))))))))) (defn b-tree [cfg & kvs] - (go-try S + (go-try (loop [[[k v] & r] (partition 2 kvs) t (data-node cfg (sorted-map-by compare))] (if k - (recur r (TestingAddr (last-key node) node))) (delete-addr [_ addr session ])) @@ -565,22 +659,25 @@ Every dirty node also gets replaced with its TestingAddr. These form a GC cycle, have fun with the unmanaged memory port :)" ([tree backend] - (go-try S + (go-try (let [session (new-session backend) - flushed (> (flush-children (:children tree) backend stats) + InsertOp ") - (.write writer (pr-str (:key op))) - (.write writer ", ") - (.write writer (pr-str (:value op))) - (.write writer ")")) - -(defmethod pp/simple-dispatch InsertOp - [op] - (print op)) - -(defmethod print-method DeleteOp - [op ^Writer writer] - (.write writer "DeleteOp") - (.write writer (str {:key (:key op)} " - " (:tag op)))) - -(defmethod print-dup DeleteOp - [op ^Writer writer] - (.write writer "(tree.messaging/->DeleteOp ") - (.write writer (pr-str (:key op))) - (.write writer ")")) - -(defmethod pp/simple-dispatch DeleteOp - [op] - (print op)) +#?(:clj + (defmethod print-method InsertOp + [op ^Writer writer] + (.write writer "InsertOp") + (.write writer (str {:key (:key op) :value (:value op) " - " (:tag op)})))) + +#?(:clj + (defmethod print-dup InsertOp + [op ^Writer writer] + (.write writer "(tree.messaging/->InsertOp ") + (.write writer (pr-str (:key op))) + (.write writer ", ") + (.write writer (pr-str (:value op))) + (.write writer ")"))) + +#?(:clj + (defmethod pp/simple-dispatch InsertOp + [op] + (print op))) + +#?(:clj + (defmethod print-method DeleteOp + [op ^Writer writer] + (.write writer "DeleteOp") + (.write writer (str {:key (:key op)} " - " (:tag op))))) + +#?(:clj + (defmethod print-dup DeleteOp + [op ^Writer writer] + (.write writer "(tree.messaging/->DeleteOp ") + (.write writer (pr-str (:key op))) + (.write writer ")"))) + +#?(:clj + (defmethod pp/simple-dispatch DeleteOp + [op] + (print op))) (defn enqueue ([tree msgs] - (go-try S + (go-try (let [deferred-ops (atom []) - msg-buffers-propagated (InsertOp key value) - :tag (java.util.UUID/randomUUID) + :tag (uuid) )])) (defn delete [tree key] (enqueue tree [(assoc (->DeleteOp key) - :tag (java.util.UUID/randomUUID) + :tag (uuid) )])) -(defn forward-iterator - "Takes the result of a search and returns an iterator going +#?(:clj + (defn forward-iterator + "Takes the result of a search and returns an iterator going forward over the tree. Does lg(n) backtracking sometimes." - [path] - (assert (core/data-node? (peek path))) - (let [first-elements (apply-ops-in-path path) - next-elements (lazy-seq - (when-let [succ ( % kons/map->KonserveAddr (assoc :store store @@ -24,14 +24,13 @@ (fn [{:keys [children cfg]}] (core/->DataNode (into (sorted-map-by compare) children) - (promise) + (promise-chan) cfg)) 'hitchhiker.tree.core.IndexNode (fn [{:keys [children cfg op-buf]}] (core/->IndexNode (->> children - #_((fn [e] (prn "reading" e) e)) catvec) - (promise) + (promise-chan) (catvec op-buf) cfg)) 'hitchhiker.tree.messaging.InsertOp diff --git a/test/hitchhiker/tree/core_test.clj b/test/hitchhiker/tree/core_test.clj index 1fa5d43..79acce9 100644 --- a/test/hitchhiker/tree/core_test.clj +++ b/test/hitchhiker/tree/core_test.clj @@ -1,7 +1,6 @@ (ns hitchhiker.tree.core-test (:refer-clojure :exclude [compare resolve]) (:require [clojure.test :refer :all] - [superv.async :refer [Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) data2 (data-node (->Config 3 3 2) (sorted-map 6 6 7 7 8 8 9 9 10 10)) root (->IndexNode [data1 data2] (promise) [] (->Config 3 3 2))] - (is (= (Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) data2 (data-node (->Config 3 3 2) (sorted-map 6 6 7 7 8 8 9 9 10 10)) @@ -31,13 +30,13 @@ (defn insert-helper [t k] - (Config 3 3 2))) + (Config 3 3 2))) v) b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) (seq (map first b-tree-order)))))) @@ -50,7 +49,7 @@ 1000 (prop/for-all [v (gen/vector gen/int)] (let [sorted-set-order (into (sorted-set) v) - b-tree (reduce insert-helper (Config 3 3 2))) v) + b-tree (reduce insert-helper (Config 3 3 2))) v) b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) (seq (map first b-tree-order)))))) @@ -60,22 +59,22 @@ num gen/nat] (let [set-a (sort the-set) set-b (take num the-set) - b-tree (reduce insert-helper (Config 3 3 2))) set-a) - b-tree-without (reduce #(Config 3 3 2))) set-a) + b-tree-without (reduce #(Config 3 3 2) (sorted-map 1 "1" 2 "2" 3 "3" 4 "4")) root (->IndexNode [data1] (promise) [] (->Config 3 3 2))] - (is (= (map second (lookup-fwd-iter (Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) root (->IndexNode [data1] (promise) [] (->Config 3 3 2))] - (are [x y] (= (map first (lookup-fwd-iter (Config 3 3 2))) (range 5))] - (is (= (map first (lookup-fwd-iter (Config 3 3 2))) (range 10))] - (is (= (map first (lookup-fwd-iter (Config 3 3 2))) (range 6))] - (is (= (map first (lookup-fwd-iter (Config 3 3 2))) (range 5))] + (is (= (map first (lookup-fwd-iter (Config 3 3 2))) (range 10))] + (is (= (map first (lookup-fwd-iter (Config 3 3 2))) (range 6))] + (is (= (map first (lookup-fwd-iter (Config 3 3 2))) the-set)] + (let [b-tree (reduce insert-helper (Config 3 3 2))) the-set)] (check-node-is-balanced b-tree)))) (defspec test-wider-balanced-after-many-inserts 1000 (prop/for-all [the-set (gen/vector (gen/no-shrink gen/int))] - (let [b-tree (reduce insert-helper (Config 200 250 17))) the-set)] + (let [b-tree (reduce insert-helper (Config 200 250 17))) the-set)] (check-node-is-balanced b-tree)))) #_(require '[criterium.core :refer (quick-bench)]) @@ -149,8 +148,8 @@ (let [x-reduced (mod x universe-size)] (condp = op :add (insert-helper t x-reduced) - :del (Config 3 3 2))) + :del (Config 3 3 2))) ops)] ; (println ops) (check-node-is-balanced b-tree))))) @@ -174,10 +173,14 @@ 1000 (prop/for-all [v (gen/vector gen/int)] (let [sorted-set-order (into (sorted-set) v) - b-tree (reduce insert-helper (Config 3 3 2))) v) + b-tree (reduce insert-helper (Config 3 3 2))) v) b-tree-order (map first (lookup-fwd-iter b-tree Integer/MIN_VALUE)) - flushed-tree (:tree (TestingBackend)))) + flushed-tree (:tree (TestingBackend)))) flushed-tree-order (map first (lookup-fwd-iter flushed-tree Integer/MIN_VALUE))] + (when-not (= (seq sorted-set-order) + (seq b-tree-order) + (seq flushed-tree-order)) + (prn "FT" sorted-set-order b-tree-order flushed-tree-order)) (= (seq sorted-set-order) (seq b-tree-order) (seq flushed-tree-order))))) diff --git a/test/hitchhiker/tree/messaging_test.clj b/test/hitchhiker/tree/messaging_test.clj index f566fe8..d6419ea 100644 --- a/test/hitchhiker/tree/messaging_test.clj +++ b/test/hitchhiker/tree/messaging_test.clj @@ -1,15 +1,14 @@ (ns hitchhiker.tree.messaging-test (:require [clojure.test.check.clojure-test :refer [defspec]] - [superv.async :refer [Config 3 3 2))) v) + b-tree (reduce insert (Config 3 3 2))) v) b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) b-tree-order)))) @@ -27,7 +26,7 @@ 1000 (prop/for-all [v (gen/vector gen/int)] (let [sorted-set-order (into (sorted-set) v) - b-tree (reduce insert (Config 3 3 2))) v) + b-tree (reduce insert (Config 3 3 2))) v) b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) b-tree-order)))) @@ -37,8 +36,8 @@ num gen/nat] (let [set-a (sort the-set) set-b (take num the-set) - b-tree (reduce insert (Config 3 3 2))) set-a) - b-tree-without (reduce #(Config 3 3 2))) set-a) + b-tree-without (reduce #(Config 3 3 2))) the-set)] + (let [b-tree (reduce insert (Config 3 3 2))) the-set)] (hitchhiker.tree.core-test/check-node-is-balanced b-tree)))) (defspec test-wider-balanced-after-many-inserts 1000 (prop/for-all [the-set (gen/vector (gen/no-shrink gen/int))] - (let [b-tree (reduce insert (Config 200 220 17))) the-set)] + (let [b-tree (reduce insert (Config 200 220 17))) the-set)] (hitchhiker.tree.core-test/check-node-is-balanced b-tree)))) ;; This test will show how if you apply killerop to the b-tree result, it corrupts @@ -79,7 +78,7 @@ (conj s x-reduced)] :del [(msg/delete t x-reduced) (disj s x-reduced)]))) - [(Config 3 3 2))) #{}] + [(Config 3 3 2))) #{}] ops) f #(case (first %1) :add (insert %2 (second %1)) :del (msg/delete %2 (second %1))) @@ -121,9 +120,9 @@ (condp = op :add [(insert t x-reduced) (conj s x-reduced)] - :del [(Config 3 3 2))) #{}] + [(Config 3 3 2))) #{}] ops)] ; (println ops) (and (= (lookup-fwd-iter b-tree -1) (seq (sort s))) @@ -140,7 +139,7 @@ (conj s x-reduced)] :del [(msg/delete t x-reduced) (disj s x-reduced)]))) - [(Config 3 3 2))) #{}] + [(Config 3 3 2))) #{}] data)] ; (println ops) (println (lookup-fwd-iter b-tree -1)) From d4e19fd3f513c97a9b9a7c497d0f7dd46f607caf Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Thu, 27 Apr 2017 00:17:22 +0200 Subject: [PATCH 05/34] Fix examples for cljs. --- project.clj | 11 ++++++++--- src/hitchhiker/konserve.cljc | 6 +++++- src/hitchhiker/tree/core.cljc | 6 ++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index e230b70..295948b 100644 --- a/project.clj +++ b/project.clj @@ -26,10 +26,15 @@ [figwheel-sidecar "0.5.8"] [com.cemerick/piggieback "0.2.1"] [org.clojure/test.check "0.9.0"]] - ;; need to add dev source path here to get user.clj loaded :source-paths ["src" "dev"] - ;; for CIDER - ;; :plugins [[cider/cider-nrepl "0.12.0"]] + ;; need to add dev source path here to get user.clj loaded + ;; TODO check whether this still makes sense, + ;; workaround for cider cljs REPL + #_:figwheel #_{:nrepl-port 7888 + :nrepl-middleware ["cider.nrepl/cider-middleware" + "cemerick.piggieback/wrap-cljs-repl"]} + :plugins [[lein-figwheel "0.5.8"] + #_[cider/cider-nrepl "0.15.0-SNAPSHOT"]] :repl-options {; for nREPL dev you really need to limit output :init (set! *print-length* 50) :nrepl-middleware [cemerick.piggieback/wrap-cljs-repl]}}} diff --git a/src/hitchhiker/konserve.cljc b/src/hitchhiker/konserve.cljc index 740aafc..53f665c 100644 --- a/src/hitchhiker/konserve.cljc +++ b/src/hitchhiker/konserve.cljc @@ -110,10 +110,14 @@ (go-try (def foo (Config 17 300 (- 300 17)))) 1 1)))) - (go-try (prn (Config 17 300 (- 300 17)))) + (go-try (def foos (Config 17 300 (- 300 17)))) 1 1)) 2 2)))) + (go-try (def bar ( Date: Thu, 27 Apr 2017 00:26:56 +0200 Subject: [PATCH 06/34] Readd temporarily for github review limitations. --- src/hitchhiker/tree/core.clj | 692 ++++++++++++++++++++++++++++++ src/hitchhiker/tree/messaging.clj | 252 +++++++++++ 2 files changed, 944 insertions(+) create mode 100644 src/hitchhiker/tree/core.clj create mode 100644 src/hitchhiker/tree/messaging.clj diff --git a/src/hitchhiker/tree/core.clj b/src/hitchhiker/tree/core.clj new file mode 100644 index 0000000..e7f1139 --- /dev/null +++ b/src/hitchhiker/tree/core.clj @@ -0,0 +1,692 @@ +(ns hitchhiker.tree.core + (:refer-clojure :exclude [compare resolve subvec]) + (:require [clojure.core.rrb-vector :refer [catvec subvec]] + #?(:clj [clojure.pprint :as pp]) + #?(:clj [clojure.core.async :refer [go chan put! IndexNode) + +(defrecord IndexNode [children storage-addr op-buf cfg] + IResolve + (index? [this] true) + (dirty? [this] (not (async/poll! storage-addr))) + (resolve [this] (go this)) ;;TODO this is a hack for testing + (last-key [this] + ;;TODO should optimize by caching to reduce IOps (can use monad) + (last-key (peek children))) + INode + (overflow? [this] + (>= (count children) (* 2 (:index-b cfg)))) + (underflow? [this] + (< (count children) (:index-b cfg))) + (split-node [this] + (let [b (:index-b cfg) + median (nth (index-node-keys children) (dec b)) + [left-buf right-buf] (split-with #(not (pos? (compare (:key %) median))) + ;;TODO this should use msg/affects-key + (sort-by :key op-buf))] + (->Split (->IndexNode (subvec children 0 b) + (promise-chan) + (vec left-buf) + cfg) + (->IndexNode (subvec children b) + (promise-chan) + (vec right-buf) + cfg) + median))) + (merge-node [this other] + (->IndexNode (catvec children (:children other)) + (promise-chan) + (catvec op-buf (:op-buf other)) + cfg)) + (lookup [root key] + ;;This is written like so because it's performance critical + (let [l (dec (count children)) + a (object-array l) + _ (dotimes [i l] + (aset a i (last-key (nth children i)))) + x #?(:clj (Arrays/binarySearch a 0 l key compare) + :cljs (goog.array/binarySearch a key compare))] + (if (neg? x) + (- (inc x)) + x)))) + +#?(:clj + (nippy/extend-freeze IndexNode :b-tree/index-node + [{:keys [storage-addr cfg children op-buf]} data-output] + (nippy/freeze-to-out! data-output cfg) + (nippy/freeze-to-out! data-output children) + ;;TODO apparently RRB-vectors don't freeze correctly; + ;;we'll force it to a normal vector as a workaround + (nippy/freeze-to-out! data-output (into [] op-buf)))) + +#?(:clj + (nippy/extend-thaw :b-tree/index-node + [data-input] + (let [cfg (nippy/thaw-from-in! data-input) + children (nippy/thaw-from-in! data-input) + op-buf (nippy/thaw-from-in! data-input)] + (->IndexNode children nil op-buf cfg)))) + +(defn index-node? + [node] + (instance? IndexNode node)) + +#?(:clj + (defn print-index-node + "Optionally include" + [node ^Writer writer fully-qualified?] + (.write writer (if fully-qualified? + (pr-str IndexNode) + "IndexNode")) + (.write writer (str {:keys (index-node-keys (:children node)) + :children (:children node)})))) + +#?(:clj + (defmethod print-method IndexNode + [node writer] + (print-index-node node writer false))) + +#?(:clj + (defmethod print-dup IndexNode + [node writer] + (print-index-node node writer true))) + +#?(:clj + (defn node-status-bits + [node] + (str "[" + (if (dirty? node) "D" " ") + "]"))) + +#?(:clj + (defmethod pp/simple-dispatch IndexNode + [node] + (let [out ^Writer *out*] + (.write out "IndexNode") + (.write out (node-status-bits node)) + (pp/pprint-logical-block + :prefix "{" :suffix "}" + (pp/pprint-logical-block + (.write out ":keys ") + (pp/write-out (index-node-keys (:children node))) + (pp/pprint-newline :linear)) + (pp/pprint-logical-block + (.write out ":op-buf ") + (pp/write-out (:op-buf node)) + (pp/pprint-newline :linear)) + (pp/pprint-logical-block + (.write out ":children ") + (pp/pprint-newline :mandatory) + (pp/write-out (:children node))))))) + +(defn nth-of-set + "Like nth, but for sorted sets. O(n)" + [set index] + (first (drop index set))) + +(defrecord DataNode [children storage-addr cfg] + IResolve + (index? [this] false) + (resolve [this] (go this)) ;;TODO this is a hack for testing + (dirty? [this] (not (async/poll! storage-addr))) + (last-key [this] + (when (seq children) + (-> children + (rseq) + (first) + (key)))) + INode + ;; Should have between b & 2b-1 children + (overflow? [this] + (>= (count children) (* 2 (:data-b cfg)))) + (underflow? [this] + (< (count children) (:data-b cfg))) + (split-node [this] + (->Split (data-node cfg (into (sorted-map-by compare) (take (:data-b cfg)) children)) + (data-node cfg (into (sorted-map-by compare) (drop (:data-b cfg)) children)) + (nth-of-set children (dec (:data-b cfg))))) + (merge-node [this other] + (data-node cfg (into children (:children other)))) + (lookup [root key] + (let [x #?(:clj (Collections/binarySearch (vec (keys children)) key compare) + :cljs (goog.array/binarySearch (into-array (keys children)) key compare))] + (if (neg? x) + (- (inc x)) + x)))) + +(defn data-node + "Creates a new data node" + [cfg children] + (->DataNode children (promise-chan) cfg)) + +(defn data-node? + [node] + (instance? DataNode node)) + +#?(:clj + (nippy/extend-freeze DataNode :b-tree/data-node + [{:keys [cfg children]} data-output] + (nippy/freeze-to-out! data-output cfg) + (nippy/freeze-to-out! data-output children))) + +#?(:clj + (nippy/extend-thaw :b-tree/data-node + [data-input] + (let [cfg (nippy/thaw-from-in! data-input) + children (nippy/thaw-from-in! data-input)] + (->DataNode children nil cfg)))) + +;(println (b-tree :foo :bar :baz)) +;(pp/pprint (apply b-tree (range 100))) +#?(:clj + (defn print-data-node + [node ^Writer writer fully-qualified?] + (.write writer (if fully-qualified? + (pr-str DataNode) + "DataNode")) + (.write writer (str {:children (:children node)})))) + +#?(:clj + (defmethod print-method DataNode + [node writer] + (print-data-node node writer false))) + +#?(:clj + (defmethod print-dup DataNode + [node writer] + (print-data-node node writer true))) + +#?(:clj + (defmethod pp/simple-dispatch DataNode + [node] + (let [out ^Writer *out*] + (.write out (str "DataNode" + (node-status-bits node))) + (.write out (str {:children (:children node)}))))) + +(defn backtrack-up-path-until + "Given a path (starting with root and ending with an index), searches backwards, + passing each pair of parent & index we just came from to the predicate function. + When that function returns true, we return the path ending in the index for which + it was true, or else we return the empty path" + [path pred] + (loop [path path] + (when (seq path) + (let [from-index (peek path) + tmp (pop path) + parent (peek tmp)] + (if (pred parent from-index) + path + (recur (pop tmp))))))) + + +(defn right-successor + "Given a node on a path, find's that node's right successor node" + [path] + ;(clojure.pprint/pprint path) + ;TODO this function would benefit from a prefetching hint + ; to keep the next several sibs in mem + (go-try + (when-let [common-parent-path + (backtrack-up-path-until + path + (fn [parent index] + (< (inc index) (count (:children parent)))))] + (let [next-index (-> common-parent-path peek inc) + parent (-> common-parent-path pop peek) + new-sibling ( s :children first) + c (if (tree-node? c) + ( % :children first)] + (if (tree-node? c) + ( (interleave sibling-lineage + (repeat 0)) + (butlast)) ; butlast ensures we end w/ node + ] + (-> (pop common-parent-path) + (conj next-index) + (into path-suffix)))))) + + + +#?(:clj + (defn forward-iterator + "Takes the result of a search and returns an iterator going + forward over the tree. Does lg(n) backtracking sometimes." + [path start-key] + (let [start-node (peek path)] + (assert (data-node? start-node)) + (let [first-elements (-> start-node + :children ; Get the indices of it + (subseq >= start-key)) ; skip to the start-index + next-elements (lazy-seq + (when-let [succ ( (:children cur) + ;;TODO what are the semantics for exceeding on the right? currently it's trunc to the last element + (nth index (peek (:children cur))) + (resolve)))) + path' (conj path index child)] + (recur path' child))) + nil)))) + +(defn lookup-key + "Given a B-tree and a key, gets an iterator into the tree" + ([tree key] + (lookup-key tree key nil)) + ([tree key not-found] + (go-try + (-> + ( (IndexNode [left right] (promise-chan) [] cfg)) + node) + (let [index (peek path) + {:keys [children keys] :as parent} (peek (pop path))] + (if (overflow? node) ; splice the split into the parent + ;;TODO refactor paths to be node/index pairs or 2 vectors or something + (let [{:keys [left right median]} (split-node node) + new-children (catvec (conj (subvec children 0 index) + left right) + (subvec children (inc index)))] + (recur (-> parent + (assoc :children new-children) + (dirty!)) + (pop (pop path)))) + (recur (-> parent + ;;TODO this assoc-in seems to be a bottleneck + (assoc-in [:children index] node) + (dirty!)) + (pop (pop path)))))))))) + +;;TODO: cool optimization: when merging children, push as many operations as you can +;;into them to opportunisitcally minimize overall IO costs + +(defn delete + [{:keys [cfg] :as tree} key] + (go-try + (let [path ( (count (:children (nth children (dec index)))) + (count (:children (nth children (inc index))))) + (dec index) ; right sib bigger + :else (inc index)) + node-first? (> bigger-sibling-idx index) ; if true, `node` is left + merged (if node-first? + (merge-node node (IndexNode (catvec (conj old-left-children left right) + old-right-children) + (promise-chan) + op-buf + cfg) + (pop (pop path)))) + (recur (->IndexNode (catvec (conj old-left-children merged) + old-right-children) + (promise-chan) + op-buf + cfg) + (pop (pop path))))) + (recur (->IndexNode (assoc children index node) + (promise-chan) + op-buf + cfg) + (pop (pop path)))))))))) + +(defn b-tree + [cfg & kvs] + (go-try + (loop [[[k v] & r] (partition 2 kvs) + t (data-node cfg (sorted-map-by compare))] + (if k + (recur r (TestingAddr (last-key node) node))) + (delete-addr [_ addr session ])) + +(defn flush-tree + "Given the tree, finds all dirty nodes, delivering addrs into them. + Every dirty node also gets replaced with its TestingAddr. + These form a GC cycle, have fun with the unmanaged memory port :)" + ([tree backend] + (go-try + (let [session (new-session backend) + flushed (> (flush-children (:children tree) backend stats) + InsertOp ") + (.write writer (pr-str (:key op))) + (.write writer ", ") + (.write writer (pr-str (:value op))) + (.write writer ")"))) + +#?(:clj + (defmethod pp/simple-dispatch InsertOp + [op] + (print op))) + +#?(:clj + (defmethod print-method DeleteOp + [op ^Writer writer] + (.write writer "DeleteOp") + (.write writer (str {:key (:key op)} " - " (:tag op))))) + +#?(:clj + (defmethod print-dup DeleteOp + [op ^Writer writer] + (.write writer "(tree.messaging/->DeleteOp ") + (.write writer (pr-str (:key op))) + (.write writer ")"))) + +#?(:clj + (defmethod pp/simple-dispatch DeleteOp + [op] + (print op))) + +(defn enqueue + ([tree msgs] + (go-try + (let [deferred-ops (atom []) + msg-buffers-propagated ( tree + (core/dirty!) + (update-in [:op-buf] into msgs)) + :else ; overflow, should be IndexNode + (do (assert (core/index-node? tree)) + ;(println "overflowing node" (:keys tree) "with buf" (:op-buf tree) + ; "with new msgs" msgs + ; ) + (loop [[child & children] (:children tree) + rebuilt-children [] + msgs (vec (sort-by affects-key ;must be a stable sort + (concat (:op-buf tree) msgs)))] + (let [took-msgs (into [] + (take-while #(>= 0 (core/compare + (affects-key %) + (core/last-key child)))) + msgs) + extra-msgs (into [] + (drop-while #(>= 0 (core/compare + (affects-key %) + (core/last-key child)))) + msgs) + ;_ (println "last-key:" (core/last-key child)) + ;_ (println "goes left:" took-msgs) + ;_ (println "goes right:" extra-msgs) + on-the-last-child? (empty? children) + + ;; Any changes to the current child? + new-child + (cond + (and on-the-last-child? (seq extra-msgs)) + ( tree + (assoc :children (conj rebuilt-children new-child)) + (assoc :op-buf []) + (core/dirty!)) + (recur children (conj rebuilt-children new-child) extra-msgs)))))))))) + + +;;TODO delete in core needs to stop using the index-node constructor to be more +;;careful about how we handle op-bufs during splits and merges. +;; +;;After we've got delete working, lookup, pred, and succ should be fixed +;; +;;broadcast nodes will need IDs so that they can combine during merges... +;; + + +(defn apply-ops-in-path + [path] + (if (>= 1 (count path)) + (:children (peek path)) + (let [ops (->> path + (into [] (comp (filter core/index-node?) + (map :op-buf))) + (rseq) ; highest node should be last in seq + (apply catvec) + (sort-by affects-key)) ;must be a stable sort + this-node-index (-> path pop peek) + parent (-> path pop pop peek) + is-first? (zero? this-node-index) + ;;We'll need to find the smallest last-key of the left siblings along the path + [left-sibs-on-path is-last?] + (loop [path path + is-last? true + left-sibs []] + (if (= 1 (count path)) ; are we at the root? + [left-sibs is-last?] + (let [this-node-index (-> path pop peek) + parent (-> path pop pop peek) + is-first? (zero? this-node-index) + local-last? (= (-> parent :children count dec) + this-node-index)] + (if is-first? + (recur (pop (pop path)) (and is-last? local-last?) left-sibs) + (recur (pop (pop path)) + (and is-last? local-last?) + (conj left-sibs + (nth (:children parent) + (dec this-node-index)))))))) + left-sibs-min-last (when (seq left-sibs-on-path) + (->> left-sibs-on-path + (map core/last-key) + (apply max))) + left-sib-filter (if left-sibs-min-last + (drop-while #(>= 0 (core/compare (affects-key %) + left-sibs-min-last))) + identity) + data-node (peek path) + my-last (core/last-key data-node) + right-side-filter (if is-last? + identity + (take-while #(>= 0 (core/compare (affects-key %) my-last)))) + correct-ops (into [] (comp left-sib-filter right-side-filter) ops) + + ;;We include op if leq my left, and not if leq left's left + ;;TODO we can't apply all ops, we should ensure to only apply ops whose keys are in the defined range, unless we're the last sibling + ] + ;(println "left-sibs-min-last" left-sibs-min-last) + ;(println "is-last?" is-last?) + ;(println "expanding data node" data-node "with ops" correct-ops) + (reduce (fn [coll op] + (apply-op-to-coll op coll)) + (:children data-node) + correct-ops)))) + +(defn lookup + ([tree key] + (lookup tree key nil)) + ([tree key not-found] + (go-try + (let [path (InsertOp key value) + :tag (uuid) + )])) + +(defn delete + [tree key] + (enqueue tree [(assoc (->DeleteOp key) + :tag (uuid) + )])) + +#?(:clj + (defn forward-iterator + "Takes the result of a search and returns an iterator going + forward over the tree. Does lg(n) backtracking sometimes." + [path] + (assert (core/data-node? (peek path))) + (let [first-elements (apply-ops-in-path path) + next-elements (lazy-seq + (when-let [succ ( Date: Thu, 27 Apr 2017 18:04:39 +0200 Subject: [PATCH 07/34] Add a mixed-ops test for konserve. --- test/hitchhiker/konserve_test.clj | 100 ++++++++++++++++++++++++------ test/hitchhiker/redis_test.clj | 2 +- 2 files changed, 82 insertions(+), 20 deletions(-) diff --git a/test/hitchhiker/konserve_test.clj b/test/hitchhiker/konserve_test.clj index 4009a14..8ef3ba7 100644 --- a/test/hitchhiker/konserve_test.clj +++ b/test/hitchhiker/konserve_test.clj @@ -1,21 +1,22 @@ (ns hitchhiker.konserve-test - (:require [clojure.core.rrb-vector :refer [catvec]] + (:refer-clojure :exclude [vec]) + (:require [clojure.core.rrb-vector :refer [catvec vec]] [clojure.test :refer :all] [clojure.test.check.clojure-test :refer [defspec]] [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop] [konserve.filestore :refer [new-fs-store delete-store]] [hitchhiker.konserve :as kons] - [hitchhiker.tree.core :as core] + [hitchhiker.tree.core :refer [ % kons/map->KonserveAddr (assoc :store store @@ -29,9 +30,9 @@ 'hitchhiker.tree.core.IndexNode (fn [{:keys [children cfg op-buf]}] (core/->IndexNode (->> children - catvec) + vec) (promise-chan) - (catvec op-buf) + (vec op-buf) cfg)) 'hitchhiker.tree.messaging.InsertOp msg/map->InsertOp @@ -41,29 +42,90 @@ core/map->Config}) store)) + (deftest simple-konserve-test (testing "Insert and lookup" (let [folder "/tmp/async-hitchhiker-tree-test" store (setup-store folder) backend (kons/->KonserveBackend store) - flushed (Config 1 3 (- 3 1)))) + (Config 1 3 (- 3 1)))) (range 1 11))) backend)) root-key (kons/get-root-key (:tree flushed)) - tree (KonserveBackend store))) + t (:tree flushed)] + #_(when root + (wcar {} (redis/drop-ref root))) + #_(println "flush") + (when-not (:storage-addr t) + (println "TTT" t flushed root op)) + [t (when (:storage-addr t) (Config 3 3 2))) nil #{}] + ops)] + #_(println "Make it to the end of a test, tree has" (count (lookup-fwd-iter b-tree -1)) "keys left") + (let [b-tree-order (lookup-fwd-iter b-tree -1) + res (= b-tree-order (seq (sort set)))] + #_(wcar {} (redis/drop-ref root)) + #_(assert (let [ks (wcar {} (car/keys "*"))] + (or (empty? ks) + (= ["refcount:expiry"] ks))) "End with no keys") + (assert res (str "These are unequal: " (pr-str b-tree-order) " " (pr-str (seq (sort set))))) + (delete-store folder) + res)))) + + +(defspec test-many-keys-bigger-trees + 100 + (mixed-op-seq 800 200 10 1000 1000)) diff --git a/test/hitchhiker/redis_test.clj b/test/hitchhiker/redis_test.clj index 0f91b49..705fbc2 100644 --- a/test/hitchhiker/redis_test.clj +++ b/test/hitchhiker/redis_test.clj @@ -26,7 +26,7 @@ [flush-freq (gen/return [:flush])] [del-freq (gen/tuple (gen/return :del) (gen/no-shrink gen/int))]]) - 0)] + num-ops)] (assert (let [ks (wcar {} (car/keys "*"))] (or (empty? ks) (= ["refcount:expiry"] ks))) From 4870521453ef608a5a3ea2279f127404580ba499 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 30 Apr 2017 00:19:37 +0200 Subject: [PATCH 08/34] Ensure empty store in the beginning. --- test/hitchhiker/konserve_test.clj | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/hitchhiker/konserve_test.clj b/test/hitchhiker/konserve_test.clj index 8ef3ba7..49550fe 100644 --- a/test/hitchhiker/konserve_test.clj +++ b/test/hitchhiker/konserve_test.clj @@ -5,7 +5,7 @@ [clojure.test.check.clojure-test :refer [defspec]] [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop] - [konserve.filestore :refer [new-fs-store delete-store]] + [konserve.filestore :refer [new-fs-store delete-store list-keys]] [hitchhiker.konserve :as kons] [hitchhiker.tree.core :refer [ Date: Thu, 4 May 2017 01:01:13 +0200 Subject: [PATCH 09/34] Use channel for iterators instead of seqs. Preparing first automatic cljs test. --- src/hitchhiker/tree/core.cljc | 74 +++++++---- src/hitchhiker/tree/messaging.cljc | 49 ++++--- test/hitchhiker/konserve_test.clj | 130 ------------------- test/hitchhiker/konserve_test.cljc | 164 ++++++++++++++++++++++++ test/hitchhiker/tree/core_test.clj | 10 +- test/hitchhiker/tree/messaging_test.clj | 2 +- 6 files changed, 254 insertions(+), 175 deletions(-) delete mode 100644 test/hitchhiker/konserve_test.clj create mode 100644 test/hitchhiker/konserve_test.cljc diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index e7f1139..dac0a5c 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -14,6 +14,8 @@ [hitchhiker.tree.core :refer [go-try start-node - :children ; Get the indices of it - (subseq >= start-key)) ; skip to the start-index - next-elements (lazy-seq - (when-let [succ ( start-node + :children ; Get the indices of it + (subseq >= start-key))] + ( % kons/map->KonserveAddr - (assoc :store store - :storage-addr (kons/synthesize-storage-addr (:konserve-key %)))) - 'hitchhiker.tree.core.DataNode - (fn [{:keys [children cfg]}] - (core/->DataNode (into (sorted-map-by - compare) children) - (promise-chan) - cfg)) - 'hitchhiker.tree.core.IndexNode - (fn [{:keys [children cfg op-buf]}] - (core/->IndexNode (->> children - vec) - (promise-chan) - (vec op-buf) - cfg)) - 'hitchhiker.tree.messaging.InsertOp - msg/map->InsertOp - 'hitchhiker.tree.messaging.DeleteOp - msg/map->DeleteOp - 'hitchhiker.tree.core.Config - core/map->Config}) - store)) - - -(deftest simple-konserve-test - (testing "Insert and lookup" - (let [folder "/tmp/async-hitchhiker-tree-test" - store (setup-store folder) - backend (kons/->KonserveBackend store) - flushed (Config 1 3 (- 3 1)))) - (range 1 11))) - backend)) - root-key (kons/get-root-key (:tree flushed)) - tree (KonserveBackend store))) - t (:tree flushed)] - #_(when root - (wcar {} (redis/drop-ref root))) - #_(println "flush") - (when-not (:storage-addr t) - (println "TTT" t flushed root op)) - [t (when (:storage-addr t) (Config 3 3 2))) nil #{}] - ops)] - #_(println "Make it to the end of a test, tree has" (count (lookup-fwd-iter b-tree -1)) "keys left") - (let [b-tree-order (lookup-fwd-iter b-tree -1) - res (= b-tree-order (seq (sort set)))] - #_(wcar {} (redis/drop-ref root)) - #_(assert (let [ks (wcar {} (car/keys "*"))] - (or (empty? ks) - (= ["refcount:expiry"] ks))) "End with no keys") - (assert res (str "These are unequal: " (pr-str b-tree-order) " " (pr-str (seq (sort set))))) - (delete-store folder) - res)))) - - -(defspec test-many-keys-bigger-trees - 100 - (mixed-op-seq 800 200 10 1000 1000)) diff --git a/test/hitchhiker/konserve_test.cljc b/test/hitchhiker/konserve_test.cljc new file mode 100644 index 0000000..ba6dc1e --- /dev/null +++ b/test/hitchhiker/konserve_test.cljc @@ -0,0 +1,164 @@ +(ns hitchhiker.konserve-test + (:refer-clojure :exclude [vec]) + (:require [clojure.core.rrb-vector :refer [catvec vec]] + [clojure.test :refer :all] + [clojure.test.check.clojure-test :refer [defspec]] + [clojure.test.check.generators :as gen] + [clojure.test.check.properties :as prop] + #?(:clj [konserve.filestore :refer [new-fs-store delete-store list-keys]] + :cljs [konserve.memory :refer [new-mem-store]]) + [hitchhiker.konserve :as kons] + [hitchhiker.tree.core :refer [ % kons/map->KonserveAddr + (assoc :store store + :storage-addr (kons/synthesize-storage-addr (:konserve-key %)))) + 'hitchhiker.tree.core.DataNode + (fn [{:keys [children cfg]}] + (core/->DataNode (into (sorted-map-by + compare) children) + (promise-chan) + cfg)) + 'hitchhiker.tree.core.IndexNode + (fn [{:keys [children cfg op-buf]}] + (core/->IndexNode (->> children + vec) + (promise-chan) + (vec op-buf) + cfg)) + 'hitchhiker.tree.messaging.InsertOp + msg/map->InsertOp + 'hitchhiker.tree.messaging.DeleteOp + msg/map->DeleteOp + 'hitchhiker.tree.core.Config + core/map->Config}) + store)) + +;; for cljs tests +(defn iter-helper [tree key] + (go-try + (let [iter-ch (async/chan) + path (KonserveBackend store) + flushed (Config 1 3 (- 3 1)))) + (range 1 11))) + backend)) + root-key (kons/get-root-key (:tree flushed)) + tree (KonserveBackend store) + flushed (Config 1 3 (- 3 1)))) + (range 1 11))) + backend)) + root-key (kons/get-root-key (:tree flushed)) + tree (KonserveBackend store))) + t (:tree flushed)] + (when-not (:storage-addr t) + (println "TTT" t flushed root op)) + [t (when (:storage-addr t) (Config 3 3 2))) nil #{}] + ops)] + #_(println "Make it to the end of a test, tree has" (count (lookup-fwd-iter b-tree -1)) "keys left") + (let [b-tree-order (lookup-fwd-iter b-tree -1) + res (= b-tree-order (seq (sort set)))] + #_(wcar {} (redis/drop-ref root)) + #_(assert (let [ks (wcar {} (car/keys "*"))] + (or (empty? ks) + (= ["refcount:expiry"] ks))) "End with no keys") + (assert res (str "These are unequal: " (pr-str b-tree-order) " " (pr-str (seq (sort set))))) + (delete-store folder) + res))))) + + +#?(:clj + (defspec test-many-keys-bigger-trees + 100 + (mixed-op-seq 800 200 10 1000 1000))) diff --git a/test/hitchhiker/tree/core_test.clj b/test/hitchhiker/tree/core_test.clj index 79acce9..f0127ae 100644 --- a/test/hitchhiker/tree/core_test.clj +++ b/test/hitchhiker/tree/core_test.clj @@ -5,7 +5,14 @@ [clojure.test.check.clojure-test :refer [defspec]] [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop] - [hitchhiker.tree.core :refer :all])) + [hitchhiker.tree.core :refer :all] + [clojure.core.async :as async])) + +(deftest reduce<-test + (is (= 45 (Config 3 3 2) (sorted-map 1 "1" 2 "2" 3 "3" 4 "4")) root (->IndexNode [data1] (promise) [] (->Config 3 3 2))] diff --git a/test/hitchhiker/tree/messaging_test.clj b/test/hitchhiker/tree/messaging_test.clj index d6419ea..367a56c 100644 --- a/test/hitchhiker/tree/messaging_test.clj +++ b/test/hitchhiker/tree/messaging_test.clj @@ -153,7 +153,7 @@ (mixed-op-seq 0.5 250 5000)) (defspec test-many-keys-bigger-trees - 100 + 1000 (mixed-op-seq 0.8 1000 1000)) (defspec test-sparse-ops From a3661549ca550d43c032c9b233d93649695de2e5 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Fri, 5 May 2017 18:25:17 +0200 Subject: [PATCH 10/34] Add a first automatic cljs test for node. --- project.clj | 27 +- src/hitchhiker/konserve.cljc | 26 +- src/hitchhiker/tree/core.clj | 692 ----------------------------- src/hitchhiker/tree/core.cljc | 28 +- src/hitchhiker/tree/messaging.clj | 252 ----------- src/hitchhiker/tree/messaging.cljc | 15 +- test/hitchhiker/konserve_test.cljc | 132 +++--- test_node.js | 26 ++ 8 files changed, 150 insertions(+), 1048 deletions(-) delete mode 100644 src/hitchhiker/tree/core.clj delete mode 100644 src/hitchhiker/tree/messaging.clj create mode 100644 test_node.js diff --git a/project.clj b/project.clj index 295948b..3203edc 100644 --- a/project.clj +++ b/project.clj @@ -4,13 +4,14 @@ :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :dependencies [[org.clojure/clojure "1.8.0"] + [org.clojure/clojurescript "1.8.51" :scope "provided"] [org.clojure/core.memoize "0.5.8"] [com.taoensso/carmine "2.12.2"] [org.clojure/core.rrb-vector "0.0.11"] [org.clojure/core.cache "0.6.5"] [io.replikativ/incognito "0.2.2-SNAPSHOT"] - [io.replikativ/konserve "0.4.8"]] + [io.replikativ/konserve "0.4.9"]] :aliases {"bench" ["with-profile" "profiling" "run" "-m" "hitchhiker.bench"]} :jvm-opts ["-server" "-Xmx3700m" "-Xms3700m"] :profiles {:test @@ -27,14 +28,7 @@ [com.cemerick/piggieback "0.2.1"] [org.clojure/test.check "0.9.0"]] :source-paths ["src" "dev"] - ;; need to add dev source path here to get user.clj loaded - ;; TODO check whether this still makes sense, - ;; workaround for cider cljs REPL - #_:figwheel #_{:nrepl-port 7888 - :nrepl-middleware ["cider.nrepl/cider-middleware" - "cemerick.piggieback/wrap-cljs-repl"]} - :plugins [[lein-figwheel "0.5.8"] - #_[cider/cider-nrepl "0.15.0-SNAPSHOT"]] + :plugins [[lein-figwheel "0.5.8"]] :repl-options {; for nREPL dev you really need to limit output :init (set! *print-length* 50) :nrepl-middleware [cemerick.piggieback/wrap-cljs-repl]}}} @@ -48,7 +42,20 @@ :compiler {:main hitchhiker.tree.core :asset-path "js/out" :output-to "resources/public/js/core.js" - :output-dir "resources/public/js/out" }}]} + :output-dir "resources/public/js/out" }} + ;; inspired by datascript project.clj + {:id "test" + :source-paths ["src" "test" "dev"] + :compiler { + :main hitchhiker-tree.konserve-test + :output-to "target/test.js" + :output-dir "target/none" + :optimizations :none + :source-map true + :recompile-dependents false + :parallel-build true + }} + ]} :plugins [[lein-figwheel "0.5.8"] [lein-cljsbuild "1.1.4" :exclusions [[org.clojure/clojure]]]]) diff --git a/src/hitchhiker/konserve.cljc b/src/hitchhiker/konserve.cljc index 53f665c..eeed82e 100644 --- a/src/hitchhiker/konserve.cljc +++ b/src/hitchhiker/konserve.cljc @@ -44,7 +44,7 @@ :storage-addr nil) cs)))) (assoc node :storage-addr nil))] (let [id (uuid pnode)] - (KonserveAddr store (core/last-key node) id (synthesize-storage-addr id)))))) (delete-addr [_ addr session] (swap! session update-in :deletes inc))) @@ -61,7 +61,8 @@ (KonserveAddr store last-key root-key (synthesize-storage-addr root-key))))))) -(defn add-read-handlers [store] + +(defn add-hitchhiker-tree-handlers [store] (swap! (:read-handlers store) merge {'hitchhiker.konserve.KonserveAddr #(-> % map->KonserveAddr @@ -76,9 +77,9 @@ 'hitchhiker.tree.core.IndexNode (fn [{:keys [children cfg op-buf]}] (core/->IndexNode (->> children - catvec) + vec) (promise-chan) - (catvec op-buf) + (vec op-buf) cfg)) 'hitchhiker.tree.messaging.InsertOp msg/map->InsertOp @@ -86,8 +87,25 @@ msg/map->DeleteOp 'hitchhiker.tree.core.Config core/map->Config}) + (swap! (:write-handlers store) merge + {'hitchhiker.konserve.KonserveAddr + (fn [addr] + (assoc addr + :store nil + :storage-addr nil)) + 'hitchhiker.tree.core.DataNode + (fn [node] + (assoc node :storage-addr nil)) + 'hitchhiker.tree.core.IndexNode + (fn [node] + (-> node + (assoc :storage-addr nil) + (update-in [:children] + (fn [cs] (map #(assoc % :store nil :storage-addr nil) cs)))))}) store) + + (comment diff --git a/src/hitchhiker/tree/core.clj b/src/hitchhiker/tree/core.clj deleted file mode 100644 index e7f1139..0000000 --- a/src/hitchhiker/tree/core.clj +++ /dev/null @@ -1,692 +0,0 @@ -(ns hitchhiker.tree.core - (:refer-clojure :exclude [compare resolve subvec]) - (:require [clojure.core.rrb-vector :refer [catvec subvec]] - #?(:clj [clojure.pprint :as pp]) - #?(:clj [clojure.core.async :refer [go chan put! IndexNode) - -(defrecord IndexNode [children storage-addr op-buf cfg] - IResolve - (index? [this] true) - (dirty? [this] (not (async/poll! storage-addr))) - (resolve [this] (go this)) ;;TODO this is a hack for testing - (last-key [this] - ;;TODO should optimize by caching to reduce IOps (can use monad) - (last-key (peek children))) - INode - (overflow? [this] - (>= (count children) (* 2 (:index-b cfg)))) - (underflow? [this] - (< (count children) (:index-b cfg))) - (split-node [this] - (let [b (:index-b cfg) - median (nth (index-node-keys children) (dec b)) - [left-buf right-buf] (split-with #(not (pos? (compare (:key %) median))) - ;;TODO this should use msg/affects-key - (sort-by :key op-buf))] - (->Split (->IndexNode (subvec children 0 b) - (promise-chan) - (vec left-buf) - cfg) - (->IndexNode (subvec children b) - (promise-chan) - (vec right-buf) - cfg) - median))) - (merge-node [this other] - (->IndexNode (catvec children (:children other)) - (promise-chan) - (catvec op-buf (:op-buf other)) - cfg)) - (lookup [root key] - ;;This is written like so because it's performance critical - (let [l (dec (count children)) - a (object-array l) - _ (dotimes [i l] - (aset a i (last-key (nth children i)))) - x #?(:clj (Arrays/binarySearch a 0 l key compare) - :cljs (goog.array/binarySearch a key compare))] - (if (neg? x) - (- (inc x)) - x)))) - -#?(:clj - (nippy/extend-freeze IndexNode :b-tree/index-node - [{:keys [storage-addr cfg children op-buf]} data-output] - (nippy/freeze-to-out! data-output cfg) - (nippy/freeze-to-out! data-output children) - ;;TODO apparently RRB-vectors don't freeze correctly; - ;;we'll force it to a normal vector as a workaround - (nippy/freeze-to-out! data-output (into [] op-buf)))) - -#?(:clj - (nippy/extend-thaw :b-tree/index-node - [data-input] - (let [cfg (nippy/thaw-from-in! data-input) - children (nippy/thaw-from-in! data-input) - op-buf (nippy/thaw-from-in! data-input)] - (->IndexNode children nil op-buf cfg)))) - -(defn index-node? - [node] - (instance? IndexNode node)) - -#?(:clj - (defn print-index-node - "Optionally include" - [node ^Writer writer fully-qualified?] - (.write writer (if fully-qualified? - (pr-str IndexNode) - "IndexNode")) - (.write writer (str {:keys (index-node-keys (:children node)) - :children (:children node)})))) - -#?(:clj - (defmethod print-method IndexNode - [node writer] - (print-index-node node writer false))) - -#?(:clj - (defmethod print-dup IndexNode - [node writer] - (print-index-node node writer true))) - -#?(:clj - (defn node-status-bits - [node] - (str "[" - (if (dirty? node) "D" " ") - "]"))) - -#?(:clj - (defmethod pp/simple-dispatch IndexNode - [node] - (let [out ^Writer *out*] - (.write out "IndexNode") - (.write out (node-status-bits node)) - (pp/pprint-logical-block - :prefix "{" :suffix "}" - (pp/pprint-logical-block - (.write out ":keys ") - (pp/write-out (index-node-keys (:children node))) - (pp/pprint-newline :linear)) - (pp/pprint-logical-block - (.write out ":op-buf ") - (pp/write-out (:op-buf node)) - (pp/pprint-newline :linear)) - (pp/pprint-logical-block - (.write out ":children ") - (pp/pprint-newline :mandatory) - (pp/write-out (:children node))))))) - -(defn nth-of-set - "Like nth, but for sorted sets. O(n)" - [set index] - (first (drop index set))) - -(defrecord DataNode [children storage-addr cfg] - IResolve - (index? [this] false) - (resolve [this] (go this)) ;;TODO this is a hack for testing - (dirty? [this] (not (async/poll! storage-addr))) - (last-key [this] - (when (seq children) - (-> children - (rseq) - (first) - (key)))) - INode - ;; Should have between b & 2b-1 children - (overflow? [this] - (>= (count children) (* 2 (:data-b cfg)))) - (underflow? [this] - (< (count children) (:data-b cfg))) - (split-node [this] - (->Split (data-node cfg (into (sorted-map-by compare) (take (:data-b cfg)) children)) - (data-node cfg (into (sorted-map-by compare) (drop (:data-b cfg)) children)) - (nth-of-set children (dec (:data-b cfg))))) - (merge-node [this other] - (data-node cfg (into children (:children other)))) - (lookup [root key] - (let [x #?(:clj (Collections/binarySearch (vec (keys children)) key compare) - :cljs (goog.array/binarySearch (into-array (keys children)) key compare))] - (if (neg? x) - (- (inc x)) - x)))) - -(defn data-node - "Creates a new data node" - [cfg children] - (->DataNode children (promise-chan) cfg)) - -(defn data-node? - [node] - (instance? DataNode node)) - -#?(:clj - (nippy/extend-freeze DataNode :b-tree/data-node - [{:keys [cfg children]} data-output] - (nippy/freeze-to-out! data-output cfg) - (nippy/freeze-to-out! data-output children))) - -#?(:clj - (nippy/extend-thaw :b-tree/data-node - [data-input] - (let [cfg (nippy/thaw-from-in! data-input) - children (nippy/thaw-from-in! data-input)] - (->DataNode children nil cfg)))) - -;(println (b-tree :foo :bar :baz)) -;(pp/pprint (apply b-tree (range 100))) -#?(:clj - (defn print-data-node - [node ^Writer writer fully-qualified?] - (.write writer (if fully-qualified? - (pr-str DataNode) - "DataNode")) - (.write writer (str {:children (:children node)})))) - -#?(:clj - (defmethod print-method DataNode - [node writer] - (print-data-node node writer false))) - -#?(:clj - (defmethod print-dup DataNode - [node writer] - (print-data-node node writer true))) - -#?(:clj - (defmethod pp/simple-dispatch DataNode - [node] - (let [out ^Writer *out*] - (.write out (str "DataNode" - (node-status-bits node))) - (.write out (str {:children (:children node)}))))) - -(defn backtrack-up-path-until - "Given a path (starting with root and ending with an index), searches backwards, - passing each pair of parent & index we just came from to the predicate function. - When that function returns true, we return the path ending in the index for which - it was true, or else we return the empty path" - [path pred] - (loop [path path] - (when (seq path) - (let [from-index (peek path) - tmp (pop path) - parent (peek tmp)] - (if (pred parent from-index) - path - (recur (pop tmp))))))) - - -(defn right-successor - "Given a node on a path, find's that node's right successor node" - [path] - ;(clojure.pprint/pprint path) - ;TODO this function would benefit from a prefetching hint - ; to keep the next several sibs in mem - (go-try - (when-let [common-parent-path - (backtrack-up-path-until - path - (fn [parent index] - (< (inc index) (count (:children parent)))))] - (let [next-index (-> common-parent-path peek inc) - parent (-> common-parent-path pop peek) - new-sibling ( s :children first) - c (if (tree-node? c) - ( % :children first)] - (if (tree-node? c) - ( (interleave sibling-lineage - (repeat 0)) - (butlast)) ; butlast ensures we end w/ node - ] - (-> (pop common-parent-path) - (conj next-index) - (into path-suffix)))))) - - - -#?(:clj - (defn forward-iterator - "Takes the result of a search and returns an iterator going - forward over the tree. Does lg(n) backtracking sometimes." - [path start-key] - (let [start-node (peek path)] - (assert (data-node? start-node)) - (let [first-elements (-> start-node - :children ; Get the indices of it - (subseq >= start-key)) ; skip to the start-index - next-elements (lazy-seq - (when-let [succ ( (:children cur) - ;;TODO what are the semantics for exceeding on the right? currently it's trunc to the last element - (nth index (peek (:children cur))) - (resolve)))) - path' (conj path index child)] - (recur path' child))) - nil)))) - -(defn lookup-key - "Given a B-tree and a key, gets an iterator into the tree" - ([tree key] - (lookup-key tree key nil)) - ([tree key not-found] - (go-try - (-> - ( (IndexNode [left right] (promise-chan) [] cfg)) - node) - (let [index (peek path) - {:keys [children keys] :as parent} (peek (pop path))] - (if (overflow? node) ; splice the split into the parent - ;;TODO refactor paths to be node/index pairs or 2 vectors or something - (let [{:keys [left right median]} (split-node node) - new-children (catvec (conj (subvec children 0 index) - left right) - (subvec children (inc index)))] - (recur (-> parent - (assoc :children new-children) - (dirty!)) - (pop (pop path)))) - (recur (-> parent - ;;TODO this assoc-in seems to be a bottleneck - (assoc-in [:children index] node) - (dirty!)) - (pop (pop path)))))))))) - -;;TODO: cool optimization: when merging children, push as many operations as you can -;;into them to opportunisitcally minimize overall IO costs - -(defn delete - [{:keys [cfg] :as tree} key] - (go-try - (let [path ( (count (:children (nth children (dec index)))) - (count (:children (nth children (inc index))))) - (dec index) ; right sib bigger - :else (inc index)) - node-first? (> bigger-sibling-idx index) ; if true, `node` is left - merged (if node-first? - (merge-node node (IndexNode (catvec (conj old-left-children left right) - old-right-children) - (promise-chan) - op-buf - cfg) - (pop (pop path)))) - (recur (->IndexNode (catvec (conj old-left-children merged) - old-right-children) - (promise-chan) - op-buf - cfg) - (pop (pop path))))) - (recur (->IndexNode (assoc children index node) - (promise-chan) - op-buf - cfg) - (pop (pop path)))))))))) - -(defn b-tree - [cfg & kvs] - (go-try - (loop [[[k v] & r] (partition 2 kvs) - t (data-node cfg (sorted-map-by compare))] - (if k - (recur r (TestingAddr (last-key node) node))) - (delete-addr [_ addr session ])) - -(defn flush-tree - "Given the tree, finds all dirty nodes, delivering addrs into them. - Every dirty node also gets replaced with its TestingAddr. - These form a GC cycle, have fun with the unmanaged memory port :)" - ([tree backend] - (go-try - (let [session (new-session backend) - flushed (> (flush-children (:children tree) backend stats) - > (flush-children (:children tree) backend stats) - > (flush-children (:children tree) backend stats) + InsertOp ") - (.write writer (pr-str (:key op))) - (.write writer ", ") - (.write writer (pr-str (:value op))) - (.write writer ")"))) - -#?(:clj - (defmethod pp/simple-dispatch InsertOp - [op] - (print op))) - -#?(:clj - (defmethod print-method DeleteOp - [op ^Writer writer] - (.write writer "DeleteOp") - (.write writer (str {:key (:key op)} " - " (:tag op))))) - -#?(:clj - (defmethod print-dup DeleteOp - [op ^Writer writer] - (.write writer "(tree.messaging/->DeleteOp ") - (.write writer (pr-str (:key op))) - (.write writer ")"))) - -#?(:clj - (defmethod pp/simple-dispatch DeleteOp - [op] - (print op))) - -(defn enqueue - ([tree msgs] - (go-try - (let [deferred-ops (atom []) - msg-buffers-propagated ( tree - (core/dirty!) - (update-in [:op-buf] into msgs)) - :else ; overflow, should be IndexNode - (do (assert (core/index-node? tree)) - ;(println "overflowing node" (:keys tree) "with buf" (:op-buf tree) - ; "with new msgs" msgs - ; ) - (loop [[child & children] (:children tree) - rebuilt-children [] - msgs (vec (sort-by affects-key ;must be a stable sort - (concat (:op-buf tree) msgs)))] - (let [took-msgs (into [] - (take-while #(>= 0 (core/compare - (affects-key %) - (core/last-key child)))) - msgs) - extra-msgs (into [] - (drop-while #(>= 0 (core/compare - (affects-key %) - (core/last-key child)))) - msgs) - ;_ (println "last-key:" (core/last-key child)) - ;_ (println "goes left:" took-msgs) - ;_ (println "goes right:" extra-msgs) - on-the-last-child? (empty? children) - - ;; Any changes to the current child? - new-child - (cond - (and on-the-last-child? (seq extra-msgs)) - ( tree - (assoc :children (conj rebuilt-children new-child)) - (assoc :op-buf []) - (core/dirty!)) - (recur children (conj rebuilt-children new-child) extra-msgs)))))))))) - - -;;TODO delete in core needs to stop using the index-node constructor to be more -;;careful about how we handle op-bufs during splits and merges. -;; -;;After we've got delete working, lookup, pred, and succ should be fixed -;; -;;broadcast nodes will need IDs so that they can combine during merges... -;; - - -(defn apply-ops-in-path - [path] - (if (>= 1 (count path)) - (:children (peek path)) - (let [ops (->> path - (into [] (comp (filter core/index-node?) - (map :op-buf))) - (rseq) ; highest node should be last in seq - (apply catvec) - (sort-by affects-key)) ;must be a stable sort - this-node-index (-> path pop peek) - parent (-> path pop pop peek) - is-first? (zero? this-node-index) - ;;We'll need to find the smallest last-key of the left siblings along the path - [left-sibs-on-path is-last?] - (loop [path path - is-last? true - left-sibs []] - (if (= 1 (count path)) ; are we at the root? - [left-sibs is-last?] - (let [this-node-index (-> path pop peek) - parent (-> path pop pop peek) - is-first? (zero? this-node-index) - local-last? (= (-> parent :children count dec) - this-node-index)] - (if is-first? - (recur (pop (pop path)) (and is-last? local-last?) left-sibs) - (recur (pop (pop path)) - (and is-last? local-last?) - (conj left-sibs - (nth (:children parent) - (dec this-node-index)))))))) - left-sibs-min-last (when (seq left-sibs-on-path) - (->> left-sibs-on-path - (map core/last-key) - (apply max))) - left-sib-filter (if left-sibs-min-last - (drop-while #(>= 0 (core/compare (affects-key %) - left-sibs-min-last))) - identity) - data-node (peek path) - my-last (core/last-key data-node) - right-side-filter (if is-last? - identity - (take-while #(>= 0 (core/compare (affects-key %) my-last)))) - correct-ops (into [] (comp left-sib-filter right-side-filter) ops) - - ;;We include op if leq my left, and not if leq left's left - ;;TODO we can't apply all ops, we should ensure to only apply ops whose keys are in the defined range, unless we're the last sibling - ] - ;(println "left-sibs-min-last" left-sibs-min-last) - ;(println "is-last?" is-last?) - ;(println "expanding data node" data-node "with ops" correct-ops) - (reduce (fn [coll op] - (apply-op-to-coll op coll)) - (:children data-node) - correct-ops)))) - -(defn lookup - ([tree key] - (lookup tree key nil)) - ([tree key not-found] - (go-try - (let [path (InsertOp key value) - :tag (uuid) - )])) - -(defn delete - [tree key] - (enqueue tree [(assoc (->DeleteOp key) - :tag (uuid) - )])) - -#?(:clj - (defn forward-iterator - "Takes the result of a search and returns an iterator going - forward over the tree. Does lg(n) backtracking sometimes." - [path] - (assert (core/data-node? (peek path))) - (let [first-elements (apply-ops-in-path path) - next-elements (lazy-seq - (when-let [succ ( % kons/map->KonserveAddr - (assoc :store store - :storage-addr (kons/synthesize-storage-addr (:konserve-key %)))) - 'hitchhiker.tree.core.DataNode - (fn [{:keys [children cfg]}] - (core/->DataNode (into (sorted-map-by - compare) children) - (promise-chan) - cfg)) - 'hitchhiker.tree.core.IndexNode - (fn [{:keys [children cfg op-buf]}] - (core/->IndexNode (->> children - vec) - (promise-chan) - (vec op-buf) - cfg)) - 'hitchhiker.tree.messaging.InsertOp - msg/map->InsertOp - 'hitchhiker.tree.messaging.DeleteOp - msg/map->DeleteOp - 'hitchhiker.tree.core.Config - core/map->Config}) - store)) ;; for cljs tests -(defn iter-helper [tree key] - (go-try - (let [iter-ch (async/chan) - path (KonserveBackend store) - flushed (Config 1 3 (- 3 1)))) - (range 1 11))) - backend)) - root-key (kons/get-root-key (:tree flushed)) - tree (KonserveBackend store) + init-tree (Config 1 3 (- 3 1)))) + (range 1 11))) + flushed (KonserveBackend store) flushed (KonserveBackend store))) t (:tree flushed)] - (when-not (:storage-addr t) + #_(when-not (:storage-addr t) (println "TTT" t flushed root op)) - [t (when (:storage-addr t) (Config 3 3 2))) nil #{}] @@ -162,3 +144,15 @@ (defspec test-many-keys-bigger-trees 100 (mixed-op-seq 800 200 10 1000 1000))) + +#?(:cljs + (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] + (if (cljs.test/successful? m) + (println "Success!") + (println "FAIL")))) + +#?(:cljs + (defn ^:export test-all [] + (run-tests) + #_(go-try + (.log js/console "konserve tests:" (clj->js ( Date: Fri, 5 May 2017 23:54:18 +0200 Subject: [PATCH 11/34] Add property based test for cljs. This has a heavy payload of 1 MiB, working around the lacking async support in test-check: https://dev.clojure.org/jira/browse/TCHECK-128 --- test/hitchhiker/konserve_test.cljc | 123 +++++++++++++++++++---------- test/hitchhiker/ops.cljc | 3 + 2 files changed, 86 insertions(+), 40 deletions(-) create mode 100644 test/hitchhiker/ops.cljc diff --git a/test/hitchhiker/konserve_test.cljc b/test/hitchhiker/konserve_test.cljc index e4a0115..e82fb66 100644 --- a/test/hitchhiker/konserve_test.cljc +++ b/test/hitchhiker/konserve_test.cljc @@ -13,6 +13,7 @@ [hitchhiker.tree.core #?(:clj :refer :cljs :refer-macros) [KonserveBackend store))) + t (:tree flushed)] + [t (Config 3 3 2))) nil #{}] + ops))] + (let [b-tree-order (map first (KonserveBackend store))) - t (:tree flushed)] - #_(when-not (:storage-addr t) - (println "TTT" t flushed root op)) - [t (Config 3 3 2))) nil #{}] - ops)] - #_(println "Make it to the end of a test, tree has" (count (lookup-fwd-iter b-tree -1)) "keys left") - (let [b-tree-order (lookup-fwd-iter b-tree -1) - res (= b-tree-order (seq (sort set)))] - #_(wcar {} (redis/drop-ref root)) - #_(assert (let [ks (wcar {} (car/keys "*"))] - (or (empty? ks) - (= ["refcount:expiry"] ks))) "End with no keys") - (assert res (str "These are unequal: " (pr-str b-tree-order) " " (pr-str (seq (sort set))))) - (delete-store folder) - res))))) + ( Date: Sat, 6 May 2017 01:52:24 +0200 Subject: [PATCH 12/34] Fix the redis backend and benchmark. --- env/profiling/hitchhiker/bench.clj | 119 +++++++++++++++-------------- src/hitchhiker/redis.clj | 34 ++++----- src/hitchhiker/tree/core.cljc | 34 +++++---- 3 files changed, 96 insertions(+), 91 deletions(-) diff --git a/env/profiling/hitchhiker/bench.clj b/env/profiling/hitchhiker/bench.clj index d47cccc..c9e13bf 100644 --- a/env/profiling/hitchhiker/bench.clj +++ b/env/profiling/hitchhiker/bench.clj @@ -4,7 +4,7 @@ [clojure.tools.cli :refer [parse-opts]] [excel-templates.build :as excel] [hitchhiker.redis :as redis] - [hitchhiker.tree.core :as core] + [hitchhiker.tree.core :refer [Config b b 0)) + {:structure (Config b b 0))) :insert core/insert :delete core/delete - :flush (fn [x] (core/flush-tree x backend))}) + :flush (fn [x] (Config sqrt-b b (- b sqrt-b))) + {:structure (Config sqrt-b b (- b sqrt-b)))) :insert msg/insert :delete msg/delete - :flush (fn [x] (core/flush-tree x backend))})) + :flush (fn [x] ( (into {} last-flush) - (assoc :ins-avg-ns avg-ns - (if inserting? - :insert - :delete) true - :n i')))))) - (cond - (seq data) - (recur data - (if log-inserts - 0 - (+ t (- after before))) - tree' - (if stats (merge-with + last-flush @stats) last-flush) - i' - inserting? - @updated-outputs) - inserting? - (recur (delete-xform dataset) - 0 - tree' - nil - i' - false - @updated-outputs) - :else - @updated-outputs))))) + ( (into {} last-flush) + (assoc :ins-avg-ns avg-ns + (if inserting? + :insert + :delete) true + :n i')))))) + (cond + (seq data) + (recur data + (if log-inserts + 0 + (+ t (- after before))) + tree' + (if stats (merge-with + last-flush @stats) last-flush) + i' + inserting? + @updated-outputs) + inserting? + (recur (delete-xform dataset) + 0 + tree' + nil + i' + false + @updated-outputs) + :else + @updated-outputs))))))) (def options [["-n" "--num-operations NUM_OPS" "The number of elements that will be applied to the data structure" diff --git a/src/hitchhiker/redis.clj b/src/hitchhiker/redis.clj index c38be84..0ac1a52 100644 --- a/src/hitchhiker/redis.clj +++ b/src/hitchhiker/redis.clj @@ -144,7 +144,7 @@ (cache/hit c redis-key) (cache/miss c redis-key run)))) val (cache/lookup cs redis-key)] - (if val @val @run))) + (if val (> (flush-children (:children tree) backend stats) - > (flush-children (:children tree) backend stats) + Date: Sat, 6 May 2017 02:11:56 +0200 Subject: [PATCH 13/34] Fix missing redis test. --- test/hitchhiker/redis_test.clj | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/hitchhiker/redis_test.clj b/test/hitchhiker/redis_test.clj index 705fbc2..646ac8d 100644 --- a/test/hitchhiker/redis_test.clj +++ b/test/hitchhiker/redis_test.clj @@ -3,7 +3,7 @@ [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop] [hitchhiker.redis :as redis] - [hitchhiker.tree.core :as core] + [hitchhiker.tree.core :refer [RedisBackend)))] + :flush (let [t (:tree (RedisBackend))))] (when root (wcar {} (redis/drop-ref root))) #_(println "flush") - [t @(:storage-addr t) set]) - :add (do #_(println "add") [(insert t x-reduced) root (conj set x-reduced)]) - :del (do #_(println "del") [(msg/delete t x-reduced) root (disj set x-reduced)])))) - [(core/b-tree (core/->Config 3 3 2)) nil #{}] - ops)] + [t (Config 3 3 2))) nil #{}] + ops)] #_(println "Make it to the end of a test, tree has" (count (lookup-fwd-iter b-tree -1)) "keys left") (let [b-tree-order (lookup-fwd-iter b-tree -1) res (= b-tree-order (seq (sort set)))] From f7bfa74ffef19966473bba39a786cd8352a7437b Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 6 May 2017 02:18:47 +0200 Subject: [PATCH 14/34] Fix sorted-set benchmark. --- env/profiling/hitchhiker/bench.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/env/profiling/hitchhiker/bench.clj b/env/profiling/hitchhiker/bench.clj index c9e13bf..2899070 100644 --- a/env/profiling/hitchhiker/bench.clj +++ b/env/profiling/hitchhiker/bench.clj @@ -35,8 +35,8 @@ "Returns a sorted set" [] {:structure (sorted-map) - :insert assoc - :delete dissoc + :insert (go-try assoc) + :delete (go-try dissoc) :flush (fn [set] {:tree set :stats (atom {})})}) From 05e4ed323c50d1b776f2b450bfb59c610726f6e8 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 6 May 2017 02:28:54 +0200 Subject: [PATCH 15/34] Next attempt at fixing sorted-set. --- env/profiling/hitchhiker/bench.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/env/profiling/hitchhiker/bench.clj b/env/profiling/hitchhiker/bench.clj index 2899070..1f5aa5c 100644 --- a/env/profiling/hitchhiker/bench.clj +++ b/env/profiling/hitchhiker/bench.clj @@ -35,8 +35,8 @@ "Returns a sorted set" [] {:structure (sorted-map) - :insert (go-try assoc) - :delete (go-try dissoc) + :insert (fn [m k v] (go-try (assoc m k v))) + :delete (fn [m k] (go-try (dissoc m k))) :flush (fn [set] {:tree set :stats (atom {})})}) From 205b7bbbef5f10ff41205b6fe7726addf8c0d422 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 10 May 2017 20:23:54 +0200 Subject: [PATCH 16/34] Fix redis test. --- test/hitchhiker/redis_test.clj | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/hitchhiker/redis_test.clj b/test/hitchhiker/redis_test.clj index 646ac8d..d2f86cf 100644 --- a/test/hitchhiker/redis_test.clj +++ b/test/hitchhiker/redis_test.clj @@ -16,6 +16,7 @@ [t v] (seq (map first (msg/lookup-fwd-iter t v)))) + (defn mixed-op-seq "This is like the basic mixed-op-seq tests, but it also mixes in flushes to redis and automatically deletes the old tree" @@ -29,7 +30,8 @@ num-ops)] (assert (let [ks (wcar {} (car/keys "*"))] (or (empty? ks) - (= ["refcount:expiry"] ks))) + (= ["refcount:expiry"] ks) + (= #{"refcount:expiry" nil} (set ks)))) "Start with no keys") (let [[b-tree root set] (reduce (fn [[t root set] [op x]] @@ -50,10 +52,13 @@ (wcar {} (redis/drop-ref root)) (assert (let [ks (wcar {} (car/keys "*"))] (or (empty? ks) - (= ["refcount:expiry"] ks))) "End with no keys") + (= ["refcount:expiry"] ks) + (= #{"refcount:expiry" nil} (into #{} ks)))) + "End with no keys") (assert res (str "These are unequal: " (pr-str b-tree-order) " " (pr-str (seq (sort set))))) res)))) + (defspec test-many-keys-bigger-trees 100 (mixed-op-seq 800 200 10 1000 1000)) From 7e9794126d92d9b15db509984fe86da64588f746 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 10 May 2017 20:34:46 +0200 Subject: [PATCH 17/34] Fiddle with multi-lang travis work-around. --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 769138f..349f6e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,13 @@ language: clojure +dist: trusty services: - redis-server +before_install: + - sudo apt-get install nodejs env: matrix: - TEST_CMD='lein test' - TEST_CMD='lein bench output --data-structure fractal -- --data-structure b-tree -- --data-structure sorted-set' - TEST_CMD='lein bench output --delete-pattern forward -- --delete-pattern reverse -- --delete-pattern shuffle -- --delete-pattern zero' + - TEST_CMD='node test_node.js' script: $TEST_CMD From d31529102b10057e47ff8ed5d3c405718219c34c Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 10 May 2017 20:39:09 +0200 Subject: [PATCH 18/34] Actually build cljs for the test. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 349f6e5..1bc9c60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,5 +9,5 @@ env: - TEST_CMD='lein test' - TEST_CMD='lein bench output --data-structure fractal -- --data-structure b-tree -- --data-structure sorted-set' - TEST_CMD='lein bench output --delete-pattern forward -- --delete-pattern reverse -- --delete-pattern shuffle -- --delete-pattern zero' - - TEST_CMD='node test_node.js' + - TEST_CMD='lein cljsbuild once && node test_node.js' script: $TEST_CMD From bd19ca1064c72aa882f99d5a05bce3f9f60040cc Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 10 May 2017 20:48:17 +0200 Subject: [PATCH 19/34] Keep dancing with travis. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1bc9c60..6583930 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,12 @@ services: - redis-server before_install: - sudo apt-get install nodejs +install: + - lein cljsbuild once env: matrix: - TEST_CMD='lein test' - TEST_CMD='lein bench output --data-structure fractal -- --data-structure b-tree -- --data-structure sorted-set' - TEST_CMD='lein bench output --delete-pattern forward -- --delete-pattern reverse -- --delete-pattern shuffle -- --delete-pattern zero' - - TEST_CMD='lein cljsbuild once && node test_node.js' + - TEST_CMD='node test_node.js' script: $TEST_CMD From 08ae90e74465c909f2d16c55425172172fa0825a Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Wed, 10 May 2017 21:39:15 +0200 Subject: [PATCH 20/34] Integrate node tests into travis. Small cleanups. --- .travis.yml | 2 +- src/hitchhiker/tree/core.cljc | 35 ++++++++++++++---------------- test/hitchhiker/konserve_test.cljc | 17 ++++++--------- test/hitchhiker/redis_test.clj | 2 +- test_node.js | 7 +++++- 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6583930..4b3cefb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,5 +11,5 @@ env: - TEST_CMD='lein test' - TEST_CMD='lein bench output --data-structure fractal -- --data-structure b-tree -- --data-structure sorted-set' - TEST_CMD='lein bench output --delete-pattern forward -- --delete-pattern reverse -- --delete-pattern shuffle -- --delete-pattern zero' - - TEST_CMD='node test_node.js' + - TEST_CMD='node ./test_node.js' script: $TEST_CMD diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index 646c9a8..70a411d 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -88,6 +88,7 @@ throwable error." (recur (> (flush-children (:children tree) backend stats) - > (flush-children (:children tree) backend stats) + js (js m)) + (if (cljs.test/successful? m) + (println "Success!") + (println "FAIL"))) + (run-tests))) diff --git a/test/hitchhiker/redis_test.clj b/test/hitchhiker/redis_test.clj index d2f86cf..1d56c0f 100644 --- a/test/hitchhiker/redis_test.clj +++ b/test/hitchhiker/redis_test.clj @@ -31,7 +31,7 @@ (assert (let [ks (wcar {} (car/keys "*"))] (or (empty? ks) (= ["refcount:expiry"] ks) - (= #{"refcount:expiry" nil} (set ks)))) + (= #{"refcount:expiry" nil} (into #{} ks)))) "Start with no keys") (let [[b-tree root set] (reduce (fn [[t root set] [op x]] diff --git a/test_node.js b/test_node.js index 7b9b898..5031219 100644 --- a/test_node.js +++ b/test_node.js @@ -22,5 +22,10 @@ nodeGlobalRequire('./target/none/cljs_deps.js'); goog.require('hitchhiker.konserve_test'); -hitchhiker.konserve_test.test_all(); +hitchhiker.konserve_test.test_all( + function(res) { + if(res.fail + res.error > 0) + process.exit(1); + else + process.exit(0);}); From 3b9c2dfaf119ef2081e7c534ea1888d2113161fa Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Thu, 11 May 2017 02:29:15 +0200 Subject: [PATCH 21/34] Try a performance hack. --- src/hitchhiker/tree/core.cljc | 55 ++++++++++++++++-------------- src/hitchhiker/tree/messaging.cljc | 14 +++----- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index 70a411d..c619943 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -52,11 +52,13 @@ {:style/indent 1} [ & body] `(if-cljs (cljs.core.async.macros/go - (try ~@body + ~@body + #_(try ~@body (catch js/Error e# e#))) (go - (try + ~@body + #_(try ~@body (catch Exception e# e#)))))) @@ -66,8 +68,8 @@ "Same as core.async common-parent-path peek inc) parent (-> common-parent-path pop peek) - new-sibling ( s :children first) c (if (tree-node? c) - ( % :children first)] - (if (tree-node? c) - ( (interleave sibling-lineage (repeat 0)) (butlast)) ; butlast ensures we end w/ node @@ -483,10 +488,10 @@ throwable error." (let [index (lookup cur key) child (if (data-node? cur) nil #_(nth-of-set (:children cur) index) - ( (:children cur) - ;;TODO what are the semantics for exceeding on the right? currently it's trunc to the last element - (nth index (peek (:children cur))) - (resolve)))) + (-> (:children cur) + ;;TODO what are the semantics for exceeding on the right? currently it's trunc to the last element + (nth index (peek (:children cur))) + ( - ( ( ( bigger-sibling-idx index) ; if true, `node` is left merged (if node-first? - (merge-node node ( Date: Thu, 11 May 2017 02:35:54 +0200 Subject: [PATCH 22/34] Missed cljs macro. --- src/hitchhiker/tree/core.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index c619943..a36ce9b 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -11,7 +11,7 @@ #?(:clj (:import java.io.Writer [java.util Arrays Collections])) #?(:cljs (:require-macros [cljs.core.async.macros :refer [go]] - [hitchhiker.tree.core :refer [go-try Date: Thu, 11 May 2017 16:04:50 +0200 Subject: [PATCH 23/34] Fix string key support in messaging, one test still failing (?). --- src/hitchhiker/tree/messaging.cljc | 12 ++++++- test/hitchhiker/tree/core_test.clj | 43 ++++++++++++++++--------- test/hitchhiker/tree/messaging_test.clj | 9 ++++++ 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/hitchhiker/tree/messaging.cljc b/src/hitchhiker/tree/messaging.cljc index c147c4d..58e97d4 100644 --- a/src/hitchhiker/tree/messaging.cljc +++ b/src/hitchhiker/tree/messaging.cljc @@ -148,6 +148,16 @@ ;;broadcast nodes will need IDs so that they can combine during merges... ;; +(defn general-max [e & r] + ;; fast track for number keys + (if (number? e) + (apply max e r) + (reduce (fn [old elem] + (if (pos? (core/compare old elem)) + old + elem)) + e r))) + (defn apply-ops-in-path [path] @@ -184,7 +194,7 @@ left-sibs-min-last (when (seq left-sibs-on-path) (->> left-sibs-on-path (map core/last-key) - (apply max))) + (apply general-max))) left-sib-filter (if left-sibs-min-last (drop-while #(>= 0 (core/compare (affects-key %) left-sibs-min-last))) diff --git a/test/hitchhiker/tree/core_test.clj b/test/hitchhiker/tree/core_test.clj index f0127ae..fe32aa4 100644 --- a/test/hitchhiker/tree/core_test.clj +++ b/test/hitchhiker/tree/core_test.clj @@ -6,7 +6,7 @@ [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop] [hitchhiker.tree.core :refer :all] - [clojure.core.async :as async])) + [clojure.core.async :refer [promise-chan] :as async])) (deftest reduce<-test (is (= 45 (Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) - data2 (data-node (->Config 3 3 2) (sorted-map 6 6 7 7 8 8 9 9 10 10)) - root (->IndexNode [data1 data2] (promise) [] (->Config 3 3 2))] + (let [data1 (data-node (->Config 3 5 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) + data2 (data-node (->Config 3 5 2) (sorted-map 6 6 7 7 8 8 9 9 10 10)) + root (->IndexNode [data1 data2] (promise-chan) [] (->Config 3 5 2))] (is (= (Config 3 5 2) (sorted-map "1" 1 "2" 2 "3" 3 "4" 4 "5" 5)) + data2 (data-node (->Config 3 5 2) (sorted-map "6" 6 "7" 7 "8" 8 "9" 9 "10" 10)) + root (->IndexNode [data1 data2] (promise-chan) [] (->Config 3 5 2))] + (is (= (Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) - data2 (data-node (->Config 3 3 2) (sorted-map 6 6 7 7 8 8 9 9 10 10)) - root (->IndexNode [data1 data2] (promise) [] (->Config 3 3 2))] + (let [data1 (data-node (->Config 3 5 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) + data2 (data-node (->Config 3 5 2) (sorted-map 6 6 7 7 8 8 9 9 10 10)) + root (->IndexNode [data1 data2] (promise-chan) [] (->Config 3 5 2))] (is (= (map first (lookup-fwd-iter root 4)) (range 4 11))) (is (= (map first (lookup-fwd-iter root 0)) (range 1 11))))) (testing "index nodes identified as such" - (let [data (data-node (->Config 3 3 2) (sorted-map 1 1)) - root (->IndexNode [data] (promise) [] (->Config 3 3 2))] + (let [data (data-node (->Config 3 5 2) (sorted-map 1 1)) + root (->IndexNode [data] (promise-chan) [] (->Config 3 5 2))] (is (index? root)) (is (not (index? data)))))) + (defn insert-helper [t k] (Config 3 3 2))) v) + b-tree-order (lookup-fwd-iter b-tree "")] + (= (seq sorted-set-order) (seq (map first b-tree-order)))))) + (defspec test-delete2 1000 (prop/for-all [the-set (gen/vector-distinct gen/int) @@ -74,14 +91,14 @@ (deftest insert-test (let [data1 (data-node (->Config 3 3 2) (sorted-map 1 "1" 2 "2" 3 "3" 4 "4")) - root (->IndexNode [data1] (promise) [] (->Config 3 3 2))] + root (->IndexNode [data1] (promise-chan) [] (->Config 3 3 2))] (is (= (map second (lookup-fwd-iter (Config 3 3 2) (sorted-map 1 1 2 2 3 3 4 4 5 5)) - root (->IndexNode [data1] (promise) [] (->Config 3 3 2))] + root (->IndexNode [data1] (promise-chan) [] (->Config 3 3 2))] (are [x y] (= (map first (lookup-fwd-iter (TestingBackend)))) flushed-tree-order (map first (lookup-fwd-iter flushed-tree Integer/MIN_VALUE))] - (when-not (= (seq sorted-set-order) - (seq b-tree-order) - (seq flushed-tree-order)) - (prn "FT" sorted-set-order b-tree-order flushed-tree-order)) (= (seq sorted-set-order) (seq b-tree-order) (seq flushed-tree-order))))) diff --git a/test/hitchhiker/tree/messaging_test.clj b/test/hitchhiker/tree/messaging_test.clj index 367a56c..1c1ac64 100644 --- a/test/hitchhiker/tree/messaging_test.clj +++ b/test/hitchhiker/tree/messaging_test.clj @@ -30,6 +30,14 @@ b-tree-order (lookup-fwd-iter b-tree Integer/MIN_VALUE)] (= (seq sorted-set-order) b-tree-order)))) +(defspec test-insert-string + 1000 + (prop/for-all [v (gen/vector gen/string)] + (let [sorted-set-order (into (sorted-set) v) + b-tree (reduce insert (Config 3 3 2))) v) + b-tree-order (lookup-fwd-iter b-tree "")] + (= (seq sorted-set-order) b-tree-order)))) + (defspec test-delete2 1000 (prop/for-all [the-set (gen/vector-distinct gen/int) @@ -159,3 +167,4 @@ (defspec test-sparse-ops 100 (mixed-op-seq 0.7 100000 1000)) + From 697ec79d34cfbd95095107d604e625dcad279523 Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Thu, 11 May 2017 23:15:24 +0200 Subject: [PATCH 24/34] Fix string key test indices. --- test/hitchhiker/tree/core_test.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hitchhiker/tree/core_test.clj b/test/hitchhiker/tree/core_test.clj index fe32aa4..5084589 100644 --- a/test/hitchhiker/tree/core_test.clj +++ b/test/hitchhiker/tree/core_test.clj @@ -24,8 +24,8 @@ (dotimes [i 10] (is (= (Config 3 5 2) (sorted-map "1" 1 "2" 2 "3" 3 "4" 4 "5" 5)) - data2 (data-node (->Config 3 5 2) (sorted-map "6" 6 "7" 7 "8" 8 "9" 9 "10" 10)) + (let [data1 (data-node (->Config 3 5 2) (sorted-map "1" 1 "10" 10 "2" 2 "3" 3 "4" 4)) + data2 (data-node (->Config 3 5 2) (sorted-map "5" 5 "6" 6 "7" 7 "8" 8 "9" 9)) root (->IndexNode [data1 data2] (promise-chan) [] (->Config 3 5 2))] (is (= ( Date: Wed, 17 May 2017 23:43:33 +0200 Subject: [PATCH 25/34] Unify codebase, reset gen-test volume. --- src/hitchhiker/tree/core.cljc | 10 ++++------ test/hitchhiker/tree/messaging_test.clj | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index a36ce9b..da9b048 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -52,13 +52,11 @@ {:style/indent 1} [ & body] `(if-cljs (cljs.core.async.macros/go - ~@body - #_(try ~@body + (try ~@body (catch js/Error e# e#))) (go - ~@body - #_(try + (try ~@body (catch Exception e# e#)))))) @@ -68,8 +66,8 @@ "Same as core.async Date: Tue, 26 Dec 2017 03:31:58 +0100 Subject: [PATCH 26/34] Add default ordering between different types. --- hs_err_pid11680.log | 388 ++++++++++++++++++++++++++++++++++ src/hitchhiker/konserve.cljc | 5 +- src/hitchhiker/tree/core.cljc | 35 ++- 3 files changed, 423 insertions(+), 5 deletions(-) create mode 100644 hs_err_pid11680.log diff --git a/hs_err_pid11680.log b/hs_err_pid11680.log new file mode 100644 index 0000000..69262f4 --- /dev/null +++ b/hs_err_pid11680.log @@ -0,0 +1,388 @@ +# +# There is insufficient memory for the Java Runtime Environment to continue. +# Native memory allocation (mmap) failed to map 2586836992 bytes for committing reserved memory. +# Possible reasons: +# The system is out of physical RAM or swap space +# In 32 bit mode, the process size limit was hit +# Possible solutions: +# Reduce memory load on the system +# Increase physical memory or swap space +# Check if swap backing store is full +# Use 64 bit Java on a 64 bit OS +# Decrease Java heap size (-Xmx/-Xms) +# Decrease number of Java threads +# Decrease Java thread stack sizes (-Xss) +# Set larger code cache with -XX:ReservedCodeCacheSize= +# This output file may be truncated or incomplete. +# +# Out of Memory Error (os_linux.cpp:2643), pid=11680, tid=0x00007f17fe6b0700 +# +# JRE version: (8.0_151-b12) (build ) +# Java VM: OpenJDK 64-Bit Server VM (25.151-b12 mixed mode linux-amd64 compressed oops) +# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again +# + +--------------- T H R E A D --------------- + +Current thread (0x00007f17f800d800): JavaThread "Unknown thread" [_thread_in_vm, id=11685, stack(0x00007f17fe5b0000,0x00007f17fe6b1000)] + +Stack: [0x00007f17fe5b0000,0x00007f17fe6b1000], sp=0x00007f17fe6af5a0, free space=1021k +Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) +V [libjvm.so+0xaa61a2] +V [libjvm.so+0x4d6d87] +V [libjvm.so+0x8e2760] +V [libjvm.so+0x8dcdce] +V [libjvm.so+0x95cc36] +V [libjvm.so+0x94cf6c] +V [libjvm.so+0x2b29be] +V [libjvm.so+0x909ed1] +V [libjvm.so+0xa6892a] +V [libjvm.so+0xa68c15] +V [libjvm.so+0x6221ef] +V [libjvm.so+0xa4c27b] +V [libjvm.so+0x6a4901] JNI_CreateJavaVM+0x61 +C [libjli.so+0x2ce3] +C [libjli.so+0x76ed] +C [libpthread.so.0+0x77fc] start_thread+0xdc + + +--------------- P R O C E S S --------------- + +Java Threads: ( => current thread ) + +Other Threads: + +=>0x00007f17f800d800 (exited) JavaThread "Unknown thread" [_thread_in_vm, id=11685, stack(0x00007f17fe5b0000,0x00007f17fe6b1000)] + +VM state:not at safepoint (not fully initialized) + +VM Mutex/Monitor currently owned by a thread: None + +GC Heap History (0 events): +No events + +Deoptimization events (0 events): +No events + +Internal exceptions (0 events): +No events + +Events (0 events): +No events + + +Dynamic libraries: +772f00000-7c0000000 rw-p 00000000 00:00 0 +55e79e2e1000-55e79e2e2000 r-xp 00000000 08:01 13506105 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java +55e79e4e1000-55e79e4e2000 r--p 00000000 08:01 13506105 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java +55e79e4e2000-55e79e4e3000 rw-p 00001000 08:01 13506105 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java +55e79e532000-55e79e553000 rw-p 00000000 00:00 0 [heap] +7f17e7471000-7f17e7942000 ---p 00000000 00:00 0 +7f17e7942000-7f17e7bb6000 rw-p 00000000 00:00 0 +7f17e7bb6000-7f17e7f6c000 ---p 00000000 00:00 0 +7f17e7f6c000-7f17e81dc000 rwxp 00000000 00:00 0 +7f17e81dc000-7f17f6f6c000 ---p 00000000 00:00 0 +7f17f6f6c000-7f17f6f74000 r-xp 00000000 08:01 13506156 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so +7f17f6f74000-7f17f7173000 ---p 00008000 08:01 13506156 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so +7f17f7173000-7f17f7174000 r--p 00007000 08:01 13506156 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so +7f17f7174000-7f17f7175000 rw-p 00008000 08:01 13506156 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so +7f17f7175000-7f17f7180000 r-xp 00000000 08:01 7864765 /lib/x86_64-linux-gnu/libnss_files-2.26.so +7f17f7180000-7f17f737f000 ---p 0000b000 08:01 7864765 /lib/x86_64-linux-gnu/libnss_files-2.26.so +7f17f737f000-7f17f7380000 r--p 0000a000 08:01 7864765 /lib/x86_64-linux-gnu/libnss_files-2.26.so +7f17f7380000-7f17f7381000 rw-p 0000b000 08:01 7864765 /lib/x86_64-linux-gnu/libnss_files-2.26.so +7f17f7381000-7f17f7387000 rw-p 00000000 00:00 0 +7f17f7387000-7f17f7392000 r-xp 00000000 08:01 7864767 /lib/x86_64-linux-gnu/libnss_nis-2.26.so +7f17f7392000-7f17f7591000 ---p 0000b000 08:01 7864767 /lib/x86_64-linux-gnu/libnss_nis-2.26.so +7f17f7591000-7f17f7592000 r--p 0000a000 08:01 7864767 /lib/x86_64-linux-gnu/libnss_nis-2.26.so +7f17f7592000-7f17f7593000 rw-p 0000b000 08:01 7864767 /lib/x86_64-linux-gnu/libnss_nis-2.26.so +7f17f7593000-7f17f75aa000 r-xp 00000000 08:01 7864762 /lib/x86_64-linux-gnu/libnsl-2.26.so +7f17f75aa000-7f17f77a9000 ---p 00017000 08:01 7864762 /lib/x86_64-linux-gnu/libnsl-2.26.so +7f17f77a9000-7f17f77aa000 r--p 00016000 08:01 7864762 /lib/x86_64-linux-gnu/libnsl-2.26.so +7f17f77aa000-7f17f77ab000 rw-p 00017000 08:01 7864762 /lib/x86_64-linux-gnu/libnsl-2.26.so +7f17f77ab000-7f17f77ad000 rw-p 00000000 00:00 0 +7f17f77ad000-7f17f77b5000 r-xp 00000000 08:01 7864763 /lib/x86_64-linux-gnu/libnss_compat-2.26.so +7f17f77b5000-7f17f79b4000 ---p 00008000 08:01 7864763 /lib/x86_64-linux-gnu/libnss_compat-2.26.so +7f17f79b4000-7f17f79b5000 r--p 00007000 08:01 7864763 /lib/x86_64-linux-gnu/libnss_compat-2.26.so +7f17f79b5000-7f17f79b6000 rw-p 00008000 08:01 7864763 /lib/x86_64-linux-gnu/libnss_compat-2.26.so +7f17f79b6000-7f17f79e3000 r-xp 00000000 08:01 13506131 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so +7f17f79e3000-7f17f7be3000 ---p 0002d000 08:01 13506131 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so +7f17f7be3000-7f17f7be4000 r--p 0002d000 08:01 13506131 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so +7f17f7be4000-7f17f7be6000 rw-p 0002e000 08:01 13506131 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so +7f17f7be6000-7f17f7bf6000 r-xp 00000000 08:01 13506152 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so +7f17f7bf6000-7f17f7df5000 ---p 00010000 08:01 13506152 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so +7f17f7df5000-7f17f7df7000 r--p 0000f000 08:01 13506152 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so +7f17f7df7000-7f17f7df8000 rw-p 00011000 08:01 13506152 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so +7f17f7df8000-7f17f7dff000 r-xp 00000000 08:01 7864772 /lib/x86_64-linux-gnu/librt-2.26.so +7f17f7dff000-7f17f7ffe000 ---p 00007000 08:01 7864772 /lib/x86_64-linux-gnu/librt-2.26.so +7f17f7ffe000-7f17f7fff000 r--p 00006000 08:01 7864772 /lib/x86_64-linux-gnu/librt-2.26.so +7f17f7fff000-7f17f8000000 rw-p 00007000 08:01 7864772 /lib/x86_64-linux-gnu/librt-2.26.so +7f17f8000000-7f17f8035000 rw-p 00000000 00:00 0 +7f17f8035000-7f17fc000000 ---p 00000000 00:00 0 +7f17fc024000-7f17fc03a000 r-xp 00000000 08:01 7864750 /lib/x86_64-linux-gnu/libgcc_s.so.1 +7f17fc03a000-7f17fc239000 ---p 00016000 08:01 7864750 /lib/x86_64-linux-gnu/libgcc_s.so.1 +7f17fc239000-7f17fc23a000 r--p 00015000 08:01 7864750 /lib/x86_64-linux-gnu/libgcc_s.so.1 +7f17fc23a000-7f17fc23b000 rw-p 00016000 08:01 7864750 /lib/x86_64-linux-gnu/libgcc_s.so.1 +7f17fc23b000-7f17fc390000 r-xp 00000000 08:01 7864759 /lib/x86_64-linux-gnu/libm-2.26.so +7f17fc390000-7f17fc58f000 ---p 00155000 08:01 7864759 /lib/x86_64-linux-gnu/libm-2.26.so +7f17fc58f000-7f17fc590000 r--p 00154000 08:01 7864759 /lib/x86_64-linux-gnu/libm-2.26.so +7f17fc590000-7f17fc591000 rw-p 00155000 08:01 7864759 /lib/x86_64-linux-gnu/libm-2.26.so +7f17fc591000-7f17fc709000 r-xp 00000000 08:01 2628521 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.24 +7f17fc709000-7f17fc908000 ---p 00178000 08:01 2628521 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.24 +7f17fc908000-7f17fc912000 r--p 00177000 08:01 2628521 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.24 +7f17fc912000-7f17fc914000 rw-p 00181000 08:01 2628521 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.24 +7f17fc914000-7f17fc917000 rw-p 00000000 00:00 0 +7f17fc917000-7f17fd5a1000 r-xp 00000000 08:01 13508278 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so +7f17fd5a1000-7f17fd7a1000 ---p 00c8a000 08:01 13508278 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so +7f17fd7a1000-7f17fd834000 r--p 00c8a000 08:01 13508278 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so +7f17fd834000-7f17fd85c000 rw-p 00d1d000 08:01 13508278 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so +7f17fd85c000-7f17fd88d000 rw-p 00000000 00:00 0 +7f17fd88d000-7f17fd8a7000 r-xp 00000000 08:01 7864770 /lib/x86_64-linux-gnu/libpthread-2.26.so +7f17fd8a7000-7f17fdaa6000 ---p 0001a000 08:01 7864770 /lib/x86_64-linux-gnu/libpthread-2.26.so +7f17fdaa6000-7f17fdaa7000 r--p 00019000 08:01 7864770 /lib/x86_64-linux-gnu/libpthread-2.26.so +7f17fdaa7000-7f17fdaa8000 rw-p 0001a000 08:01 7864770 /lib/x86_64-linux-gnu/libpthread-2.26.so +7f17fdaa8000-7f17fdaac000 rw-p 00000000 00:00 0 +7f17fdaac000-7f17fdaaf000 r-xp 00000000 08:01 7864758 /lib/x86_64-linux-gnu/libdl-2.26.so +7f17fdaaf000-7f17fdcae000 ---p 00003000 08:01 7864758 /lib/x86_64-linux-gnu/libdl-2.26.so +7f17fdcae000-7f17fdcaf000 r--p 00002000 08:01 7864758 /lib/x86_64-linux-gnu/libdl-2.26.so +7f17fdcaf000-7f17fdcb0000 rw-p 00003000 08:01 7864758 /lib/x86_64-linux-gnu/libdl-2.26.so +7f17fdcb0000-7f17fdccc000 r-xp 00000000 08:01 7869027 /lib/x86_64-linux-gnu/libz.so.1.2.11 +7f17fdccc000-7f17fdecb000 ---p 0001c000 08:01 7869027 /lib/x86_64-linux-gnu/libz.so.1.2.11 +7f17fdecb000-7f17fdecc000 r--p 0001b000 08:01 7869027 /lib/x86_64-linux-gnu/libz.so.1.2.11 +7f17fdecc000-7f17fdecd000 rw-p 0001c000 08:01 7869027 /lib/x86_64-linux-gnu/libz.so.1.2.11 +7f17fdecd000-7f17fe0a3000 r-xp 00000000 08:01 7864755 /lib/x86_64-linux-gnu/libc-2.26.so +7f17fe0a3000-7f17fe2a3000 ---p 001d6000 08:01 7864755 /lib/x86_64-linux-gnu/libc-2.26.so +7f17fe2a3000-7f17fe2a7000 r--p 001d6000 08:01 7864755 /lib/x86_64-linux-gnu/libc-2.26.so +7f17fe2a7000-7f17fe2a9000 rw-p 001da000 08:01 7864755 /lib/x86_64-linux-gnu/libc-2.26.so +7f17fe2a9000-7f17fe2ad000 rw-p 00000000 00:00 0 +7f17fe2ad000-7f17fe2bb000 r-xp 00000000 08:01 13506116 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so +7f17fe2bb000-7f17fe4ba000 ---p 0000e000 08:01 13506116 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so +7f17fe4ba000-7f17fe4bb000 r--p 0000d000 08:01 13506116 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so +7f17fe4bb000-7f17fe4bc000 rw-p 0000e000 08:01 13506116 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so +7f17fe4bc000-7f17fe4e3000 r-xp 00000000 08:01 7864751 /lib/x86_64-linux-gnu/ld-2.26.so +7f17fe5b0000-7f17fe5b3000 ---p 00000000 00:00 0 +7f17fe5b3000-7f17fe6b6000 rw-p 00000000 00:00 0 +7f17fe6d1000-7f17fe6d9000 rw-s 00000000 08:01 5256118 /tmp/hsperfdata_christian/11680 +7f17fe6d9000-7f17fe6da000 rw-p 00000000 00:00 0 +7f17fe6da000-7f17fe6db000 r--p 00000000 00:00 0 +7f17fe6db000-7f17fe6de000 rw-p 00000000 00:00 0 +7f17fe6de000-7f17fe6e1000 r--p 00000000 00:00 0 [vvar] +7f17fe6e1000-7f17fe6e3000 r-xp 00000000 00:00 0 [vdso] +7f17fe6e3000-7f17fe6e4000 r--p 00027000 08:01 7864751 /lib/x86_64-linux-gnu/ld-2.26.so +7f17fe6e4000-7f17fe6e5000 rw-p 00028000 08:01 7864751 /lib/x86_64-linux-gnu/ld-2.26.so +7f17fe6e5000-7f17fe6e6000 rw-p 00000000 00:00 0 +7fff687d9000-7fff687fc000 rw-p 00000000 00:00 0 [stack] +ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] + +VM Arguments: +jvm_args: -Dfile.encoding=UTF-8 -Xmx3700m -Xms3700m -Dclojure.compile.path=/home/christian/Development/hitchhiker-tree/target/classes -Dhitchhiker-tree.version=0.1.0-SNAPSHOT -Dclojure.debug=false +java_command: clojure.main -i /tmp/form-init7961262958693552825.clj +java_class_path (initial): /home/christian/Development/hitchhiker-tree/test:/home/christian/Development/hitchhiker-tree/src:/home/christian/Development/hitchhiker-tree/dev:/home/christian/Development/hitchhiker-tree/src:/home/christian/Development/hitchhiker-tree/dev-resources:/home/christian/Development/hitchhiker-tree/resources:/home/christian/Development/hitchhiker-tree/target/classes:/home/christian/.m2/repository/io/replikativ/hasch/0.3.4/hasch-0.3.4.jar:/home/christian/.m2/repository/org/iq80/snappy/snappy/0.4/snappy-0.4.jar:/home/christian/.m2/repository/org/clojure/clojure/1.8.0/clojure-1.8.0.jar:/home/christian/.m2/repository/net/incongru/watchservice/barbary-watchservice/1.0/barbary-watchservice-1.0.jar:/home/christian/.m2/repository/net/cgrand/parsley/0.9.3/parsley-0.9.3.jar:/home/christian/.m2/repository/net/cgrand/regex/1.1.0/regex-1.1.0.jar:/home/christian/.m2/repository/org/clojure/google-closure-library/0.0-20151016-61277aea/google-closure-library-0.0-20151016-61277aea.jar:/home/christian/.m2/repository/com/cemerick/austin/0.1.6/austin-0.1.6.jar:/home/christian/.m2/repository/clj-time/clj-time/0.13.0/clj-time-0.13.0.jar:/home/christian/.m2/repository/com/taoensso/truss/1.3.7/truss-1.3.7.jar:/home/christian/.m2/repository/io/replikativ/konserve/0.4.9/konserve-0.4.9.jar:/home/christian/.m2/repository/crypto-equality/crypto-equality/1.0.0/crypto-equality-1.0.0.jar:/home/christian/.m2/repository/org/clojure/tools.analyzer.jvm/0.6.10/tools.analyzer.jvm-0.6.10.jar:/home/christian/.m2/repository/com/google/javascript/closure-compiler-externs/v20160315/closure-compiler-externs-v20160315.jar:/home/christian/.m2/repository/org/clojure/clojurescript/1.8.51/clojurescript-1.8.51.jar:/home/christian/.m2/repository/net/jpountz/lz4/lz4/1.3/lz4-1.3.jar:/home/christian/.m2/repository/net/java/dev/jna/jna/3.2.2/jna-3.2.2.jar:/home/christian/.m2/repository/org/clojure/tools.analyzer/0.6.9/tools.analyzer-0.6.9.jar:/home/christian/.m2/repository/org/tukaani/xz/1.6/xz-1. +Launcher Type: SUN_STANDARD + +Environment Variables: +CLASSPATH=/home/christian/.lein/self-installs/leiningen-2.7.1-standalone.jar +PATH=/home/christian/.cargo/bin:/home/christian/miniconda3/bin:/home/christian/torch/install/bin:/home/christian/.opam/4.03.0/bin:/home/christian/.fzf/bin:/home/christian/.cargo/bin:/home/christian/miniconda3/bin:/home/christian/torch/install/bin:/home/christian/.opam/4.03.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games +USERNAME=christian +LD_LIBRARY_PATH=/home/christian/torch/install/lib/home/christian/torch/install/lib +SHELL=/usr/bin/fish +DISPLAY=:0 +DYLD_LIBRARY_PATH=/home/christian/torch/install/lib/home/christian/torch/install/lib + +Signal Handlers: +SIGSEGV: [libjvm.so+0xaa6ae0], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGBUS: [libjvm.so+0xaa6ae0], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGFPE: [libjvm.so+0x8ddf00], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGPIPE: [libjvm.so+0x8ddf00], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGXFSZ: [libjvm.so+0x8ddf00], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGILL: [libjvm.so+0x8ddf00], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGUSR1: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGUSR2: [libjvm.so+0x8dddb0], sa_mask[0]=00100000000000000000000000000000, sa_flags=SA_RESTART|SA_SIGINFO +SIGHUP: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGINT: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGTERM: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGQUIT: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none + + +--------------- S Y S T E M --------------- + +OS:DISTRIB_ID=Ubuntu +DISTRIB_RELEASE=17.10 +DISTRIB_CODENAME=artful +DISTRIB_DESCRIPTION="Ubuntu 17.10" + +uname:Linux 4.13.0-19-generic #22-Ubuntu SMP Mon Dec 4 11:58:07 UTC 2017 x86_64 +libc:glibc 2.26 NPTL 2.26 +rlimit: STACK 8192k, CORE 0k, NPROC 28899, NOFILE 4096, AS infinity +load average:0.91 1.11 1.25 + +/proc/meminfo: +MemTotal: 7451208 kB +MemFree: 119372 kB +MemAvailable: 586760 kB +Buffers: 34624 kB +Cached: 577720 kB +SwapCached: 36268 kB +Active: 5629520 kB +Inactive: 1330768 kB +Active(anon): 5447192 kB +Inactive(anon): 869144 kB +Active(file): 182328 kB +Inactive(file): 461624 kB +Unevictable: 112 kB +Mlocked: 112 kB +SwapTotal: 2097148 kB +SwapFree: 907448 kB +Dirty: 1012 kB +Writeback: 0 kB +AnonPages: 6315900 kB +Mapped: 440724 kB +Shmem: 325956 kB +Slab: 210284 kB +SReclaimable: 85932 kB +SUnreclaim: 124352 kB +KernelStack: 18864 kB +PageTables: 83560 kB +NFS_Unstable: 0 kB +Bounce: 0 kB +WritebackTmp: 0 kB +CommitLimit: 5822752 kB +Committed_AS: 18522868 kB +VmallocTotal: 34359738367 kB +VmallocUsed: 0 kB +VmallocChunk: 0 kB +HardwareCorrupted: 0 kB +AnonHugePages: 0 kB +ShmemHugePages: 0 kB +ShmemPmdMapped: 0 kB +CmaTotal: 0 kB +CmaFree: 0 kB +HugePages_Total: 0 +HugePages_Free: 0 +HugePages_Rsvd: 0 +HugePages_Surp: 0 +Hugepagesize: 2048 kB +DirectMap4k: 362660 kB +DirectMap2M: 7297024 kB +DirectMap1G: 0 kB + + +CPU:total 4 (initial active 4) (2 cores per cpu, 2 threads per core) family 6 model 78 stepping 3, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, avx, avx2, aes, clmul, erms, 3dnowpref, lzcnt, ht, tsc, tscinvbit, bmi1, bmi2, adx + +/proc/cpuinfo: +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 78 +model name : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz +stepping : 3 +microcode : 0xba +cpu MHz : 2600.000 +cache size : 4096 KB +physical id : 0 +siblings : 4 +core id : 0 +cpu cores : 2 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 22 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp +bugs : +bogomips : 5184.00 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 1 +vendor_id : GenuineIntel +cpu family : 6 +model : 78 +model name : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz +stepping : 3 +microcode : 0xba +cpu MHz : 2600.000 +cache size : 4096 KB +physical id : 0 +siblings : 4 +core id : 1 +cpu cores : 2 +apicid : 2 +initial apicid : 2 +fpu : yes +fpu_exception : yes +cpuid level : 22 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp +bugs : +bogomips : 5184.00 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 2 +vendor_id : GenuineIntel +cpu family : 6 +model : 78 +model name : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz +stepping : 3 +microcode : 0xba +cpu MHz : 2600.000 +cache size : 4096 KB +physical id : 0 +siblings : 4 +core id : 0 +cpu cores : 2 +apicid : 1 +initial apicid : 1 +fpu : yes +fpu_exception : yes +cpuid level : 22 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp +bugs : +bogomips : 5184.00 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 3 +vendor_id : GenuineIntel +cpu family : 6 +model : 78 +model name : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz +stepping : 3 +microcode : 0xba +cpu MHz : 2600.000 +cache size : 4096 KB +physical id : 0 +siblings : 4 +core id : 1 +cpu cores : 2 +apicid : 3 +initial apicid : 3 +fpu : yes +fpu_exception : yes +cpuid level : 22 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp +bugs : +bogomips : 5184.00 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + + + +Memory: 4k page, physical 7451208k(119372k free), swap 2097148k(907448k free) + +vm_info: OpenJDK 64-Bit Server VM (25.151-b12) for linux-amd64 JRE (1.8.0_151-8u151-b12-0ubuntu0.17.10.2-b12), built on Oct 23 2017 22:43:02 by "buildd" with gcc 7.2.0 + +time: Mon Dec 18 10:52:45 2017 +elapsed time: 0 seconds (0d 0h 0m 0s) + diff --git a/src/hitchhiker/konserve.cljc b/src/hitchhiker/konserve.cljc index eeed82e..107e62b 100644 --- a/src/hitchhiker/konserve.cljc +++ b/src/hitchhiker/konserve.cljc @@ -51,7 +51,10 @@ (defn get-root-key [tree] - (-> tree :storage-addr (async/poll!))) + ;; TODO find out why this is inconsistent + (or + (-> tree :storage-addr (async/poll!) :konserve-key) + (-> tree :storage-addr (async/poll!)))) (defn create-tree-from-root-key [store root-key] diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index da9b048..aaaaf2b 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -162,21 +162,47 @@ throwable error." ;; (based on proven data), and then send generated loads to check we stay within ;; our targets. +;; TODO add full edn support including records +(defn order-on-edn-types [t] + (cond (map? t) 0 + (vector? t) 1 + (set? t) 2 + (number? t) 3 + (string? t) 4 + (symbol? t) 5 + (keyword? t) 6)) + (extend-protocol IKeyCompare ;; By default, we use the default comparator #?@(:clj [Object - (compare [key1 key2] (clojure.core/compare key1 key2)) + (compare [key1 key2] + (if (or (= (type key1) (type key2)) + (= (order-on-edn-types key1) + (order-on-edn-types key2))) + (try + (clojure.core/compare key1 key2) + (catch ClassCastException e + (- (order-on-edn-types key2) + (order-on-edn-types key1)))))) Double (compare [^Double key1 key2] (if (instance? Double key2) (.compareTo key1 key2) - (clojure.core/compare key1 key2))) + (try + (clojure.core/compare key1 key2) + (catch ClassCastException e + (- (order-on-edn-types key2) + (order-on-edn-types key1)))))) Long (compare [^Long key1 key2] (if (instance? Long key2) - (.compareTo key1 key2)) - (clojure.core/compare key1 key2))] + (.compareTo key1 key2) + (try + (clojure.core/compare key1 key2) + (catch ClassCastException e + (- (order-on-edn-types key2) + (order-on-edn-types key1))))))] :cljs [number (compare [key1 key2] (cljs.core/compare key1 key2)) @@ -514,6 +540,7 @@ throwable error." (cons v (lazy-seq (chan-seq ch)))))) + #?(:clj (defn lookup-fwd-iter "Compatibility helper to clojure sequences. Please prefer the channel From 46674aa73e17dae85a4244364ba25481a3b71ead Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Tue, 26 Dec 2017 11:20:33 +0100 Subject: [PATCH 27/34] Remove reflection warnings. --- src/hitchhiker/tree/core.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index e7049dc..585b6b6 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -37,7 +37,7 @@ to maintain a full stack trace when jumping between multiple contexts." [x] (if (instance? #?(:clj Exception :cljs js/Error) x) - (throw (ex-info (or #?(:clj (.getMessage x)) (str x)) + (throw (ex-info (or #?(:clj (.getMessage ^Exception x)) (str x)) (or (ex-data x) {}) x)) x)) @@ -321,7 +321,7 @@ throwable error." [node] (let [out ^Writer *out*] (.write out "IndexNode") - (.write out (node-status-bits node)) + (.write out ^String (node-status-bits node)) (pp/pprint-logical-block :prefix "{" :suffix "}" (pp/pprint-logical-block From ce75c977b80ad4343ffdc26754c22ce309f2dbeb Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Fri, 29 Dec 2017 01:39:35 +0100 Subject: [PATCH 28/34] Fix comparison, fast-path right-successor. --- src/hitchhiker/tree/core.cljc | 18 ++++++++++++++---- src/hitchhiker/tree/messaging.cljc | 3 ++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index 585b6b6..eb3a01f 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -11,7 +11,8 @@ #?(:clj (:import java.io.Writer [java.util Arrays Collections])) #?(:cljs (:require-macros [cljs.core.async.macros :refer [go]] - [hitchhiker.tree.core :refer [go-try Split (->IndexNode (subvec children 0 b) (promise-chan) (vec left-buf) @@ -467,9 +468,18 @@ throwable error." sibling-lineage (loop [res [new-sibling] s new-sibling] (let [c (-> s :children first) - c (if (tree-node? c) + ;_ (prn (type c) (= (class c) clojure.lang.PersistentTreeMap$BlackVal)) + c (cond + ;; fast path + (or (index-node? c) + (data-node? c) + (= (class c) clojure.lang.PersistentTreeMap$BlackVal)) + c + + (tree-node? c) (= 0 (core/compare @@ -168,7 +169,7 @@ (map :op-buf))) (rseq) ; highest node should be last in seq (apply catvec) - (sort-by affects-key)) ;must be a stable sort + (sort-by affects-key core/compare)) ;must be a stable sort this-node-index (-> path pop peek) parent (-> path pop pop peek) is-first? (zero? this-node-index) From eaa1dcd17a115ec7e4c5cc04d9d1d9416ee4cd5f Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sat, 30 Dec 2017 15:17:51 +0100 Subject: [PATCH 29/34] Support different async backends. --- src/hitchhiker/konserve.cljc | 75 +++++++------ src/hitchhiker/tree/core.cljc | 139 ++++++++++++++++-------- src/hitchhiker/tree/messaging.cljc | 73 +++++++++---- test/hitchhiker/konserve_test.cljc | 24 ++-- test/hitchhiker/tree/messaging_test.clj | 1 + 5 files changed, 204 insertions(+), 108 deletions(-) diff --git a/src/hitchhiker/konserve.cljc b/src/hitchhiker/konserve.cljc index 107e62b..60046c0 100644 --- a/src/hitchhiker/konserve.cljc +++ b/src/hitchhiker/konserve.cljc @@ -7,7 +7,7 @@ [konserve.memory :refer [new-mem-store]] [hasch.core :refer [uuid]] [clojure.set :as set] - #?(:clj [hitchhiker.tree.core :refer [go-try ( (case core/*async-backend* + :none (async/KonserveAddr store (core/last-key node) id (synthesize-storage-addr id)))))) (delete-addr [_ addr session] - (swap! session update-in :deletes inc))) + (swap! session update :deletes inc))) (defn get-root-key [tree] ;; TODO find out why this is inconsistent (or - (-> tree :storage-addr (async/poll!) :konserve-key) - (-> tree :storage-addr (async/poll!)))) + (-> tree :storage-addr (async/poll!) :konserve-key) + (-> tree :storage-addr (async/poll!)))) -(defn create-tree-from-root-key - [store root-key] - (go-try - (let [val (KonserveAddr store last-key root-key (synthesize-storage-addr root-key))))))) - - -(defn add-hitchhiker-tree-handlers [store] - (swap! (:read-handlers store) merge - {'hitchhiker.konserve.KonserveAddr - #(-> % map->KonserveAddr - (assoc :store store - :storage-addr (synthesize-storage-addr (:konserve-key %)))) - 'hitchhiker.tree.core.DataNode - (fn [{:keys [children cfg]}] - (core/->DataNode (into (sorted-map-by - compare) children) - (promise-chan) - cfg)) - 'hitchhiker.tree.core.IndexNode - (fn [{:keys [children cfg op-buf]}] - (core/->IndexNode (->> children - vec) + (defn create-tree-from-root-key + [store root-key] + (go-try + (let [val (let [ch (k/get-in store [root-key])] + (case core/*async-backend* + :none (async/KonserveAddr store last-key root-key (synthesize-storage-addr root-key))))))) + + + (defn add-hitchhiker-tree-handlers [store] + (swap! (:read-handlers store) merge + {'hitchhiker.konserve.KonserveAddr + #(-> % map->KonserveAddr + (assoc :store store + :storage-addr (synthesize-storage-addr (:konserve-key %)))) + 'hitchhiker.tree.core.DataNode + (fn [{:keys [children cfg]}] + (core/->DataNode (into (sorted-map-by + compare) children) (promise-chan) + cfg)) + 'hitchhiker.tree.core.IndexNode + (fn [{:keys [children cfg op-buf]}] + (core/->IndexNode (->> children + vec) + (promise-chan) (vec op-buf) cfg)) 'hitchhiker.tree.messaging.InsertOp diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index eb3a01f..c311ca7 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -2,6 +2,7 @@ (:refer-clojure :exclude [compare resolve subvec]) (:require [clojure.core.rrb-vector :refer [catvec subvec]] #?(:clj [clojure.pprint :as pp]) + #?(:clj [clojure.core.async :refer [promise-chan poll! put!]]) #?(:clj [clojure.core.async :refer [go chan put! children @@ -448,6 +467,7 @@ throwable error." (recur (pop tmp))))))) + (defn right-successor "Given a node on a path, find's that node's right successor node" [path] @@ -470,10 +490,12 @@ throwable error." (let [c (-> s :children first) ;_ (prn (type c) (= (class c) clojure.lang.PersistentTreeMap$BlackVal)) c (cond + ;; TODO cleanup path ;; fast path (or (index-node? c) (data-node? c) - (= (class c) clojure.lang.PersistentTreeMap$BlackVal)) + #?(:clj (= (class c) clojure.lang.PersistentTreeMap$Black)) + #?(:clj (= (class c) clojure.lang.PersistentTreeMap$BlackVal))) c (tree-node? c) @@ -493,21 +515,7 @@ throwable error." (into path-suffix)))))) -(defn forward-iterator - "Takes the result of a search and puts the iterated elements onto iter-ch - going forward over the tree as needed. Does lg(n) backtracking sometimes." - [iter-ch path start-key] - (go-try - (loop [path path] - (if path - (let [start-node (peek path) - _ (assert (data-node? start-node)) - elements (-> start-node - :children ; Get the indices of it - (subseq >= start-key))] - ( start-node + :children ; Get the indices of it + (subseq >= start-key)) ; skip to the start-index + next-elements (lazy-seq + (when-let [succ (right-successor (pop path))] + (forward-iterator succ start-key)))] + (concat first-elements next-elements)))) + + + (defn lookup-fwd-iter + [tree key] + (let [path (lookup-path tree key)] + (when path + (forward-iterator path key))))) + + :core.async + (do + #?(:clj + (defn chan-seq [ch] + (when-some [v ( start-node + :children ; Get the indices of it + (subseq >= start-key))] + (KonserveBackend store) init-tree (Config 1 3 (- 3 1)))) @@ -61,7 +65,7 @@ #?(:clj (let [folder "/tmp/async-hitchhiker-tree-test" _ (delete-store folder) - store (kons/add-hitchhiker-tree-handlers (KonserveBackend store) flushed ( Date: Fri, 5 Jan 2018 21:04:34 +0100 Subject: [PATCH 30/34] Provide tree from replikativ. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 3203edc..a48978c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject hitchhiker-tree "0.1.0-SNAPSHOT" +(defproject io.replikativ/hitchhiker-tree "0.1.0-SNAPSHOT" :description "A Hitchhiker Tree Library" :url "https://github.com/dgrnbrg/hitchhiker-tree" :license {:name "Eclipse Public License" From a876beea377d6ad732be525ff1755b64a5af3ddc Mon Sep 17 00:00:00 2001 From: Christian Weilbach Date: Sun, 1 Apr 2018 21:03:32 +0200 Subject: [PATCH 31/34] Add cache support and release first experimental version. --- project.clj | 6 +++--- src/hitchhiker/konserve.cljc | 2 +- src/hitchhiker/tree/core.cljc | 2 +- src/hitchhiker/tree/messaging.cljc | 32 ++++++++++++++++++++++-------- test/hitchhiker/konserve_test.cljc | 10 ++++++---- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/project.clj b/project.clj index a48978c..41dfd55 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject io.replikativ/hitchhiker-tree "0.1.0-SNAPSHOT" +(defproject io.replikativ/hitchhiker-tree "0.1.0" :description "A Hitchhiker Tree Library" :url "https://github.com/dgrnbrg/hitchhiker-tree" :license {:name "Eclipse Public License" @@ -10,8 +10,8 @@ [org.clojure/core.rrb-vector "0.0.11"] [org.clojure/core.cache "0.6.5"] - [io.replikativ/incognito "0.2.2-SNAPSHOT"] - [io.replikativ/konserve "0.4.9"]] + [io.replikativ/incognito "0.2.2"] + [io.replikativ/konserve "0.5-beta1"]] :aliases {"bench" ["with-profile" "profiling" "run" "-m" "hitchhiker.bench"]} :jvm-opts ["-server" "-Xmx3700m" "-Xms3700m"] :profiles {:test diff --git a/src/hitchhiker/konserve.cljc b/src/hitchhiker/konserve.cljc index 60046c0..9a24e07 100644 --- a/src/hitchhiker/konserve.cljc +++ b/src/hitchhiker/konserve.cljc @@ -3,7 +3,7 @@ (:require [clojure.core.rrb-vector :refer [catvec subvec]] #?(:clj [clojure.core.async :refer [chan promise-chan put!] :as async] :cljs [cljs.core.async :refer [chan promise-chan put!] :as async]) - [konserve.core :as k] + [konserve.cache :as k] [konserve.memory :refer [new-mem-store]] [hasch.core :refer [uuid]] [clojure.set :as set] diff --git a/src/hitchhiker/tree/core.cljc b/src/hitchhiker/tree/core.cljc index c311ca7..b39a334 100644 --- a/src/hitchhiker/tree/core.cljc +++ b/src/hitchhiker/tree/core.cljc @@ -197,7 +197,7 @@ throwable error." #?@(:clj [Object (compare [key1 key2] - (if (or (= (type key1) (type key2)) + (if (or (= (class key1) (class key2)) (= (order-on-edn-types key1) (order-on-edn-types key2))) (try diff --git a/src/hitchhiker/tree/messaging.cljc b/src/hitchhiker/tree/messaging.cljc index 880df26..60af51f 100644 --- a/src/hitchhiker/tree/messaging.cljc +++ b/src/hitchhiker/tree/messaging.cljc @@ -281,18 +281,34 @@ (recur (KonserveBackend store) init-tree (Config 1 3 (- 3 1)))) @@ -65,7 +66,7 @@ #?(:clj (let [folder "/tmp/async-hitchhiker-tree-test" _ (delete-store folder) - store (kons/add-hitchhiker-tree-handlers (async/KonserveBackend store) flushed ( Date: Wed, 23 May 2018 14:05:39 +0200 Subject: [PATCH 32/34] Cleanup and bump deps for release. --- README.md | 6 + hs_err_pid11680.log | 388 ----------------------------- project.clj | 17 +- src/hitchhiker/konserve.cljc | 9 +- src/hitchhiker/tree/core.cljc | 36 ++- src/hitchhiker/tree/messaging.cljc | 5 +- test/hitchhiker/konserve_test.cljc | 5 +- 7 files changed, 56 insertions(+), 410 deletions(-) delete mode 100644 hs_err_pid11680.log diff --git a/README.md b/README.md index 357f9f7..8e88f21 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,12 @@ If you'd like to see the options for the benchmarking tool, just run `lein bench See the `doc/` folder for technical details of the hitchhiker tree and Redis garbage collection system. +### Async support + +We have preliminary async support that has to be selected before macro expansion +time by setting `hitchhiker.tree.async/*async-backend*` either to `none` or +`core.async`. + ## Gratitude Thanks to the early reviewers, Kovas Boguta & Leif Walsh. diff --git a/hs_err_pid11680.log b/hs_err_pid11680.log deleted file mode 100644 index 69262f4..0000000 --- a/hs_err_pid11680.log +++ /dev/null @@ -1,388 +0,0 @@ -# -# There is insufficient memory for the Java Runtime Environment to continue. -# Native memory allocation (mmap) failed to map 2586836992 bytes for committing reserved memory. -# Possible reasons: -# The system is out of physical RAM or swap space -# In 32 bit mode, the process size limit was hit -# Possible solutions: -# Reduce memory load on the system -# Increase physical memory or swap space -# Check if swap backing store is full -# Use 64 bit Java on a 64 bit OS -# Decrease Java heap size (-Xmx/-Xms) -# Decrease number of Java threads -# Decrease Java thread stack sizes (-Xss) -# Set larger code cache with -XX:ReservedCodeCacheSize= -# This output file may be truncated or incomplete. -# -# Out of Memory Error (os_linux.cpp:2643), pid=11680, tid=0x00007f17fe6b0700 -# -# JRE version: (8.0_151-b12) (build ) -# Java VM: OpenJDK 64-Bit Server VM (25.151-b12 mixed mode linux-amd64 compressed oops) -# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again -# - ---------------- T H R E A D --------------- - -Current thread (0x00007f17f800d800): JavaThread "Unknown thread" [_thread_in_vm, id=11685, stack(0x00007f17fe5b0000,0x00007f17fe6b1000)] - -Stack: [0x00007f17fe5b0000,0x00007f17fe6b1000], sp=0x00007f17fe6af5a0, free space=1021k -Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) -V [libjvm.so+0xaa61a2] -V [libjvm.so+0x4d6d87] -V [libjvm.so+0x8e2760] -V [libjvm.so+0x8dcdce] -V [libjvm.so+0x95cc36] -V [libjvm.so+0x94cf6c] -V [libjvm.so+0x2b29be] -V [libjvm.so+0x909ed1] -V [libjvm.so+0xa6892a] -V [libjvm.so+0xa68c15] -V [libjvm.so+0x6221ef] -V [libjvm.so+0xa4c27b] -V [libjvm.so+0x6a4901] JNI_CreateJavaVM+0x61 -C [libjli.so+0x2ce3] -C [libjli.so+0x76ed] -C [libpthread.so.0+0x77fc] start_thread+0xdc - - ---------------- P R O C E S S --------------- - -Java Threads: ( => current thread ) - -Other Threads: - -=>0x00007f17f800d800 (exited) JavaThread "Unknown thread" [_thread_in_vm, id=11685, stack(0x00007f17fe5b0000,0x00007f17fe6b1000)] - -VM state:not at safepoint (not fully initialized) - -VM Mutex/Monitor currently owned by a thread: None - -GC Heap History (0 events): -No events - -Deoptimization events (0 events): -No events - -Internal exceptions (0 events): -No events - -Events (0 events): -No events - - -Dynamic libraries: -772f00000-7c0000000 rw-p 00000000 00:00 0 -55e79e2e1000-55e79e2e2000 r-xp 00000000 08:01 13506105 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -55e79e4e1000-55e79e4e2000 r--p 00000000 08:01 13506105 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -55e79e4e2000-55e79e4e3000 rw-p 00001000 08:01 13506105 /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -55e79e532000-55e79e553000 rw-p 00000000 00:00 0 [heap] -7f17e7471000-7f17e7942000 ---p 00000000 00:00 0 -7f17e7942000-7f17e7bb6000 rw-p 00000000 00:00 0 -7f17e7bb6000-7f17e7f6c000 ---p 00000000 00:00 0 -7f17e7f6c000-7f17e81dc000 rwxp 00000000 00:00 0 -7f17e81dc000-7f17f6f6c000 ---p 00000000 00:00 0 -7f17f6f6c000-7f17f6f74000 r-xp 00000000 08:01 13506156 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so -7f17f6f74000-7f17f7173000 ---p 00008000 08:01 13506156 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so -7f17f7173000-7f17f7174000 r--p 00007000 08:01 13506156 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so -7f17f7174000-7f17f7175000 rw-p 00008000 08:01 13506156 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libzip.so -7f17f7175000-7f17f7180000 r-xp 00000000 08:01 7864765 /lib/x86_64-linux-gnu/libnss_files-2.26.so -7f17f7180000-7f17f737f000 ---p 0000b000 08:01 7864765 /lib/x86_64-linux-gnu/libnss_files-2.26.so -7f17f737f000-7f17f7380000 r--p 0000a000 08:01 7864765 /lib/x86_64-linux-gnu/libnss_files-2.26.so -7f17f7380000-7f17f7381000 rw-p 0000b000 08:01 7864765 /lib/x86_64-linux-gnu/libnss_files-2.26.so -7f17f7381000-7f17f7387000 rw-p 00000000 00:00 0 -7f17f7387000-7f17f7392000 r-xp 00000000 08:01 7864767 /lib/x86_64-linux-gnu/libnss_nis-2.26.so -7f17f7392000-7f17f7591000 ---p 0000b000 08:01 7864767 /lib/x86_64-linux-gnu/libnss_nis-2.26.so -7f17f7591000-7f17f7592000 r--p 0000a000 08:01 7864767 /lib/x86_64-linux-gnu/libnss_nis-2.26.so -7f17f7592000-7f17f7593000 rw-p 0000b000 08:01 7864767 /lib/x86_64-linux-gnu/libnss_nis-2.26.so -7f17f7593000-7f17f75aa000 r-xp 00000000 08:01 7864762 /lib/x86_64-linux-gnu/libnsl-2.26.so -7f17f75aa000-7f17f77a9000 ---p 00017000 08:01 7864762 /lib/x86_64-linux-gnu/libnsl-2.26.so -7f17f77a9000-7f17f77aa000 r--p 00016000 08:01 7864762 /lib/x86_64-linux-gnu/libnsl-2.26.so -7f17f77aa000-7f17f77ab000 rw-p 00017000 08:01 7864762 /lib/x86_64-linux-gnu/libnsl-2.26.so -7f17f77ab000-7f17f77ad000 rw-p 00000000 00:00 0 -7f17f77ad000-7f17f77b5000 r-xp 00000000 08:01 7864763 /lib/x86_64-linux-gnu/libnss_compat-2.26.so -7f17f77b5000-7f17f79b4000 ---p 00008000 08:01 7864763 /lib/x86_64-linux-gnu/libnss_compat-2.26.so -7f17f79b4000-7f17f79b5000 r--p 00007000 08:01 7864763 /lib/x86_64-linux-gnu/libnss_compat-2.26.so -7f17f79b5000-7f17f79b6000 rw-p 00008000 08:01 7864763 /lib/x86_64-linux-gnu/libnss_compat-2.26.so -7f17f79b6000-7f17f79e3000 r-xp 00000000 08:01 13506131 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so -7f17f79e3000-7f17f7be3000 ---p 0002d000 08:01 13506131 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so -7f17f7be3000-7f17f7be4000 r--p 0002d000 08:01 13506131 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so -7f17f7be4000-7f17f7be6000 rw-p 0002e000 08:01 13506131 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libjava.so -7f17f7be6000-7f17f7bf6000 r-xp 00000000 08:01 13506152 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so -7f17f7bf6000-7f17f7df5000 ---p 00010000 08:01 13506152 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so -7f17f7df5000-7f17f7df7000 r--p 0000f000 08:01 13506152 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so -7f17f7df7000-7f17f7df8000 rw-p 00011000 08:01 13506152 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/libverify.so -7f17f7df8000-7f17f7dff000 r-xp 00000000 08:01 7864772 /lib/x86_64-linux-gnu/librt-2.26.so -7f17f7dff000-7f17f7ffe000 ---p 00007000 08:01 7864772 /lib/x86_64-linux-gnu/librt-2.26.so -7f17f7ffe000-7f17f7fff000 r--p 00006000 08:01 7864772 /lib/x86_64-linux-gnu/librt-2.26.so -7f17f7fff000-7f17f8000000 rw-p 00007000 08:01 7864772 /lib/x86_64-linux-gnu/librt-2.26.so -7f17f8000000-7f17f8035000 rw-p 00000000 00:00 0 -7f17f8035000-7f17fc000000 ---p 00000000 00:00 0 -7f17fc024000-7f17fc03a000 r-xp 00000000 08:01 7864750 /lib/x86_64-linux-gnu/libgcc_s.so.1 -7f17fc03a000-7f17fc239000 ---p 00016000 08:01 7864750 /lib/x86_64-linux-gnu/libgcc_s.so.1 -7f17fc239000-7f17fc23a000 r--p 00015000 08:01 7864750 /lib/x86_64-linux-gnu/libgcc_s.so.1 -7f17fc23a000-7f17fc23b000 rw-p 00016000 08:01 7864750 /lib/x86_64-linux-gnu/libgcc_s.so.1 -7f17fc23b000-7f17fc390000 r-xp 00000000 08:01 7864759 /lib/x86_64-linux-gnu/libm-2.26.so -7f17fc390000-7f17fc58f000 ---p 00155000 08:01 7864759 /lib/x86_64-linux-gnu/libm-2.26.so -7f17fc58f000-7f17fc590000 r--p 00154000 08:01 7864759 /lib/x86_64-linux-gnu/libm-2.26.so -7f17fc590000-7f17fc591000 rw-p 00155000 08:01 7864759 /lib/x86_64-linux-gnu/libm-2.26.so -7f17fc591000-7f17fc709000 r-xp 00000000 08:01 2628521 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.24 -7f17fc709000-7f17fc908000 ---p 00178000 08:01 2628521 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.24 -7f17fc908000-7f17fc912000 r--p 00177000 08:01 2628521 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.24 -7f17fc912000-7f17fc914000 rw-p 00181000 08:01 2628521 /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.24 -7f17fc914000-7f17fc917000 rw-p 00000000 00:00 0 -7f17fc917000-7f17fd5a1000 r-xp 00000000 08:01 13508278 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so -7f17fd5a1000-7f17fd7a1000 ---p 00c8a000 08:01 13508278 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so -7f17fd7a1000-7f17fd834000 r--p 00c8a000 08:01 13508278 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so -7f17fd834000-7f17fd85c000 rw-p 00d1d000 08:01 13508278 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so -7f17fd85c000-7f17fd88d000 rw-p 00000000 00:00 0 -7f17fd88d000-7f17fd8a7000 r-xp 00000000 08:01 7864770 /lib/x86_64-linux-gnu/libpthread-2.26.so -7f17fd8a7000-7f17fdaa6000 ---p 0001a000 08:01 7864770 /lib/x86_64-linux-gnu/libpthread-2.26.so -7f17fdaa6000-7f17fdaa7000 r--p 00019000 08:01 7864770 /lib/x86_64-linux-gnu/libpthread-2.26.so -7f17fdaa7000-7f17fdaa8000 rw-p 0001a000 08:01 7864770 /lib/x86_64-linux-gnu/libpthread-2.26.so -7f17fdaa8000-7f17fdaac000 rw-p 00000000 00:00 0 -7f17fdaac000-7f17fdaaf000 r-xp 00000000 08:01 7864758 /lib/x86_64-linux-gnu/libdl-2.26.so -7f17fdaaf000-7f17fdcae000 ---p 00003000 08:01 7864758 /lib/x86_64-linux-gnu/libdl-2.26.so -7f17fdcae000-7f17fdcaf000 r--p 00002000 08:01 7864758 /lib/x86_64-linux-gnu/libdl-2.26.so -7f17fdcaf000-7f17fdcb0000 rw-p 00003000 08:01 7864758 /lib/x86_64-linux-gnu/libdl-2.26.so -7f17fdcb0000-7f17fdccc000 r-xp 00000000 08:01 7869027 /lib/x86_64-linux-gnu/libz.so.1.2.11 -7f17fdccc000-7f17fdecb000 ---p 0001c000 08:01 7869027 /lib/x86_64-linux-gnu/libz.so.1.2.11 -7f17fdecb000-7f17fdecc000 r--p 0001b000 08:01 7869027 /lib/x86_64-linux-gnu/libz.so.1.2.11 -7f17fdecc000-7f17fdecd000 rw-p 0001c000 08:01 7869027 /lib/x86_64-linux-gnu/libz.so.1.2.11 -7f17fdecd000-7f17fe0a3000 r-xp 00000000 08:01 7864755 /lib/x86_64-linux-gnu/libc-2.26.so -7f17fe0a3000-7f17fe2a3000 ---p 001d6000 08:01 7864755 /lib/x86_64-linux-gnu/libc-2.26.so -7f17fe2a3000-7f17fe2a7000 r--p 001d6000 08:01 7864755 /lib/x86_64-linux-gnu/libc-2.26.so -7f17fe2a7000-7f17fe2a9000 rw-p 001da000 08:01 7864755 /lib/x86_64-linux-gnu/libc-2.26.so -7f17fe2a9000-7f17fe2ad000 rw-p 00000000 00:00 0 -7f17fe2ad000-7f17fe2bb000 r-xp 00000000 08:01 13506116 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so -7f17fe2bb000-7f17fe4ba000 ---p 0000e000 08:01 13506116 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so -7f17fe4ba000-7f17fe4bb000 r--p 0000d000 08:01 13506116 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so -7f17fe4bb000-7f17fe4bc000 rw-p 0000e000 08:01 13506116 /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/jli/libjli.so -7f17fe4bc000-7f17fe4e3000 r-xp 00000000 08:01 7864751 /lib/x86_64-linux-gnu/ld-2.26.so -7f17fe5b0000-7f17fe5b3000 ---p 00000000 00:00 0 -7f17fe5b3000-7f17fe6b6000 rw-p 00000000 00:00 0 -7f17fe6d1000-7f17fe6d9000 rw-s 00000000 08:01 5256118 /tmp/hsperfdata_christian/11680 -7f17fe6d9000-7f17fe6da000 rw-p 00000000 00:00 0 -7f17fe6da000-7f17fe6db000 r--p 00000000 00:00 0 -7f17fe6db000-7f17fe6de000 rw-p 00000000 00:00 0 -7f17fe6de000-7f17fe6e1000 r--p 00000000 00:00 0 [vvar] -7f17fe6e1000-7f17fe6e3000 r-xp 00000000 00:00 0 [vdso] -7f17fe6e3000-7f17fe6e4000 r--p 00027000 08:01 7864751 /lib/x86_64-linux-gnu/ld-2.26.so -7f17fe6e4000-7f17fe6e5000 rw-p 00028000 08:01 7864751 /lib/x86_64-linux-gnu/ld-2.26.so -7f17fe6e5000-7f17fe6e6000 rw-p 00000000 00:00 0 -7fff687d9000-7fff687fc000 rw-p 00000000 00:00 0 [stack] -ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] - -VM Arguments: -jvm_args: -Dfile.encoding=UTF-8 -Xmx3700m -Xms3700m -Dclojure.compile.path=/home/christian/Development/hitchhiker-tree/target/classes -Dhitchhiker-tree.version=0.1.0-SNAPSHOT -Dclojure.debug=false -java_command: clojure.main -i /tmp/form-init7961262958693552825.clj -java_class_path (initial): /home/christian/Development/hitchhiker-tree/test:/home/christian/Development/hitchhiker-tree/src:/home/christian/Development/hitchhiker-tree/dev:/home/christian/Development/hitchhiker-tree/src:/home/christian/Development/hitchhiker-tree/dev-resources:/home/christian/Development/hitchhiker-tree/resources:/home/christian/Development/hitchhiker-tree/target/classes:/home/christian/.m2/repository/io/replikativ/hasch/0.3.4/hasch-0.3.4.jar:/home/christian/.m2/repository/org/iq80/snappy/snappy/0.4/snappy-0.4.jar:/home/christian/.m2/repository/org/clojure/clojure/1.8.0/clojure-1.8.0.jar:/home/christian/.m2/repository/net/incongru/watchservice/barbary-watchservice/1.0/barbary-watchservice-1.0.jar:/home/christian/.m2/repository/net/cgrand/parsley/0.9.3/parsley-0.9.3.jar:/home/christian/.m2/repository/net/cgrand/regex/1.1.0/regex-1.1.0.jar:/home/christian/.m2/repository/org/clojure/google-closure-library/0.0-20151016-61277aea/google-closure-library-0.0-20151016-61277aea.jar:/home/christian/.m2/repository/com/cemerick/austin/0.1.6/austin-0.1.6.jar:/home/christian/.m2/repository/clj-time/clj-time/0.13.0/clj-time-0.13.0.jar:/home/christian/.m2/repository/com/taoensso/truss/1.3.7/truss-1.3.7.jar:/home/christian/.m2/repository/io/replikativ/konserve/0.4.9/konserve-0.4.9.jar:/home/christian/.m2/repository/crypto-equality/crypto-equality/1.0.0/crypto-equality-1.0.0.jar:/home/christian/.m2/repository/org/clojure/tools.analyzer.jvm/0.6.10/tools.analyzer.jvm-0.6.10.jar:/home/christian/.m2/repository/com/google/javascript/closure-compiler-externs/v20160315/closure-compiler-externs-v20160315.jar:/home/christian/.m2/repository/org/clojure/clojurescript/1.8.51/clojurescript-1.8.51.jar:/home/christian/.m2/repository/net/jpountz/lz4/lz4/1.3/lz4-1.3.jar:/home/christian/.m2/repository/net/java/dev/jna/jna/3.2.2/jna-3.2.2.jar:/home/christian/.m2/repository/org/clojure/tools.analyzer/0.6.9/tools.analyzer-0.6.9.jar:/home/christian/.m2/repository/org/tukaani/xz/1.6/xz-1. -Launcher Type: SUN_STANDARD - -Environment Variables: -CLASSPATH=/home/christian/.lein/self-installs/leiningen-2.7.1-standalone.jar -PATH=/home/christian/.cargo/bin:/home/christian/miniconda3/bin:/home/christian/torch/install/bin:/home/christian/.opam/4.03.0/bin:/home/christian/.fzf/bin:/home/christian/.cargo/bin:/home/christian/miniconda3/bin:/home/christian/torch/install/bin:/home/christian/.opam/4.03.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games -USERNAME=christian -LD_LIBRARY_PATH=/home/christian/torch/install/lib/home/christian/torch/install/lib -SHELL=/usr/bin/fish -DISPLAY=:0 -DYLD_LIBRARY_PATH=/home/christian/torch/install/lib/home/christian/torch/install/lib - -Signal Handlers: -SIGSEGV: [libjvm.so+0xaa6ae0], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGBUS: [libjvm.so+0xaa6ae0], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGFPE: [libjvm.so+0x8ddf00], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGPIPE: [libjvm.so+0x8ddf00], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGXFSZ: [libjvm.so+0x8ddf00], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGILL: [libjvm.so+0x8ddf00], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGUSR1: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGUSR2: [libjvm.so+0x8dddb0], sa_mask[0]=00100000000000000000000000000000, sa_flags=SA_RESTART|SA_SIGINFO -SIGHUP: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGINT: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGTERM: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGQUIT: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none - - ---------------- S Y S T E M --------------- - -OS:DISTRIB_ID=Ubuntu -DISTRIB_RELEASE=17.10 -DISTRIB_CODENAME=artful -DISTRIB_DESCRIPTION="Ubuntu 17.10" - -uname:Linux 4.13.0-19-generic #22-Ubuntu SMP Mon Dec 4 11:58:07 UTC 2017 x86_64 -libc:glibc 2.26 NPTL 2.26 -rlimit: STACK 8192k, CORE 0k, NPROC 28899, NOFILE 4096, AS infinity -load average:0.91 1.11 1.25 - -/proc/meminfo: -MemTotal: 7451208 kB -MemFree: 119372 kB -MemAvailable: 586760 kB -Buffers: 34624 kB -Cached: 577720 kB -SwapCached: 36268 kB -Active: 5629520 kB -Inactive: 1330768 kB -Active(anon): 5447192 kB -Inactive(anon): 869144 kB -Active(file): 182328 kB -Inactive(file): 461624 kB -Unevictable: 112 kB -Mlocked: 112 kB -SwapTotal: 2097148 kB -SwapFree: 907448 kB -Dirty: 1012 kB -Writeback: 0 kB -AnonPages: 6315900 kB -Mapped: 440724 kB -Shmem: 325956 kB -Slab: 210284 kB -SReclaimable: 85932 kB -SUnreclaim: 124352 kB -KernelStack: 18864 kB -PageTables: 83560 kB -NFS_Unstable: 0 kB -Bounce: 0 kB -WritebackTmp: 0 kB -CommitLimit: 5822752 kB -Committed_AS: 18522868 kB -VmallocTotal: 34359738367 kB -VmallocUsed: 0 kB -VmallocChunk: 0 kB -HardwareCorrupted: 0 kB -AnonHugePages: 0 kB -ShmemHugePages: 0 kB -ShmemPmdMapped: 0 kB -CmaTotal: 0 kB -CmaFree: 0 kB -HugePages_Total: 0 -HugePages_Free: 0 -HugePages_Rsvd: 0 -HugePages_Surp: 0 -Hugepagesize: 2048 kB -DirectMap4k: 362660 kB -DirectMap2M: 7297024 kB -DirectMap1G: 0 kB - - -CPU:total 4 (initial active 4) (2 cores per cpu, 2 threads per core) family 6 model 78 stepping 3, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, avx, avx2, aes, clmul, erms, 3dnowpref, lzcnt, ht, tsc, tscinvbit, bmi1, bmi2, adx - -/proc/cpuinfo: -processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 78 -model name : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz -stepping : 3 -microcode : 0xba -cpu MHz : 2600.000 -cache size : 4096 KB -physical id : 0 -siblings : 4 -core id : 0 -cpu cores : 2 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp -bugs : -bogomips : 5184.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 78 -model name : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz -stepping : 3 -microcode : 0xba -cpu MHz : 2600.000 -cache size : 4096 KB -physical id : 0 -siblings : 4 -core id : 1 -cpu cores : 2 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp -bugs : -bogomips : 5184.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 78 -model name : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz -stepping : 3 -microcode : 0xba -cpu MHz : 2600.000 -cache size : 4096 KB -physical id : 0 -siblings : 4 -core id : 0 -cpu cores : 2 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp -bugs : -bogomips : 5184.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 78 -model name : Intel(R) Core(TM) i7-6500U CPU @ 2.50GHz -stepping : 3 -microcode : 0xba -cpu MHz : 2600.000 -cache size : 4096 KB -physical id : 0 -siblings : 4 -core id : 1 -cpu cores : 2 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp -bugs : -bogomips : 5184.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - - - -Memory: 4k page, physical 7451208k(119372k free), swap 2097148k(907448k free) - -vm_info: OpenJDK 64-Bit Server VM (25.151-b12) for linux-amd64 JRE (1.8.0_151-8u151-b12-0ubuntu0.17.10.2-b12), built on Oct 23 2017 22:43:02 by "buildd" with gcc 7.2.0 - -time: Mon Dec 18 10:52:45 2017 -elapsed time: 0 seconds (0d 0h 0m 0s) - diff --git a/project.clj b/project.clj index 41dfd55..2b3a9b9 100644 --- a/project.clj +++ b/project.clj @@ -1,17 +1,16 @@ -(defproject io.replikativ/hitchhiker-tree "0.1.0" +(defproject io.replikativ/hitchhiker-tree "0.1.1" :description "A Hitchhiker Tree Library" :url "https://github.com/dgrnbrg/hitchhiker-tree" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :dependencies [[org.clojure/clojure "1.8.0"] + :dependencies [[org.clojure/clojure "1.9.0"] [org.clojure/clojurescript "1.8.51" :scope "provided"] [org.clojure/core.memoize "0.5.8"] [com.taoensso/carmine "2.12.2"] [org.clojure/core.rrb-vector "0.0.11"] [org.clojure/core.cache "0.6.5"] - [io.replikativ/incognito "0.2.2"] - [io.replikativ/konserve "0.5-beta1"]] + [io.replikativ/konserve "0.5.0-beta3"]] :aliases {"bench" ["with-profile" "profiling" "run" "-m" "hitchhiker.bench"]} :jvm-opts ["-server" "-Xmx3700m" "-Xms3700m"] :profiles {:test @@ -23,15 +22,15 @@ [org.clojure/tools.cli "0.3.3"] [org.clojure/test.check "0.9.0"] [com.infolace/excel-templates "0.3.3"]]} - :dev {:dependencies [[binaryage/devtools "0.8.2"] - [figwheel-sidecar "0.5.8"] - [com.cemerick/piggieback "0.2.1"] + :dev {:dependencies [#_[binaryage/devtools "0.8.2"] + #_[figwheel-sidecar "0.5.8"] + #_[com.cemerick/piggieback "0.2.1"] [org.clojure/test.check "0.9.0"]] :source-paths ["src" "dev"] - :plugins [[lein-figwheel "0.5.8"]] + ;:plugins [[lein-figwheel "0.5.8"]] :repl-options {; for nREPL dev you really need to limit output :init (set! *print-length* 50) - :nrepl-middleware [cemerick.piggieback/wrap-cljs-repl]}}} + #_:nrepl-middleware #_[cemerick.piggieback/wrap-cljs-repl]}}} :clean-targets ^{:protect false} ["resources/public/js/compiled" "target"] diff --git a/src/hitchhiker/konserve.cljc b/src/hitchhiker/konserve.cljc index 9a24e07..6327ea7 100644 --- a/src/hitchhiker/konserve.cljc +++ b/src/hitchhiker/konserve.cljc @@ -9,7 +9,8 @@ [clojure.set :as set] #?(:clj [hitchhiker.tree.core :refer [go-try (case core/*async-backend* + (-> (case *async-backend* :none (async/KonserveAddr store (core/last-key node) id (synthesize-storage-addr id)))))) @@ -66,7 +67,7 @@ [store root-key] (go-try (let [val (let [ch (k/get-in store [root-key])] - (case core/*async-backend* + (case *async-backend* :none (async/ Date: Fri, 1 Jun 2018 23:24:26 +0200 Subject: [PATCH 33/34] Support flushing without root and faster cached lookup in konserve. The lookup in konserve is not hitting a blocking core.async (case *async-backend* - :none (async/ (case *async-backend* + :none (async/> (flush-children-without-root (:children tree) backend stats) + Date: Mon, 4 Jun 2018 10:46:23 +0200 Subject: [PATCH 34/34] Add plotting similar to the presentation. Provide missing binding file. --- project.clj | 10 +- src/hitchhiker/plot.clj | 211 +++++++++++++++++++++++++++++++++ src/hitchhiker/tree/async.cljc | 6 + 3 files changed, 224 insertions(+), 3 deletions(-) create mode 100644 src/hitchhiker/plot.clj create mode 100644 src/hitchhiker/tree/async.cljc diff --git a/project.clj b/project.clj index d3ae357..5307388 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject io.replikativ/hitchhiker-tree "0.1.2-SNAPSHOT" +(defproject io.replikativ/hitchhiker-tree "0.1.2" :description "A Hitchhiker Tree Library" :url "https://github.com/dgrnbrg/hitchhiker-tree" :license {:name "Eclipse Public License" @@ -10,7 +10,8 @@ [org.clojure/core.rrb-vector "0.0.11"] [org.clojure/core.cache "0.6.5"] - [io.replikativ/konserve "0.5.0-beta3"]] + [io.replikativ/konserve "0.5.0-beta3"] + ] :aliases {"bench" ["with-profile" "profiling" "run" "-m" "hitchhiker.bench"]} :jvm-opts ["-server" "-Xmx3700m" "-Xms3700m"] :profiles {:test @@ -25,7 +26,10 @@ :dev {:dependencies [#_[binaryage/devtools "0.8.2"] #_[figwheel-sidecar "0.5.8"] #_[com.cemerick/piggieback "0.2.1"] - [org.clojure/test.check "0.9.0"]] + [org.clojure/test.check "0.9.0"] + ;; plotting + [aysylu/loom "1.0.1"] + [cheshire "5.8.0"]] :source-paths ["src" "dev"] ;:plugins [[lein-figwheel "0.5.8"]] :repl-options {; for nREPL dev you really need to limit output diff --git a/src/hitchhiker/plot.clj b/src/hitchhiker/plot.clj new file mode 100644 index 0000000..485d42d --- /dev/null +++ b/src/hitchhiker/plot.clj @@ -0,0 +1,211 @@ +(ns hitchhiker.plot + "This namespace provides functions to help visualizing hh-trees. + + It provides a visualization similar to those in https://youtu.be/jdn617M3-P4?t=1583 + " + (:require [konserve.memory :refer [new-mem-store]] + [hitchhiker.konserve :as kons] + [konserve.cache :as kc] + [hitchhiker.tree.core :refer [ \"") + "\" -- \"")) + (.append (dot-esc n2l)) + (.append \")) + (let [eattrs (dissoc eattrs :compass)] + (when (or (:label eattrs) (< 1 (count eattrs))) + (.append sb \space) + (.append sb (dot-attrs eattrs)))) + (.append sb "\n"))) + (doseq [n (nodes g)] + (doto sb + (.append " \"") + (.append (dot-esc (str (or (node-label n) n)))) + (.append \")) + (when-let [nattrs (when a? + (dot-attrs (attrs g n)))] + (.append sb \space) + (.append sb nattrs)) + (.append sb "\n")) + (str (doto sb (.append "}"))))) + + +(ns hitchhiker.plot) + +(def store (kons/add-hitchhiker-tree-handlers + (kc/ensure-cache (async/Config 3 3 3))) + (shuffle (range 1 25)))) + (kons/->KonserveBackend store)))) + + +(def flushed (KonserveBackend store)))) + + + +(defn init-graph [store] + (apply lg/digraph + (->> @(:state store) + (filter (fn [[id {:keys [children]}]] children)) + (map (fn [[id {:keys [children] :as node}]] + (if (:op-buf node) + {id (mapv (fn [c] (:konserve-key c)) children)} + {id []})))))) + + +(defn use-record-nodes [g] + (attr/add-attr-to-nodes g :shape "record" (lg/nodes g))) + + +(defn node-layout [g [id {:keys [children] :as node}]] + (if (core/index-node? node) + (attr/add-attr + g id + :label (str + ;; key space separators + (str/join " | " + (map #(str "<" % "> " ;; compass point (invisible) + "\\<" % "\\>") + (map core/last-key (:children node)))) + ;; render op-log + " | {" + (str/join " | " (map (fn [{:keys [key value]}] + key) + (:op-buf node))) + "}")) + (attr/add-attr + g id + :label (str (str/join "|" (map key (:children node))))))) + + +(defn set-node-layouts [g store] + (->> @(:state store) + (filter (fn [[id {:keys [children]}]] children)) + (reduce node-layout g))) + +(defn edge-hash [id] + (-> id str (subs 0 4))) + +(defn set-edge-layouts [g store] + (let [node-map @(:state store) + edges (lg/edges g)] + (reduce + (fn [g [n1 n2]] + (let [h (:konserve-key (clojure.core.async/ g + (attr/add-attr-to-edges :compass (core/last-key (node-map n2)) + [[n1 n2]]) + (attr/add-attr-to-edges :label (edge-hash h) [[n1 n2]])))) + g + edges))) + + +(defn create-graph [store] + (-> (init-graph store) + use-record-nodes + (set-node-layouts store) + (set-edge-layouts store))) + + + +(comment + (view (create-graph store)) + + (println (lio/dot-str g))) + + + +(defn remove-storage-addrs [[k v]] + (if (core/index-node? v) + [k (-> v + (dissoc :storage-addr :cfg) + (update :children (fn [cs] (mapv #(dissoc % :storage-addr :store) cs))) + (update :op-buf (fn [cs] (mapv #(into {} %) cs))) + )] + [k (dissoc v :storage-addr :cfg)])) + + +(comment + (spit "/tmp/sample-tree.json" + (json/generate-string + (into {} (map remove-storage-addrs @(:state store))))) + + + (prn (map remove-storage-addrs @(:state store)))) + + + + + + + + diff --git a/src/hitchhiker/tree/async.cljc b/src/hitchhiker/tree/async.cljc new file mode 100644 index 0000000..7f75fed --- /dev/null +++ b/src/hitchhiker/tree/async.cljc @@ -0,0 +1,6 @@ +(ns hitchhiker.tree.async) + +;; rebind this *before* loading any other +;; hh-tree namespace, so it has effect at +;; macro-expansion time +(def ^:dynamic *async-backend* :none)