From 02ef17d58297a3fcd614b09ab9d8d3df3efdf9ed Mon Sep 17 00:00:00 2001 From: keesterbrugge Date: Mon, 28 Jan 2019 15:02:40 +0100 Subject: [PATCH 1/5] add function that selects columns based on regex --- src/huri/core.clj | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/huri/core.clj b/src/huri/core.clj index 0015745..737bbfc 100644 --- a/src/huri/core.clj +++ b/src/huri/core.clj @@ -375,6 +375,13 @@ the arguments passed in are not nill. Else returns nil." [cols df] (map (juxtm (map-from-keys ->keyfn cols)) df)) +(defn select-cols-regex + [pattern df] + (hc/select-cols (filter #(re-matches pattern + (name %)) + (hc/cols df)) + df)) + (s/def ::join-on (s/and (s/or :vec (s/cat :left ::keyfn :right ::keyfn) :singleton ::keyfn) From 0f20d64128e24be30056a692cf53667472da162a Mon Sep 17 00:00:00 2001 From: keesterbrugge Date: Mon, 28 Jan 2019 15:12:36 +0100 Subject: [PATCH 2/5] Add comparator function for custom sorting --- src/huri/core.clj | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/huri/core.clj b/src/huri/core.clj index 737bbfc..1743d92 100644 --- a/src/huri/core.clj +++ b/src/huri/core.clj @@ -382,6 +382,25 @@ the arguments passed in are not nill. Else returns nil." (hc/cols df)) df)) +(defn compare-by [& key-cmp-pairs] + "Adapted this function from https://groups.google.com/d/msg/clojure/VVVa3TS15pU/pT3iG_W2VroJ + Changed how nil is handled. Now it's always sorted last." + (fn [x y] + (loop [[map-k cmp-k & more] key-cmp-pairs] + (let [x' (map-k x) + y' (map-k y) + compare-fn (cmp-k {:asc compare :desc #(compare %2 %1)}) + result (compare-fn x' y')] + (cond + (and (= nil x' y') more) (recur more) + (= nil x' y') 0 + (nil? x') 1 + (nil? y') -1 + (and (zero? result) more) (recur more) + :else result))))) +;; (sort (compare-by :a :asc :b :desc) [{:a 3 :b 3} {:a 2 :b 4}{:a nil :b 4} {:a nil :b nil} {:a nil :b 5}]) +;; => ({:a 2, :b 4} {:a 3, :b 3} {:a nil, :b 5} {:a nil, :b 4} {:a nil, :b nil}) + (s/def ::join-on (s/and (s/or :vec (s/cat :left ::keyfn :right ::keyfn) :singleton ::keyfn) From 523e06f41067672a6d123b9a58c7a5e84af0f9a0 Mon Sep 17 00:00:00 2001 From: keesterbrugge Date: Mon, 28 Jan 2019 17:40:36 +0100 Subject: [PATCH 3/5] add function filter-cols and reimplement select-cols-regex Say I want to filter columns based on some predicate like a regular expression, whether it is an element of some collection of keywords or some composition of these. In this case it would be nice to have a function like `filter-cols` as it simplifies the code as follows: From ``` clojure (->> df ;; .. some transformation, perhaps new columns are added (#(hc/select-cols (filter pred' (hc/cols %)) % ))) ``` to ``` clojure (->> df ;; .. some transformation, perhaps new columns are added (filter-cols pred')) ``` I've also reimplemented `select-cols-regex` using this new function --- src/huri/core.clj | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/huri/core.clj b/src/huri/core.clj index 1743d92..a89f257 100644 --- a/src/huri/core.clj +++ b/src/huri/core.clj @@ -375,29 +375,32 @@ the arguments passed in are not nill. Else returns nil." [cols df] (map (juxtm (map-from-keys ->keyfn cols)) df)) -(defn select-cols-regex - [pattern df] - (hc/select-cols (filter #(re-matches pattern - (name %)) - (hc/cols df)) - df)) +(defn filter-cols + [pred df] + "filters the column name keywords with pred and selects all remaining columns" + (select-cols (filter pred (cols df)) df)) + +(defn select-cols-regex + "select the columns whose stringified keyword match the regular expression" + [pattern df] + (filter-cols #(re-matches pattern (name %)) df)) (defn compare-by [& key-cmp-pairs] - "Adapted this function from https://groups.google.com/d/msg/clojure/VVVa3TS15pU/pT3iG_W2VroJ - Changed how nil is handled. Now it's always sorted last." - (fn [x y] - (loop [[map-k cmp-k & more] key-cmp-pairs] - (let [x' (map-k x) - y' (map-k y) - compare-fn (cmp-k {:asc compare :desc #(compare %2 %1)}) - result (compare-fn x' y')] - (cond - (and (= nil x' y') more) (recur more) - (= nil x' y') 0 - (nil? x') 1 - (nil? y') -1 - (and (zero? result) more) (recur more) - :else result))))) + "Adapted this function from https://groups.google.com/d/msg/clojure/VVVa3TS15pU/pT3iG_W2VroJ + Changed how nil is handled. Now it's always sorted last." + (fn [x y] + (loop [[map-k cmp-k & more] key-cmp-pairs] + (let [x' (map-k x) + y' (map-k y) + compare-fn (cmp-k {:asc compare :desc #(compare %2 %1)}) + result (compare-fn x' y')] + (cond + (and (= nil x' y') more) (recur more) + (= nil x' y') 0 + (nil? x') 1 + (nil? y') -1 + (and (zero? result) more) (recur more) + :else result))))) ;; (sort (compare-by :a :asc :b :desc) [{:a 3 :b 3} {:a 2 :b 4}{:a nil :b 4} {:a nil :b nil} {:a nil :b 5}]) ;; => ({:a 2, :b 4} {:a 3, :b 3} {:a nil, :b 5} {:a nil, :b 4} {:a nil, :b nil}) From bc4a3ec1b9fa3e289444f2d345cc79bc532b74ea Mon Sep 17 00:00:00 2001 From: keesterbrugge Date: Tue, 5 Feb 2019 17:00:56 +0100 Subject: [PATCH 4/5] remove fn select-cols-regex remove fn as it doesn't add enough value --- src/huri/core.clj | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/huri/core.clj b/src/huri/core.clj index a89f257..6e43dd4 100644 --- a/src/huri/core.clj +++ b/src/huri/core.clj @@ -380,11 +380,6 @@ the arguments passed in are not nill. Else returns nil." "filters the column name keywords with pred and selects all remaining columns" (select-cols (filter pred (cols df)) df)) -(defn select-cols-regex - "select the columns whose stringified keyword match the regular expression" - [pattern df] - (filter-cols #(re-matches pattern (name %)) df)) - (defn compare-by [& key-cmp-pairs] "Adapted this function from https://groups.google.com/d/msg/clojure/VVVa3TS15pU/pT3iG_W2VroJ Changed how nil is handled. Now it's always sorted last." From 9609273a64ab8d7fd43f00932e8d229cf24f5021 Mon Sep 17 00:00:00 2001 From: keesterbrugge Date: Tue, 5 Feb 2019 17:12:13 +0100 Subject: [PATCH 5/5] add derive-cols* fn that adds columns sequentially --- src/huri/core.clj | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/huri/core.clj b/src/huri/core.clj index 6e43dd4..e546027 100644 --- a/src/huri/core.clj +++ b/src/huri/core.clj @@ -129,6 +129,23 @@ the arguments passed in are not nill. Else returns nil." (apply comp)) df)) +(defn derive-cols* [new-cols df] + "respects ordering of the operations. + + ```clojure + (= (derive-cols* (ordered-map :c [inc :b] :d [inc :c]) [{:a 1 :b 2}{:a 3 :b 10}]) + (derive-cols* [:c [inc :b] :d [inc :c]] [{:a 1 :b 2}{:a 3 :b 10}]) + '({:a 1, :b 2, :c 3, :d 4} {:a 3, :b 10, :c 11, :d 12})) ; => true + ``` + " + (loop [[k op-vec & remaining :as new-cols'] (if (map? new-cols) + (mapcat vec new-cols) + new-cols) + df df] + (if (seq new-cols') + (recur remaining (derive-cols {k op-vec} df)) + df))) + (defn update-cols [update-fns df] (derive-cols (for-map [[k f] update-fns]