From 6f8c2ab80a1e918a3acf804eb550fc0218179b10 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Mon, 26 Feb 2024 21:38:19 -0500 Subject: [PATCH 01/12] Update to use latest index avail if index fails --- deps.edn | 3 ++- src/fluree/db/ledger/consensus/dbsync2.clj | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/deps.edn b/deps.edn index 38e4300f..b90b2cac 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,8 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.1"} org.clojure/data.xml {:mvn/version "0.2.0-alpha8"} com.fluree/alphabase {:mvn/version "3.3.0"} - com.fluree/db {:mvn/version "1.0.5"} + com.fluree/db {:git/url "https://github.com/fluree/db.git" + :sha "bbd33250e826bf51259c1a954dc00d1324fa4819"} com.fluree/raft {:mvn/version "1.0.0-beta1"} com.fluree/crypto {:mvn/version "0.4.0"} diff --git a/src/fluree/db/ledger/consensus/dbsync2.clj b/src/fluree/db/ledger/consensus/dbsync2.clj index be3d6aae..8377c0ab 100644 --- a/src/fluree/db/ledger/consensus/dbsync2.clj +++ b/src/fluree/db/ledger/consensus/dbsync2.clj @@ -436,10 +436,11 @@ (defn consistency-full-check [conn ledgers-info remote-sync-servers] - (let [sync-chan (async/chan) ;; files to sync are placed on this channel - res-chan (async/chan) ;; results file sync (error/success) are placed on this channel - parallelism 8] - (if (empty? ledgers-info) + (let [sync-chan (async/chan) ;; files to sync are placed on this channel + res-chan (async/chan) ;; results file sync (error/success) are placed on this channel + parallelism 8] + (if (or (empty? ledgers-info) + (empty? remote-sync-servers)) (go ::done) (let [remote-copy-fn (remote-copy-fn* conn remote-sync-servers 3000)] (check-all-ledgers-consistency conn ledgers-info sync-chan) From b69745e7d6064591ca75a3d73b87217131df5953 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Wed, 28 Feb 2024 15:18:40 -0500 Subject: [PATCH 02/12] fluree_start.sh remote repl option --- resources/fluree_start.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/resources/fluree_start.sh b/resources/fluree_start.sh index de2f8ac8..eb224941 100755 --- a/resources/fluree_start.sh +++ b/resources/fluree_start.sh @@ -21,6 +21,8 @@ FLUREE_SERVER="" FLUREE_PROPERTIES="" FLUREE_LOGBACK_CONFIGURATION_FILE="" +FLUREE_REMOTE_REPL_PORT="" + MINIMUM_JAVA_VERSION=11 function find_java() { @@ -179,6 +181,11 @@ while [ $# -gt 0 ]; do -Xms*) XMS=$1 ;; + -repl-port=*) + if [[ "$1" =~ ^-repl-port=([0-9]+) ]]; then + FLUREE_REMOTE_REPL_PORT=${BASH_REMATCH[1]} + fi + ;; test) break ;; @@ -217,6 +224,15 @@ echo "Fluree ledger starting with java options: ${XMS} ${XMX} ${JAVA_OPTS}" echo "fluree.db log level is ${FLUREE_DB_LOG_LEVEL}" echo "fluree.raft log level is ${FLUREE_RAFT_LOG_LEVEL}" -exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \ - -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \ - -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER} +if [[ $FLUREE_REMOTE_REPL_PORT ]] +then + echo "WARNING: Starting with remote REPL on port ${FLUREE_REMOTE_REPL_PORT}" + exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \ + -Dclojure.server.repl="{:port ${FLUREE_REMOTE_REPL_PORT} :accept clojure.core.server/repl}" \ + -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \ + -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER} +else + exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \ + -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \ + -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER} +fi From 563ba6cfcf4de862803d739978cfb2a38e346c15 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Wed, 28 Feb 2024 15:19:04 -0500 Subject: [PATCH 03/12] Store system in var for remote repl use --- src/fluree/db/server.clj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fluree/db/server.clj b/src/fluree/db/server.clj index 7b8f1eee..11230327 100644 --- a/src/fluree/db/server.clj +++ b/src/fluree/db/server.clj @@ -27,6 +27,8 @@ (set! *warn-on-reflection* true) +(def system nil) + ;; instantiates server operations (defn local-message-process @@ -205,6 +207,7 @@ (if-let [command (:fdb-command environ/env)] (execute-command command) (let [system (startup)] + (alter-var-root #'system (constantly system)) (.addShutdownHook (Runtime/getRuntime) (Thread. ^Runnable From bd43be1ade74d0925e1063b8b83a4f6f741a3a92 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Wed, 28 Feb 2024 15:19:38 -0500 Subject: [PATCH 04/12] Log less group state --- src/fluree/db/ledger/stats.clj | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/fluree/db/ledger/stats.clj b/src/fluree/db/ledger/stats.clj index a2c0d7be..af31f9cb 100644 --- a/src/fluree/db/ledger/stats.clj +++ b/src/fluree/db/ledger/stats.clj @@ -68,14 +68,29 @@ (assoc acc k v)) acc)) {} input) walk/keywordize-keys)) +(defn compact-group-state + "Report out a smaller version of the raft group state to save logging" + [{:keys [networks] :as group-state}] + (let [networks* (reduce-kv + (fn [acc network network-map] + (let [dbs* (reduce-kv (fn [acc dbid db-data] + (assoc acc dbid (dissoc db-data :indexes))) + {} (:dbs network-map)) + network-map* (assoc network-map :dbs dbs*)] + (assoc acc network network-map*))) + {} + networks)] + (-> group-state + (select-keys [:version :leases :_work]) + (assoc :networks networks*)))) + + (defn report-stats [system] (log/info "Memory: " (-> (memory-stats) (json/encode))) - (let [group-state (txproto/-local-state (:group system)) - state-report (-> group-state - (select-keys [:version :leases :_work :networks]))] - (log/info "Group state: " (json/encode state-report)) - (log/debug "Full group state: " (json/encode group-state)))) + (let [group-state (txproto/-local-state (:group system))] + (log/info "Group state: " (json/encode (compact-group-state group-state))) + (log/trace "Full group state: " (json/encode group-state)))) From 13bcbcaefb92d6519706a38602aae688c861d1d2 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Wed, 28 Feb 2024 15:20:37 -0500 Subject: [PATCH 05/12] Log out tons of detail on idx leaf writing issue --- src/fluree/db/ledger/indexing.clj | 76 +++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/src/fluree/db/ledger/indexing.clj b/src/fluree/db/ledger/indexing.clj index f60d2109..0af1870f 100644 --- a/src/fluree/db/ledger/indexing.clj +++ b/src/fluree/db/ledger/indexing.clj @@ -105,6 +105,23 @@ :else (recur (inc n) his*))))))) +(defn throw-write-leaf-condition + "In the case there is a leaf write exception, dump out as much data as possible to help diagnose. + + condition-str is overflow, underflow or standard" + [e condition-str network dbid block t idx-type base-id leaf-i flakes history + old-children new-children resolved overflow-data underflow-data] + (log/error e "Error creating leaf index segment from" + condition-str ".\n Data: " + (pr-str {:network network :dbid dbid :block block :t t + :idx-type idx-type :base-id base-id :leaf-i leaf-i + :flakes flakes :history history + :old-children old-children :new-children new-children + :resolved resolved + :overflow-data overflow-data + :underflow-data underflow-data})) + (throw e)) + (defn index-leaf "Given a node, idx-novelty, returns [ [nodes] skip n]. @@ -139,8 +156,16 @@ (first flakes)) base-id (str (util/random-uuid)) his-split (get-history-in-range history first-flake rhs' comparator) - id (UnresolvedNode {:conn conn :config config @@ -182,8 +207,18 @@ current-node-his (get-history-in-range history fflake rhs comparator) his-in-range (into current-node-his combine-his) flakes (into (:flakes resolved) (:flakes combine-leaf)) - id (UnresolvedNode {:conn conn :config config :dbid dbid :id id :leaf true @@ -266,14 +308,14 @@ child-n (count children) at-leaf? (:leaf (val (first children))) children* (loop [child-i (dec child-n) - rhs rhs + rhs* rhs children* (empty children)] (if (< child-i 0) ;; at end of result set children* (let [child (val (nth children child-i)) child-rhs (:rhs child) - _ (when-not (or (= (dec child-i) child-n) (= child-rhs rhs)) - (throw (ex-info (str "Something went wrong. Child-rhs does not equal rhs: " {:child-rhs child-rhs :rhs rhs}) + _ (when-not (or (= (dec child-i) child-n) (= child-rhs rhs*)) + (throw (ex-info (str "Something went wrong. Child-rhs does not equal rhs: " {:child-rhs child-rhs :rhs rhs*}) {:status 500 :error :db/unexpected-error}))) child-first (:first child) @@ -289,7 +331,21 @@ dirty? (or (seq novelty) (seq remove-preds?)) [new-nodes skip n] (if dirty? (if at-leaf? - ( Date: Wed, 28 Feb 2024 15:21:22 -0500 Subject: [PATCH 06/12] Ensure idx-history node writes if no-history preds --- src/fluree/db/ledger/indexing.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fluree/db/ledger/indexing.clj b/src/fluree/db/ledger/indexing.clj index 0af1870f..70bb6353 100644 --- a/src/fluree/db/ledger/indexing.clj +++ b/src/fluree/db/ledger/indexing.clj @@ -142,7 +142,8 @@ node-bytes (flake/size-bytes resolved-flakes) overflow? (overflow? node-bytes) underflow? (and (underflow? node-bytes) (not= 1 (count old-children))) - history ( Date: Wed, 28 Feb 2024 15:23:40 -0500 Subject: [PATCH 07/12] Update to use latest corresponding fluree/db --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index b90b2cac..ceab166c 100644 --- a/deps.edn +++ b/deps.edn @@ -2,7 +2,7 @@ org.clojure/data.xml {:mvn/version "0.2.0-alpha8"} com.fluree/alphabase {:mvn/version "3.3.0"} com.fluree/db {:git/url "https://github.com/fluree/db.git" - :sha "bbd33250e826bf51259c1a954dc00d1324fa4819"} + :sha "add56b63e276f55200f4e21d074531dbccc66765"} com.fluree/raft {:mvn/version "1.0.0-beta1"} com.fluree/crypto {:mvn/version "0.4.0"} From a2064f8c3a8de1f14a4977f2adac2e06ab0fd118 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Wed, 28 Feb 2024 15:31:11 -0500 Subject: [PATCH 08/12] Update version to 1.0.5-SNAPSHOT pending 1.0.6 release --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index ceab166c..5bc96c27 100644 --- a/deps.edn +++ b/deps.edn @@ -39,7 +39,7 @@ :aliases {:mvn/group-id com.fluree :mvn/artifact-id ledger - :mvn/version "1.0.5" + :mvn/version "1.0.5-SNAPSHOT" :dev {:extra-paths ["dev", "test"] From 2268e13a273a84acc0161c724f0f6205759aee7e Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Thu, 29 Feb 2024 06:20:49 -0500 Subject: [PATCH 09/12] Add data readers to assist in debugging --- src/data_readers.cljc | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/data_readers.cljc diff --git a/src/data_readers.cljc b/src/data_readers.cljc new file mode 100644 index 00000000..667ac2f7 --- /dev/null +++ b/src/data_readers.cljc @@ -0,0 +1,3 @@ +{Flake fluree.db.flake/parts->Flake + FdbIndexConfig fluree.db.index/map->IndexConfig + FlureeUnresolvedNode fluree.db.storage.core/map->UnresolvedNode} \ No newline at end of file From f7e47d2f4a754a025c57cb393316d014c2b505e2 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Thu, 29 Feb 2024 06:21:44 -0500 Subject: [PATCH 10/12] Remote debug instructions --- dev/remote_debug/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 dev/remote_debug/README.md diff --git a/dev/remote_debug/README.md b/dev/remote_debug/README.md new file mode 100644 index 00000000..c4ed5428 --- /dev/null +++ b/dev/remote_debug/README.md @@ -0,0 +1,10 @@ +# Working with remote socket REPL +Optionally, Fluree can be started with a Remote REPL socket open for debugging purpose. + +The ./fluree_start.sh script has an option that automatically enables this feature. + +When starting with ./fluree.start.sh, simply add the option: +```bash +./fluree_start.sh -repl-port=5555 +``` +This will start Fluree with a remote REPL socket open on port 5555. From 2f46a9c5c03d2b7023875e01861bc35bc8295cd6 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Wed, 3 Apr 2024 14:07:45 -0400 Subject: [PATCH 11/12] Bump fluree/raft to use auto-repair features --- deps.edn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 5bc96c27..77ade361 100644 --- a/deps.edn +++ b/deps.edn @@ -3,7 +3,8 @@ com.fluree/alphabase {:mvn/version "3.3.0"} com.fluree/db {:git/url "https://github.com/fluree/db.git" :sha "add56b63e276f55200f4e21d074531dbccc66765"} - com.fluree/raft {:mvn/version "1.0.0-beta1"} + com.fluree/raft {:git/url "https://github.com/fluree/raft.git" + :sha "904d915d636cdbc9eced518e5675eedf98410965"} ;; one commit after 1.0.0-beta2 com.fluree/crypto {:mvn/version "0.4.0"} ;; network comm From 7c6740c21d1d0ace0b05309be39ccc6c3ce539dc Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Wed, 3 Apr 2024 14:08:21 -0400 Subject: [PATCH 12/12] v1.0.6-RC1 --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 77ade361..471c5b09 100644 --- a/deps.edn +++ b/deps.edn @@ -40,7 +40,7 @@ :aliases {:mvn/group-id com.fluree :mvn/artifact-id ledger - :mvn/version "1.0.5-SNAPSHOT" + :mvn/version "1.0.6-RC1" :dev {:extra-paths ["dev", "test"]