diff --git a/deps.edn b/deps.edn index 38e4300f..471c5b09 100644 --- a/deps.edn +++ b/deps.edn @@ -1,8 +1,10 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.1"} org.clojure/data.xml {:mvn/version "0.2.0-alpha8"} com.fluree/alphabase {:mvn/version "3.3.0"} - com.fluree/db {:mvn/version "1.0.5"} - com.fluree/raft {:mvn/version "1.0.0-beta1"} + com.fluree/db {:git/url "https://github.com/fluree/db.git" + :sha "add56b63e276f55200f4e21d074531dbccc66765"} + com.fluree/raft {:git/url "https://github.com/fluree/raft.git" + :sha "904d915d636cdbc9eced518e5675eedf98410965"} ;; one commit after 1.0.0-beta2 com.fluree/crypto {:mvn/version "0.4.0"} ;; network comm @@ -38,7 +40,7 @@ :aliases {:mvn/group-id com.fluree :mvn/artifact-id ledger - :mvn/version "1.0.5" + :mvn/version "1.0.6-RC1" :dev {:extra-paths ["dev", "test"] diff --git a/dev/remote_debug/README.md b/dev/remote_debug/README.md new file mode 100644 index 00000000..c4ed5428 --- /dev/null +++ b/dev/remote_debug/README.md @@ -0,0 +1,10 @@ +# Working with remote socket REPL +Optionally, Fluree can be started with a Remote REPL socket open for debugging purpose. + +The ./fluree_start.sh script has an option that automatically enables this feature. + +When starting with ./fluree.start.sh, simply add the option: +```bash +./fluree_start.sh -repl-port=5555 +``` +This will start Fluree with a remote REPL socket open on port 5555. diff --git a/resources/fluree_start.sh b/resources/fluree_start.sh index de2f8ac8..eb224941 100755 --- a/resources/fluree_start.sh +++ b/resources/fluree_start.sh @@ -21,6 +21,8 @@ FLUREE_SERVER="" FLUREE_PROPERTIES="" FLUREE_LOGBACK_CONFIGURATION_FILE="" +FLUREE_REMOTE_REPL_PORT="" + MINIMUM_JAVA_VERSION=11 function find_java() { @@ -179,6 +181,11 @@ while [ $# -gt 0 ]; do -Xms*) XMS=$1 ;; + -repl-port=*) + if [[ "$1" =~ ^-repl-port=([0-9]+) ]]; then + FLUREE_REMOTE_REPL_PORT=${BASH_REMATCH[1]} + fi + ;; test) break ;; @@ -217,6 +224,15 @@ echo "Fluree ledger starting with java options: ${XMS} ${XMX} ${JAVA_OPTS}" echo "fluree.db log level is ${FLUREE_DB_LOG_LEVEL}" echo "fluree.raft log level is ${FLUREE_RAFT_LOG_LEVEL}" -exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \ - -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \ - -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER} +if [[ $FLUREE_REMOTE_REPL_PORT ]] +then + echo "WARNING: Starting with remote REPL on port ${FLUREE_REMOTE_REPL_PORT}" + exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \ + -Dclojure.server.repl="{:port ${FLUREE_REMOTE_REPL_PORT} :accept clojure.core.server/repl}" \ + -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \ + -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER} +else + exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \ + -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \ + -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER} +fi diff --git a/src/data_readers.cljc b/src/data_readers.cljc new file mode 100644 index 00000000..667ac2f7 --- /dev/null +++ b/src/data_readers.cljc @@ -0,0 +1,3 @@ +{Flake fluree.db.flake/parts->Flake + FdbIndexConfig fluree.db.index/map->IndexConfig + FlureeUnresolvedNode fluree.db.storage.core/map->UnresolvedNode} \ No newline at end of file diff --git a/src/fluree/db/ledger/consensus/dbsync2.clj b/src/fluree/db/ledger/consensus/dbsync2.clj index be3d6aae..8377c0ab 100644 --- a/src/fluree/db/ledger/consensus/dbsync2.clj +++ b/src/fluree/db/ledger/consensus/dbsync2.clj @@ -436,10 +436,11 @@ (defn consistency-full-check [conn ledgers-info remote-sync-servers] - (let [sync-chan (async/chan) ;; files to sync are placed on this channel - res-chan (async/chan) ;; results file sync (error/success) are placed on this channel - parallelism 8] - (if (empty? ledgers-info) + (let [sync-chan (async/chan) ;; files to sync are placed on this channel + res-chan (async/chan) ;; results file sync (error/success) are placed on this channel + parallelism 8] + (if (or (empty? ledgers-info) + (empty? remote-sync-servers)) (go ::done) (let [remote-copy-fn (remote-copy-fn* conn remote-sync-servers 3000)] (check-all-ledgers-consistency conn ledgers-info sync-chan) diff --git a/src/fluree/db/ledger/indexing.clj b/src/fluree/db/ledger/indexing.clj index f60d2109..70bb6353 100644 --- a/src/fluree/db/ledger/indexing.clj +++ b/src/fluree/db/ledger/indexing.clj @@ -105,6 +105,23 @@ :else (recur (inc n) his*))))))) +(defn throw-write-leaf-condition + "In the case there is a leaf write exception, dump out as much data as possible to help diagnose. + + condition-str is overflow, underflow or standard" + [e condition-str network dbid block t idx-type base-id leaf-i flakes history + old-children new-children resolved overflow-data underflow-data] + (log/error e "Error creating leaf index segment from" + condition-str ".\n Data: " + (pr-str {:network network :dbid dbid :block block :t t + :idx-type idx-type :base-id base-id :leaf-i leaf-i + :flakes flakes :history history + :old-children old-children :new-children new-children + :resolved resolved + :overflow-data overflow-data + :underflow-data underflow-data})) + (throw e)) + (defn index-leaf "Given a node, idx-novelty, returns [ [nodes] skip n]. @@ -125,7 +142,8 @@ node-bytes (flake/size-bytes resolved-flakes) overflow? (overflow? node-bytes) underflow? (and (underflow? node-bytes) (not= 1 (count old-children))) - history (UnresolvedNode {:conn conn :config config @@ -182,8 +208,18 @@ current-node-his (get-history-in-range history fflake rhs comparator) his-in-range (into current-node-his combine-his) flakes (into (:flakes resolved) (:flakes combine-leaf)) - id (UnresolvedNode {:conn conn :config config :dbid dbid :id id :leaf true @@ -266,14 +309,14 @@ child-n (count children) at-leaf? (:leaf (val (first children))) children* (loop [child-i (dec child-n) - rhs rhs + rhs* rhs children* (empty children)] (if (< child-i 0) ;; at end of result set children* (let [child (val (nth children child-i)) child-rhs (:rhs child) - _ (when-not (or (= (dec child-i) child-n) (= child-rhs rhs)) - (throw (ex-info (str "Something went wrong. Child-rhs does not equal rhs: " {:child-rhs child-rhs :rhs rhs}) + _ (when-not (or (= (dec child-i) child-n) (= child-rhs rhs*)) + (throw (ex-info (str "Something went wrong. Child-rhs does not equal rhs: " {:child-rhs child-rhs :rhs rhs*}) {:status 500 :error :db/unexpected-error}))) child-first (:first child) @@ -289,7 +332,21 @@ dirty? (or (seq novelty) (seq remove-preds?)) [new-nodes skip n] (if dirty? (if at-leaf? - ( group-state + (select-keys [:version :leases :_work]) + (assoc :networks networks*)))) + + (defn report-stats [system] (log/info "Memory: " (-> (memory-stats) (json/encode))) - (let [group-state (txproto/-local-state (:group system)) - state-report (-> group-state - (select-keys [:version :leases :_work :networks]))] - (log/info "Group state: " (json/encode state-report)) - (log/debug "Full group state: " (json/encode group-state)))) + (let [group-state (txproto/-local-state (:group system))] + (log/info "Group state: " (json/encode (compact-group-state group-state))) + (log/trace "Full group state: " (json/encode group-state)))) diff --git a/src/fluree/db/server.clj b/src/fluree/db/server.clj index 7b8f1eee..11230327 100644 --- a/src/fluree/db/server.clj +++ b/src/fluree/db/server.clj @@ -27,6 +27,8 @@ (set! *warn-on-reflection* true) +(def system nil) + ;; instantiates server operations (defn local-message-process @@ -205,6 +207,7 @@ (if-let [command (:fdb-command environ/env)] (execute-command command) (let [system (startup)] + (alter-var-root #'system (constantly system)) (.addShutdownHook (Runtime/getRuntime) (Thread. ^Runnable