diff --git a/deps.edn b/deps.edn index 62646266..bc7d1bcd 100644 --- a/deps.edn +++ b/deps.edn @@ -1,45 +1,45 @@ -{:deps {org.clojure/clojure {:mvn/version "1.10.3"} - org.clojure/data.xml {:mvn/version "0.2.0-alpha6"} - com.fluree/alphabase {:mvn/version "3.2.2"} - com.fluree/db {:git/url "https://github.com/fluree/db.git" - :sha "6b05ab1cd5ebb7d123bcaf8e903303998bf34474"} - com.fluree/raft {:mvn/version "1.0.0-beta1"} - com.fluree/crypto {:mvn/version "0.3.9"} - - ;; network comm - net.async/async {:mvn/version "0.1.1"} - - ;; Lucene - clucie/clucie {:mvn/version "0.4.2"} - - ;; AWS S3 API - com.cognitect.aws/api {:mvn/version "0.8.561"} - com.cognitect.aws/endpoints {:mvn/version "1.1.12.230"} - com.cognitect.aws/s3 {:mvn/version "822.2.1145.0"} - - ;; web server - http-kit/http-kit {:mvn/version "2.6.0"} - ring/ring-core {:mvn/version "1.9.5"} - ring-cors/ring-cors {:mvn/version "0.1.13"} - compojure/compojure {:mvn/version "1.7.0"} - - ;; logging - ch.qos.logback/logback-classic {:mvn/version "1.2.11"} - - ;; config - environ/environ {:git/url "https://github.com/cap10morgan/environ.git" - :sha "32682e865e8248d9df09643d6321ca4259fdbc19" - :deps/root "environ"} - - ;; retries - robert/bruce {:mvn/version "0.8.0"}} +{:deps {org.clojure/clojure {:mvn/version "1.10.3"} + org.clojure/data.xml {:mvn/version "0.2.0-alpha6"} + com.fluree/alphabase {:mvn/version "3.2.2"} + com.fluree/db {:git/url "https://github.com/fluree/db.git" + :sha "6b05ab1cd5ebb7d123bcaf8e903303998bf34474"} + com.fluree/raft {:mvn/version "1.0.0-beta1"} + com.fluree/crypto {:mvn/version "0.3.9"} + + ;; network comm + net.async/async {:mvn/version "0.1.1"} + + ;; Lucene + clucie/clucie {:mvn/version "0.4.2"} + + ;; AWS S3 API + com.cognitect.aws/api {:mvn/version "0.8.561"} + com.cognitect.aws/endpoints {:mvn/version "1.1.12.230"} + com.cognitect.aws/s3 {:mvn/version "822.2.1145.0"} + + ;; web server + http-kit/http-kit {:mvn/version "2.6.0"} + ring/ring-core {:mvn/version "1.9.5"} + ring-cors/ring-cors {:mvn/version "0.1.13"} + compojure/compojure {:mvn/version "1.7.0"} + + ;; logging + ch.qos.logback/logback-classic {:mvn/version "1.2.11"} + + ;; config + environ/environ {:git/url "https://github.com/cap10morgan/environ.git" + :sha "32682e865e8248d9df09643d6321ca4259fdbc19" + :deps/root "environ"} + + ;; retries + robert/bruce {:mvn/version "0.8.0"}} :paths ["src" "resources"] :aliases - {:mvn/group-id com.fluree + {:mvn/group-id com.fluree :mvn/artifact-id ledger - :mvn/version "1.0.4" + :mvn/version "1.0.4" :dev {:extra-paths ["dev", "test"] @@ -49,52 +49,52 @@ {:extra-paths ["test" "test-resources"] :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git" - :sha "a85b3b02765fb68684ab9ee4a8598eacf7e471d2"}} + :sha "a85b3b02765fb68684ab9ee4a8598eacf7e471d2"}} :exec-fn cognitect.test-runner.api/test} :jar {:replace-deps {com.github.seancorfield/depstar {:mvn/version "2.1.303"}} - :exec-fn hf.depstar/jar - :exec-args {:jar "target/fluree-ledger.jar" - :group-id :mvn/group-id - :artifact-id :mvn/artifact-id - :version :mvn/version - :sync-pom true}} + :exec-fn hf.depstar/jar + :exec-args {:jar "target/fluree-ledger.jar" + :group-id :mvn/group-id + :artifact-id :mvn/artifact-id + :version :mvn/version + :sync-pom true}} :uberjar {:replace-deps {com.github.seancorfield/depstar {:mvn/version "2.1.303"}} - :exec-fn hf.depstar/uberjar - :exec-args {:jar "target/fluree-ledger.standalone.jar" - :aot [fluree.db.server] - :main-class fluree.db.server - :group-id :mvn/group-id - :artifact-id :mvn/artifact-id - :version :mvn/version - :sync-pom true}} + :exec-fn hf.depstar/uberjar + :exec-args {:jar "target/fluree-ledger.standalone.jar" + :aot [fluree.db.server] + :main-class fluree.db.server + :group-id :mvn/group-id + :artifact-id :mvn/artifact-id + :version :mvn/version + :sync-pom true}} :native-image - {:main-opts ["-m" "clj.native-image" "fluree.db.server" - "-H:Name=fluree-ledger" "--no-fallback" - "-H:+ReportExceptionStackTraces" - ;; IncludeResources is pretty finicky. Lots of regexes I've tried don't - ;; work and the logging the docs say you can turn on doesn't work. - ;; So I'm just including everything for now. - WSM 2021/08/20 - "-H:IncludeResources=.*" - "--enable-url-protocols=http,https" - "--enable-all-security-services" - "--report-unsupported-elements-at-runtime" - "--initialize-at-build-time" - "--allow-incomplete-classpath" - "--install-exit-handlers" - - ;; Most of these initialize-at-run-time classes are from https://github.com/oracle/graal/issues/2050#issuecomment-797689154 - "--initialize-at-run-time=org.asynchttpclient.RequestBuilderBase,org.asynchttpclient.ntlm.NtlmEngine,io.netty.channel.kqueue.KQueue,io.netty.channel.kqueue.Native,io.netty.channel.kqueue.KQueueEventLoop,io.netty.channel.kqueue.KQueueEventArray,io.netty.util.internal.logging.Log4JLogger,io.netty.channel.epoll.Epoll,io.netty.channel.epoll.Native,io.netty.channel.epoll.EpollEventLoop,io.netty.channel.epoll.EpollEventArray,io.netty.channel.unix.Errors,io.netty.channel.unix.IovArray,io.netty.channel.unix.Limits,io.netty.channel.DefaultFileRegion,io.netty.handler.ssl.ReferenceCountedOpenSslContext,io.netty.handler.ssl.ReferenceCountedOpenSslEngine,io.netty.handler.ssl.JdkNpnApplicationProtocolNegotiator,io.netty.handler.ssl.JettyNpnSslEngine,io.netty.handler.ssl.ConscryptAlpnSslEngine,io.netty.handler.ssl.JettyAlpnSslEngine$ServerEngine,io.netty.handler.ssl.JettyAlpnSslEngine$ClientEngine,org.httpkit.client.ClientSslEngineFactory$SSLHolder,abracad.avro.ClojureData$Vars,org.apache.lucene.analysis.ja.dict.UnknownDictionary$SingletonHolder,org.apache.lucene.analysis.ja.dict.TokenInfoDictionary$SingletonHolder" - - ;; In theory this shouldn't be necessary w/ sufficient type hinting, but there's a bug in the go macro (I think) - ;; that causes type hinting to not always work correctly inside them and you can't access fields of Java - ;; types like fluree.db.flake.Flake b/c it will resort to reflection and that has to be configured under - ;; graalvm native-images. I haven't figured out a minimal reproduction yet though. In the meantime, this fixes it. - "-H:ReflectionConfigurationFiles=resources/native-image-config/reflect-config.json"] + {:main-opts ["-m" "clj.native-image" "fluree.db.server" + "-H:Name=fluree-ledger" "--no-fallback" + "-H:+ReportExceptionStackTraces" + ;; IncludeResources is pretty finicky. Lots of regexes I've tried don't + ;; work and the logging the docs say you can turn on doesn't work. + ;; So I'm just including everything for now. - WSM 2021/08/20 + "-H:IncludeResources=.*" + "--enable-url-protocols=http,https" + "--enable-all-security-services" + "--report-unsupported-elements-at-runtime" + "--initialize-at-build-time" + "--allow-incomplete-classpath" + "--install-exit-handlers" + + ;; Most of these initialize-at-run-time classes are from https://github.com/oracle/graal/issues/2050#issuecomment-797689154 + "--initialize-at-run-time=org.asynchttpclient.RequestBuilderBase,org.asynchttpclient.ntlm.NtlmEngine,io.netty.channel.kqueue.KQueue,io.netty.channel.kqueue.Native,io.netty.channel.kqueue.KQueueEventLoop,io.netty.channel.kqueue.KQueueEventArray,io.netty.util.internal.logging.Log4JLogger,io.netty.channel.epoll.Epoll,io.netty.channel.epoll.Native,io.netty.channel.epoll.EpollEventLoop,io.netty.channel.epoll.EpollEventArray,io.netty.channel.unix.Errors,io.netty.channel.unix.IovArray,io.netty.channel.unix.Limits,io.netty.channel.DefaultFileRegion,io.netty.handler.ssl.ReferenceCountedOpenSslContext,io.netty.handler.ssl.ReferenceCountedOpenSslEngine,io.netty.handler.ssl.JdkNpnApplicationProtocolNegotiator,io.netty.handler.ssl.JettyNpnSslEngine,io.netty.handler.ssl.ConscryptAlpnSslEngine,io.netty.handler.ssl.JettyAlpnSslEngine$ServerEngine,io.netty.handler.ssl.JettyAlpnSslEngine$ClientEngine,org.httpkit.client.ClientSslEngineFactory$SSLHolder,abracad.avro.ClojureData$Vars,org.apache.lucene.analysis.ja.dict.UnknownDictionary$SingletonHolder,org.apache.lucene.analysis.ja.dict.TokenInfoDictionary$SingletonHolder" + + ;; In theory this shouldn't be necessary w/ sufficient type hinting, but there's a bug in the go macro (I think) + ;; that causes type hinting to not always work correctly inside them and you can't access fields of Java + ;; types like fluree.db.flake.Flake b/c it will resort to reflection and that has to be configured under + ;; graalvm native-images. I haven't figured out a minimal reproduction yet though. In the meantime, this fixes it. + "-H:ReflectionConfigurationFiles=resources/native-image-config/reflect-config.json"] :jvm-opts ["-Dclojure.compiler.direct-linking=true"] :extra-deps {clj.native-image/clj.native-image {:git/url "https://github.com/taylorwood/clj.native-image.git" @@ -110,12 +110,12 @@ :eastwood {:extra-deps {jonase/eastwood {:mvn/version "1.2.4"}} - :main-opts ["-m" "eastwood.lint" {:source-paths ["src"] :test-paths ["test"]}]} + :main-opts ["-m" "eastwood.lint" {:source-paths ["src"] :test-paths ["test"]}]} :ancient {:extra-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}} - :main-opts ["-m" "antq.core" "--skip=github-action"]} + :main-opts ["-m" "antq.core" "--skip=github-action"]} :clj-kondo {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2022.06.22"}} - :main-opts ["-m" "clj-kondo.main" "--lint" "src" "--config" ".clj-kondo/config.edn"]}}} + :main-opts ["-m" "clj-kondo.main" "--lint" "src" "--config" ".clj-kondo/config.edn"]}}} diff --git a/src/fluree/db/peer/messages.clj b/src/fluree/db/peer/messages.clj index 7bffc5c1..f8adc9c4 100644 --- a/src/fluree/db/peer/messages.clj +++ b/src/fluree/db/peer/messages.clj @@ -390,7 +390,7 @@ (async/ {"content-type" "application/json"} - (:token opts) (assoc "Authorization" (str "Bearer " (:token opts)))) + (:token opts) (assoc "Authorization" (str "Bearer " (:token opts)))) :body (json/stringify body)})) ;; ENDPOINT TEST: /transact (deftest add-schema* (testing "Add schema" - (let [filename "../test/fluree/db/ledger/Resources/ChatAltVersion/schema.edn" - tx (edn/read-string (slurp (io/resource filename))) - schema-res @(http/post (str endpoint-url "transact") (standard-request tx)) - status (:status schema-res) - body (-> schema-res :body bs/to-string json/parse) - body-keys (keys body)] + (let [filename "../test/fluree/db/ledger/Resources/ChatAltVersion/schema.edn" + tx (edn/read-string (slurp (io/resource filename))) + schema-res @(http/post (str endpoint-url "transact") (standard-request tx)) + status (:status schema-res) + body (-> schema-res :body bs/to-string json/parse) + body-keys (keys body)] (is (= 200 status) (str "Response: " (pr-str schema-res))) @@ -178,13 +178,13 @@ (deftest query-collections-predicates-multiquery (testing "Querying all collections and predicates in multi-query" - (let [query {:coll {:select ["*"] :from "_collection"} - :pred {:select ["*"] :from "_predicate"}} - multi-res @(http/post (str endpoint-url "multi-query") (standard-request query)) - status (:status multi-res) - body (-> multi-res :body bs/to-string json/parse) - collections (into #{} (map #(:_collection/name %) (:coll body))) - predicates (into #{} (map #(:_predicate/name %) (:pred body)))] + (let [query {:coll {:select ["*"] :from "_collection"} + :pred {:select ["*"] :from "_predicate"}} + multi-res @(http/post (str endpoint-url "multi-query") (standard-request query)) + status (:status multi-res) + body (-> multi-res :body bs/to-string json/parse) + collections (into #{} (map #(:_collection/name %) (:coll body))) + predicates (into #{} (map #(:_predicate/name %) (:pred body)))] (is (= 200 status)) @@ -199,23 +199,23 @@ (deftest sign-multi-query (testing "sign multi-query where collections are not named in alphanumeric order" - (let [private-key (slurp "default-private-key.txt") - qry-str (str "{\"collections\":{\"select\":[\"*\"],\"from\":\"_collection\"},\n " - " \"predicates\":{\"select\":[\"*\"],\"from\":\"_predicate\"},\n " - " \"_setting\":{\"select\":[\"*\"],\"from\":\"_setting\"},\n " - " \"_rule\":{\"select\":[\"*\"],\"from\":\"_rule\"},\n " - " \"_role\":{\"select\":[\"*\"],\"from\":\"_role\"},\n " - " \"_user\":{\"select\":[\"*\"],\"from\":\"_user\"}\n }") - request {:headers {"content-type" "application/json"} - :body qry-str} - q-endpoint (str endpoint-url "multi-query") - signed-req (http-signatures/sign-request :post q-endpoint request private-key) - resp @(http/post q-endpoint signed-req) - status (:status resp) - body (some-> resp :body bs/to-string json/parse) - collections (into #{} (map #(:_collection/name %) (:collections body))) - predicates (into #{} (map #(:_predicate/name %) (:predicates body))) - roles (into #{} (map #(:_role/id %) (:_role body)))] + (let [private-key (slurp "default-private-key.txt") + qry-str (str "{\"collections\":{\"select\":[\"*\"],\"from\":\"_collection\"},\n " + " \"predicates\":{\"select\":[\"*\"],\"from\":\"_predicate\"},\n " + " \"_setting\":{\"select\":[\"*\"],\"from\":\"_setting\"},\n " + " \"_rule\":{\"select\":[\"*\"],\"from\":\"_rule\"},\n " + " \"_role\":{\"select\":[\"*\"],\"from\":\"_role\"},\n " + " \"_user\":{\"select\":[\"*\"],\"from\":\"_user\"}\n }") + request {:headers {"content-type" "application/json"} + :body qry-str} + q-endpoint (str endpoint-url "multi-query") + signed-req (http-signatures/sign-request :post q-endpoint request private-key) + resp @(http/post q-endpoint signed-req) + status (:status resp) + body (some-> resp :body bs/to-string json/parse) + collections (into #{} (map #(:_collection/name %) (:collections body))) + predicates (into #{} (map #(:_predicate/name %) (:predicates body))) + roles (into #{} (map #(:_role/id %) (:_role body)))] (is (= 200 status)) @@ -317,23 +317,23 @@ (deftest sign-all-collections-graphql (testing "sign a query for all collections through the graphql endpoint" - (let [private-key (slurp "default-private-key.txt") - graphql-str (str "{ graph { _collection " - "(sort: {predicate: \"name\", order: ASC})" - "{ _id name spec version doc}}}") - qry-str (json/stringify {:query graphql-str}) - request {:headers {"content-type" "application/json"} - :body qry-str} - q-endpoint (str endpoint-url "graphql") - signed-req (http-signatures/sign-request :post q-endpoint request private-key) - resp @(http/post q-endpoint signed-req) - body (-> resp :body bs/to-string json/parse) - collections (-> body :data :_collection) - collection-keys (reduce-kv - (fn [result _ collection] (->> collection keys (into result))) - #{} - collections) - collection-names (set (map :name collections))] + (let [private-key (slurp "default-private-key.txt") + graphql-str (str "{ graph { _collection " + "(sort: {predicate: \"name\", order: ASC})" + "{ _id name spec version doc}}}") + qry-str (json/stringify {:query graphql-str}) + request {:headers {"content-type" "application/json"} + :body qry-str} + q-endpoint (str endpoint-url "graphql") + signed-req (http-signatures/sign-request :post q-endpoint request private-key) + resp @(http/post q-endpoint signed-req) + body (-> resp :body bs/to-string json/parse) + collections (-> body :data :_collection) + collection-keys (reduce-kv + (fn [result _ collection] (->> collection keys (into result))) + #{} + collections) + collection-names (set (map :name collections))] ; Are the keys in the collections what we expect? (is (test/contains-many? collection-keys :_id :name :version :doc)) @@ -442,16 +442,16 @@ (deftest sign-sql-query (testing "sign a query for all collections through the sql endpoint" - (let [private-key (slurp "default-private-key.txt") - qry-str (json/stringify "SELECT * FROM _collection") - request {:headers {"content-type" "application/json"} - :body qry-str} - q-endpoint (str endpoint-url "sql") - signed-req (http-signatures/sign-request :post q-endpoint request private-key) - resp @(http/post q-endpoint signed-req) - status (:status resp) - body (some-> resp :body bs/to-string json/parse) - collections (into #{} (map #(:_collection/name %) body))] + (let [private-key (slurp "default-private-key.txt") + qry-str (json/stringify "SELECT * FROM _collection") + request {:headers {"content-type" "application/json"} + :body qry-str} + q-endpoint (str endpoint-url "sql") + signed-req (http-signatures/sign-request :post q-endpoint request private-key) + resp @(http/post q-endpoint signed-req) + status (:status resp) + body (some-> resp :body bs/to-string json/parse) + collections (into #{} (map #(:_collection/name %) body))] (is (= 200 status)) ; The keys in the response are -> :opts :body :headers :status @@ -541,7 +541,7 @@ :stringNotUnique (rand-str)}) persons) person-res @(http/post (str endpoint-url "transact") (standard-request person-tx)) person-body (-> person-res :body bs/to-string json/parse) - person-keys (keys person-body) + person-keys (keys person-body) flakes (:flakes person-body) tempids (:tempids person-body)] @@ -658,7 +658,7 @@ res @(http/post (str endpoint-url "command") (standard-request cmd-map)) body (-> res :body bs/to-string json/parse)] - + (is (= 200 (:status res))) (is (string? body)) @@ -707,12 +707,12 @@ ;; ENDPOINT TEST: signed /delete-ledger request (deftest delete-ledger-tests (testing "delete ledger - open api" - (let [network "deleteme" - db-id "one" - ledger-id (str network "/" db-id) - new-db-res @(http/post (str endpoint-url-short "new-ledger") - (standard-request {:db/id ledger-id})) - new-db-body (-> new-db-res :body bs/to-string json/parse)] + (let [network "deleteme" + db-id "one" + ledger-id (str network "/" db-id) + new-db-res @(http/post (str endpoint-url-short "new-ledger") + (standard-request {:db/id ledger-id})) + new-db-body (-> new-db-res :body bs/to-string json/parse)] (is (= 200 (:status new-db-res))) (is (string? new-db-body)) (is (= 64 (count new-db-body))) @@ -723,18 +723,18 @@ (is (= 200 (:status res))) (is (= ledger-id (:deleted body))))))) (testing "deprecated delete-db cmd still works" - (let [network "deleteme" - db-id "two" - ledger-id (str network "/" db-id) - new-db-res @(http/post (str endpoint-url-short "new-ledger") - (standard-request {:db/id ledger-id})) + (let [network "deleteme" + db-id "two" + ledger-id (str network "/" db-id) + new-db-res @(http/post (str endpoint-url-short "new-ledger") + (standard-request {:db/id ledger-id})) new-db-body (-> new-db-res :body bs/to-string json/parse)] (is (= 200 (:status new-db-res))) (is (string? new-db-body)) (is (= 64 (count new-db-body))) (when (wait-for-db network db-id 100) - (let [res @(http/post (str endpoint-url-short "delete-db") - (standard-request {:db/id ledger-id})) + (let [res @(http/post (str endpoint-url-short "delete-db") + (standard-request {:db/id ledger-id})) body (some-> res :body bs/to-string json/parse)] (is (= 200 (:status res))) (is (= ledger-id (:deleted body))))))))