diff --git a/CHANGELOG.md b/CHANGELOG.md index 95555ce86..a7035a2e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Added the `-p`/`--include-path` CLI command to prepend entries to the `sys.path` as an alternative to `PYTHONPATH` (#1027) * Added an empty entry to `sys.path` for all CLI entrypoints (`basilisp run`, `basilisp repl`, etc.) (#1027) + * Added test runner to `basilisp.test` #980 + * Added test ops to nrepl #980 ### Changed * The compiler will no longer require `Var` indirection for top-level `do` forms unless those forms specify `^:use-var-indirection` metadata (which currently is only used in the `ns` macro) (#1034) diff --git a/src/basilisp/contrib/nrepl_server.lpy b/src/basilisp/contrib/nrepl_server.lpy index 99bd50f3d..b1bb0dba9 100644 --- a/src/basilisp/contrib/nrepl_server.lpy +++ b/src/basilisp/contrib/nrepl_server.lpy @@ -3,7 +3,8 @@ (ns basilisp.contrib.nrepl-server "A port of `nbb `_ 's nREPL server implementation to Basilisp." (:require [basilisp.contrib.bencode :as bc] - [basilisp.string :as str]) + [basilisp.string :as str] + [basilisp.contrib.nrepl-server.test :as test]) (:import basilisp.logconfig logging socketserver @@ -330,7 +331,9 @@ :complete handle-complete ;; :macroexpand handle-macroexpand ;; :classpath handle-classpath - }) + :test test/handle-test + :test-all test/handle-test-all + :retest test/handle-retest}) (defn- handle-request [{:keys [op] :as request} send-fn] (if-let [op-fn (get ops op)] diff --git a/src/basilisp/contrib/nrepl_server/test.lpy b/src/basilisp/contrib/nrepl_server/test.lpy new file mode 100644 index 000000000..2f11783a7 --- /dev/null +++ b/src/basilisp/contrib/nrepl_server/test.lpy @@ -0,0 +1,190 @@ +(ns basilisp.contrib.nrepl-server.test + (:import time) + (:require [basilisp.stacktrace :refer [print-stack-trace]] + [basilisp.test :as test] + [basilisp.set :as set])) + +(defonce current-report + (atom nil)) + +(defn- now + [] + (time/time)) + +(defrecord Report [fail-fast? failed? tests namespaces start end] + test/Report + + (continue? [self] + (not (and fail-fast? failed?))) + + (report-begin [self] + (assoc-in self [:start nil] (now))) + (report-end [self] + (assoc-in self [:end nil] (now))) + + (report-namespace-begin [self ns] + (assoc-in self [:start ns] (now))) + (report-namespace-end [self ns assertions] + (-> self + (assoc :failed? (boolean (or failed? (test/failures assertions)))) + (assoc-in [:namespaces ns] assertions) + (assoc-in [:end ns] (now)))) + + (report-test-begin [self test-var] + (assoc-in self [:start test-var] (now))) + (report-test-end [self test-var assertions] + (-> self + (assoc :failed? (boolean (or failed? (test/failures assertions)))) + (assoc-in [:tests test-var] assertions) + (assoc-in [:end test-var] (now))))) + +(defn- make-result + [assertions] + (vec + (map-indexed (fn [i {:as assertion + :keys [ns var test-section message type + actual expected expr line]}] + (let [error? (= type :error) + fail? (#{:error :failure} type) + file (-> var meta :file)] + (cond-> {:index i + :context test-section + :message message + :type (case type + :failure "fail" + (name type)) + :var (if var (-> var name str) "unknown") + :ns (if ns (name ns) "unknown")} + error? (assoc :fault "true" + :error + (when (instance? BaseException + actual) + (with-out-str + (print-stack-trace actual)))) + (and line fail?) (assoc :line line) + (and file fail?) (assoc :file file) + expr (assoc :expr (pr-str expr)) + fail? (assoc :actual (str (pr-str actual) + \newline) + :expected (str (pr-str expected) + \newline)) + :always (update-keys name)))) + assertions))) + +(defn- make-elapsed-time + [start end] + (let [ms (python/int (* (- end start) 1000))] + {"ms" ms + "humanized" (format "Completed in %d ms" ms)})) + +(defn- var-path + [var] + [(-> var namespace name) (-> var name str)]) + +(defn- make-response + [{:as report :keys [tests namespaces start end]}] + (let [elapsed-time #(make-elapsed-time (get start %) (get end %))] + {"testing-ns" (str (or (some-> namespaces first key) + (some-> tests first key namespace))) + "results" (->> (update-keys namespaces + #(symbol (str %) "unknown")) + (filter (comp test/failures val)) + (concat tests) + (map (juxt (comp var-path key) + (comp make-result val))) + (reduce #(apply assoc-in %1 %2) {})) + "summary" (-> (reduce (let [+count (fnil + 0)] + (fn [summary [k assertions]] + (->> assertions + (map :type) + frequencies + (merge-with +count summary)))) + (zipmap [:pass :failure :error] + (repeat 0)) + (concat tests namespaces)) + (set/rename-keys {:failure :fail}) + (assoc :ns (count namespaces) + :var (count tests) + :test (->> (vals tests) + (concat (vals namespaces)) + (transduce (map count) +))) + (update-keys name)) + "var-elapsed-time" (reduce (fn [acc v] + (assoc-in acc + (var-path v) + {"elapsed-time" (elapsed-time v)})) + {} + (keys tests)) + "ns-elapsed-time" (into {} + (map (juxt name elapsed-time)) + (keys namespaces)) + "elapsed-time" (elapsed-time nil)})) + +(defn- make-report! + [request vars] + (let [{:keys [fail-fast + include + exclude]} request + include? (if (seq include) + (apply some-fn (map keyword include)) + (constantly true)) + exclude? (if (seq exclude) + (apply some-fn (map keyword exclude)) + (constantly false))] + (->> vars + (filter (comp (every-pred ::test/test include? (complement exclude?)) + meta)) + (test/compile-report (map->Report {:fail-fast? (= fail-fast "true")})) + (reset! current-report)))) + +(defmacro ^:private print-errors + [& body] + `(try + ~@body + (catch python/Exception ~'e + (print-stack-trace ~'e) + (throw ~'e)))) + +(defn handle-test + "Handle \"test\" nrepl command. Run specified tests or all tests in + specified namespace. Tests must be loaded." + [request send-fn] + (print-errors + (let [{:keys [ns tests]} request] + (->> (if (seq tests) + (keep (comp resolve (partial symbol ns)) tests) + (some-> ns symbol find-ns ns-publics vals)) + (make-report! request) + make-response + (send-fn request))))) + +(defn handle-test-all + "Handle \"test-all\" nrepl command. Run all tests in all loaded + namespaces. Unable to load additional namespaces." + [request send-fn] + (print-errors + (->> (all-ns) + (mapcat (comp vals ns-publics)) + (make-report! request) + make-response + (send-fn request)))) + +(defn- failing-tests + [{:as report + :keys [tests namespaces]}] + (let [ns-fail? (comp (memoize #(test/failures (get namespaces %))) + namespace)] + (keep (fn [[test-var assertions]] + (when (or (ns-fail? test-var) + (test/failures assertions)) + test-var)) + tests))) + +(defn handle-retest + "Handle \"retest\" nrepl command. Re-run any previously failing tests." + [request send-fn] + (print-errors + (->> (failing-tests @current-report) + (make-report! request) + make-response + (send-fn request)))) diff --git a/src/basilisp/test.lpy b/src/basilisp/test.lpy index 4379d0161..426f88254 100644 --- a/src/basilisp/test.lpy +++ b/src/basilisp/test.lpy @@ -19,12 +19,102 @@ Tests may take advantage of Basilisp fixtures via :lpy:fn:`use-fixtures` to perform setup and teardown functions for each test or namespace. Fixtures are not the same (nor are they compatible with) PyTest fixtures." + (:import types) (:require - [basilisp.template :as template])) - -(def ^:dynamic *test-name* nil) -(def ^:dynamic *test-section* nil) -(def ^:dynamic *test-failures* nil) + [basilisp.template :as template] + [basilisp.stacktrace :refer [print-stack-trace]])) + +(def ^:dynamic *test-ns* nil) +(def ^:dynamic *test-var* nil) +(def ^:dynamic *test-name* nil) +(def ^:dynamic *test-section* nil) +(def ^:dynamic *test-assertions* nil) + +(def ^{:deprecated true + :dynamic true} + *test-failures* + "Deprecated. Use :lpy:var:`*test-assertions*` instead." + nil) + +(defprotocol Report + + (continue? [self] + "Return true if more tests should be processed, else false. Will + be called after each test to determine if the test runner should + continue.") + + (report-begin [self] + "Record the start of this report. Return the modified report.") + (report-end [self] + "Record the end of this report. Return the modified report.") + + (report-namespace-begin [self ns] + "Record the current namespace being tested. Return the modified report.") + (report-namespace-end [self ns assertions] + "Record the end of testing the namespace. Return the modified report.") + + (report-test-begin [self test-var] + "Record the start of a test. Return the modified report.") + (report-test-end [self test-var assertions] + "Record the result of a test. Return the modified report.")) + +(defn failures + "Get all assertions that represent either a failure or an + error. return ``nil`` if there are none." + [assertions] + (seq (filter (comp #{:failure :error} :type) assertions))) + +(defn- print-failures + [ns test-var assertions] + (when-let [fails (failures assertions)] + (let [file (-> test-var meta (:file (name ns)))] + (doseq [fail fails] + (let [{:keys [type expected actual + message test-section line]} fail + error? (= :error type)] + (println (format "%s in (%s) (%s)" + (if error? "ERROR" "FAIL") + (if test-var (name test-var) "unknown") + (cond-> file + line (str ":" line)))) + (when test-section (println test-section)) + (when message (println message)) + (if error? + (when (instance? BaseException actual) + (print-stack-trace actual) + (println)) + (println "expected:" expected "\n actual:" actual \newline))))))) + +(defrecord SimpleReport [pass failure error tests] + Report + + (continue? [self] true) + + (report-begin [self] self) + (report-end [self] + (println + (format "\nRan %d tests containing %d assertions.\n%d failures, %d errors." + tests + (+ pass failure) + failure + error)) + self) + + (report-namespace-begin [self ns] + (println "\nTesting " ns "\n") + self) + (report-namespace-end [self ns assertions] + (print-failures ns nil assertions) + (reduce-kv #(update %1 %2 + %3) + self + (->> assertions (map :type) frequencies))) + + (report-test-begin [self _] self) + (report-test-end [self test-var assertions] + (print-failures (namespace test-var) test-var assertions) + (reduce-kv #(update %1 %2 + %3) + (update self :tests inc) + (->> assertions (map :type) frequencies)))) (defmulti ^{:arglists '([fixture-type & fixtures])} use-fixtures @@ -67,6 +157,91 @@ [_ & fixtures] (alter-meta! *ns* assoc ::once-fixtures fixtures)) +(defn generator? + "Return true if ``x`` is a generator type, else false." + [x] + (instance? types/GeneratorType x)) + +(defmacro with-fixtures + "Wrap the ``body`` in the ``fixtures`` in the given order. Handle + setup and teardown for each of the ``fixtures``." + [fixtures & body] + (assert (vector? fixtures) "Expected a literal vector of fixtures") + (let [result (gensym "result")] + (reduce (fn [form fixture] + `(let [~result (~fixture)] + (if (generator? ~result) + (try + (python/next ~result) + ~form + (finally + (try + (python/next ~result) + (catch python/StopIteration ~'_ nil)))) + ~form))) + `(do ~@body) + (reverse fixtures)))) + +(defmacro compose-fixtures + "Compose any number of ``fixtures``, in order, creating a new fixture + that combines their behavior. Always returns a valid fixture + function, even if no fixtures are given." + [& fixtures] + `(fn [] (with-fixtures [~@fixtures] (yield)))) + +(defn join-fixtures + "Composes a collection of ``fixtures``, in order. Always returns a valid + fixture function, even if the collection is empty. + + Prefer :lpy:macro`compose-fixtures` if fixtures are known at compile time." + [fixtures] + (if (seq fixtures) + (reduce #(compose-fixtures %1 %2) fixtures) + (constantly nil))) + +(defn assert! + "Emit an assertion. + + Use as a return value for :lpy:fn`gen-assert` methods." + [type expr msg line-num additional-data] + (vswap! *test-assertions* conj + (assoc additional-data + :ns *test-ns* + :var *test-var* + :test-name *test-name* + :test-section *test-section* + :expr expr + :message msg + :line line-num + :type type))) + +(defn pass! + "Emit an assertion that indicates a test criteria has pass. + + Use as a return value for :lpy:fn`gen-assert` methods." + [expr msg line-num & {:as additional-data}] + (assert! :pass expr msg line-num additional-data)) + +(defn fail! + "Emit an assertion that indicates a test failure. + + Use as a return value for :lpy:fn`gen-assert` methods." + [expr msg line-num expected actual & {:as additional-data}] + (assert! :failure expr msg line-num + (assoc additional-data + :actual actual + :expected expected))) + +(defn error! + "Emit an assertion that indicates an error ``e`` has occurred. + + Use as a return value for :lpy:fn`gen-assert` methods." + [expr msg line-num e & {:as additional-data}] + (assert! :error expr (str msg " " (python/repr e)) line-num + (assoc additional-data + :actual e + :expected (:expected additional-data expr)))) + (defmulti ^{:arglists '([expr msg line-num])} gen-assert @@ -88,46 +263,32 @@ [expr msg line-num] `(let [actual# ~(nth expr 2) expected# ~(second expr)] - (when-not (= expected# actual#) - (vswap! *test-failures* - conj - {:test-name *test-name* - :test-section *test-section* - :message ~msg - :expr (quote ~expr) - :actual actual# - :expected expected# - :line ~line-num - :type :failure})))) + (if (= expected# actual#) + (pass! (quote ~expr) ~msg ~line-num) + (fail! (quote ~expr) ~msg ~line-num expected# actual#)))) + +(defmethod gen-assert 'instance? + [expr msg line-num] + `(let [value# ~(nth expr 2) + cls# ~(second expr)] + (if (instance? cls# value#) + (pass! (quote ~expr) ~msg ~line-num) + (fail! (quote ~expr) ~msg ~line-num cls# (list '~'type value#))))) (defmethod gen-assert 'thrown? [expr msg line-num] (let [exc-type (second expr) body (nthnext expr 2)] `(try - (let [result# (do ~@body)] - (vswap! *test-failures* - conj - {:test-name *test-name* - :test-section *test-section* - :message ~msg - :expr (quote ~expr) - :actual result# - :expected (quote ~exc-type) - :line ~line-num - :type :failure})) - (catch ~exc-type _ nil) + (fail! (quote ~expr) ~msg ~line-num (quote ~exc-type) (do ~@body)) + (catch ~exc-type _ + (pass! (quote ~expr) ~msg ~line-num)) (catch python/Exception e# - (vswap! *test-failures* - conj - {:test-name *test-name* - :test-section *test-section* - :message (str "Expected " ~exc-type "; got " (python/type e#) " instead") - :expr (quote ~expr) - :actual e# - :expected ~exc-type - :line ~line-num - :type :failure}))))) + (fail! (quote ~expr) + (str "Expected " ~exc-type "; got " (python/type e#) " instead") + ~line-num + ~exc-type + e#))))) (defmethod gen-assert 'thrown-with-msg? [expr msg line-num] @@ -135,58 +296,31 @@ pattern (nth expr 2) body (nthnext expr 3)] `(try - (let [result# (do ~@body)] - (vswap! *test-failures* - conj - {:test-name *test-name* - :test-section *test-section* - :message ~msg - :expr (quote ~expr) - :actual result# - :expected (quote ~exc-type) - :line ~line-num - :type :failure})) + (fail! (quote ~expr) ~msg ~line-num (quote ~exc-type) (do ~@body)) (catch ~exc-type e# ;; Use python/str rather than Basilisp str to get the raw "message" ;; from the exception. (let [string-exc# (python/str e#)] - (when-not (re-find ~pattern string-exc#) - (vswap! *test-failures* - conj - {:test-name *test-name* - :test-section *test-section* - :message "Regex pattern did not match" - :expr (quote ~expr) - :actual string-exc# - :expected ~pattern - :line ~line-num - :type :failure})))) + (if (re-find ~pattern string-exc#) + (pass! (quote ~expr) ~msg ~line-num) + (fail! (quote ~expr) + "Regex pattern did not match" + ~line-num + ~pattern + string-exc#)))) (catch python/Exception e# - (vswap! *test-failures* - conj - {:test-name *test-name* - :test-section *test-section* - :message (str "Expected " ~exc-type "; got " (python/type e#) " instead") - :expr (quote ~expr) - :actual e# - :expected ~exc-type - :line ~line-num - :type :failure}))))) + (fail! (quote ~expr) + (str "Expected " ~exc-type "; got " (python/type e#) " instead") + ~line-num + ~exc-type + e#))))) (defmethod gen-assert :default [expr msg line-num] `(let [computed# ~expr] - (when-not computed# - (vswap! *test-failures* - conj - {:test-name *test-name* - :test-section *test-section* - :message ~msg - :expr (quote ~expr) - :actual computed# - :expected (list (symbol "not") computed#) - :line ~line-num - :type :failure})))) + (if computed# + (pass! (quote ~expr) ~msg ~line-num) + (fail! (quote ~expr) ~msg ~line-num (list '~'not computed#) computed#)))) (defmacro is "Assert that a test condition, ``expr``, is true. :lpy:fn:`is` assertion failures are @@ -217,18 +351,13 @@ `(try ~(gen-assert expr msg line-num) (catch python/Exception ~exc-name - (vswap! *test-failures* - conj - {:test-name *test-name* - :test-section *test-section* - :message (str "Unexpected exception thrown during test run: " (python/repr ~exc-name)) - :expr (quote ~expr) - :actual ~exc-name - :expected (quote ~expr) - :line ~(if line-num - line-num - `(.-tb_lineno (.-__traceback__ ~exc-name))) - :type :error})))))) + (error! (quote ~expr) + "Unexpected exception thrown during test run:" + ~(if line-num + line-num + `(.-tb_lineno + (.-__traceback__ ~exc-name))) + ~exc-name)))))) (defmacro are "Generate assertions using the template expression ``expr``. Template expressions @@ -276,6 +405,31 @@ ~msg)] ~@body)) +(defn execute* + "Execute the test var ``v`` with the given ``fixture`` if any. Return + a list of test assertions." + ([test-var] + (execute* (constantly nil) test-var)) + ([fixture test-var] + (binding [*ns* (namespace test-var) + *test-ns* (namespace test-var) + *test-var* test-var + *test-name* (-> test-var name str) + *test-section* nil + *test-assertions* (volatile! nil) + *test-failures* *test-assertions*] + (let [{testf :test + line :line} (meta test-var)] + (try + (with-fixtures [fixture] + (try + (testf) + (catch Exception e + (error! nil "Exception in test:" line e)))) + (catch Exception e + (error! nil "Exception in test fixture:" line e))) + (reverse @*test-assertions*))))) + (defmacro deftest "Define a new test. @@ -284,15 +438,91 @@ Tests defined by `deftest` will be run by default by the PyTest test runner using Basilisp's builtin PyTest hook." - [name-sym & body] - (let [test-name-sym (vary-meta name-sym assoc ::test true) - test-name-str (name test-name-sym) - test-ns-name `(quote ~(symbol (name *ns*)))] - `(defn ~test-name-sym - [] - (binding [*ns* (the-ns ~test-ns-name) - *test-name* ~test-name-str - *test-section* nil - *test-failures* (volatile! [])] - ~@body - {:failures (deref *test-failures*)})))) + [test-name & body] + `(defn ~(vary-meta test-name assoc + ::test true + ;; This key is used by some tools (emacs-cider) to + ;; identify test vars. + :test `(fn ~test-name [] ~@body)) + [] + {:failures (failures (execute* (var ~test-name)))})) + +(defn- report-test + [report fixtures test-var] + (-> report + (report-test-begin test-var) + (report-test-end test-var (execute* fixtures test-var)))) + +(defn- report-namespace + [report ns test-vars] + (let [report (report-namespace-begin report ns) + once-fixture (-> ns meta ::once-fixtures join-fixtures) + each-fixture (-> ns meta ::each-fixtures join-fixtures)] + (try + (report-namespace-end (with-fixtures [once-fixture] + (reduce (fn [report test-var] + (if (continue? report) + (report-test report each-fixture + test-var) + (reduced report))) + report + test-vars)) + ns + nil) + (catch Exception e + (report-namespace-end report + ns + [{:test-name nil + :test-section nil + :ns ns + :var nil + :expr nil + :line nil + :type :error + :expected nil + :actual e + :message + (str "Exception in namespace fixture: " + (python/repr e))}]))))) + +(defn collect-tests + "Get all tests from the given vars and/or namespaces. If none are + given then default to the tests current namespace. Return a mapping + of namespaces to test vars." + [vars-or-namespaces] + (->> (or (seq vars-or-namespaces) + [*ns*]) + (mapcat #(if (var? %) (list %) (-> % ns-interns vals))) + (filter (comp ::test meta)) + (group-by namespace))) + +(defn compile-report + "Build a report by running all tests from the given vars and/or + namespaces. If none are given then default to the tests current + namespace." + [report vars-or-namespaces] + (report-end + (reduce (fn [report [ns test-vars]] + (if (continue? report) + (report-namespace report ns test-vars) + (reduced report))) + (report-begin report) + (collect-tests vars-or-namespaces)))) + +(defn run-tests + "Run all tests from the given vars and/or namespaces. If none are + given then default to the tests current namespace. Print failures to + standard output." + [& vars-or-namespaces] + (into {} (compile-report (->SimpleReport 0 0 0 0) vars-or-namespaces))) + +(defn run-all-tests + "Run all loaded tests. Print failures to standard output. Optionally + filter tests by regex ``re``." + ([] + (apply run-tests (all-ns))) + ([re] + (->> (all-ns) + (mapcat (comp vals ns-interns)) + (filter #(re-find re (subs (str %) 2))) + (apply run-tests)))) diff --git a/tests/basilisp/contrib/nrepl_server_test.lpy b/tests/basilisp/contrib/nrepl_server_test.lpy index 33a7e366c..bffd19187 100644 --- a/tests/basilisp/contrib/nrepl_server_test.lpy +++ b/tests/basilisp/contrib/nrepl_server_test.lpy @@ -5,7 +5,7 @@ [basilisp.contrib.nrepl-server :as nr] [basilisp.set :as set] [basilisp.string :as str :refer [starts-with?]] - [basilisp.test :refer [deftest are is testing use-fixtures]] + [basilisp.test :refer [deftest are is testing use-fixtures with-fixtures]] [basilisp.test.fixtures :as fixtures :refer [*tempdir*]]) (:import os @@ -14,6 +14,33 @@ threading time)) +(defn- generated-tests-fixture + [] + (with-fixtures [fixtures/reset-path + fixtures/tempdir] + (.insert sys/path 0 fixtures/*tempdir*) + (let [file (.join os/path fixtures/*tempdir* + "nrepl_server_test_generated/test1.lpy")] + (os/makedirs (.dirname os/path file) ** :exist_ok true) + (with-open [wtr (python/open file ** :mode "w")] + (.write wtr + "(ns nrepl-server-test-generated.test1 + (:require [basilisp.test :refer :all])) + + (deftest error-test + (throw (ex-info \"Boom!\" {}))) + + (deftest fail-test + (is (= 0 1))) + + (deftest pass-test + (testing \"section\" + (is (= 1 1))) + (is (thrown? Exception (throw (ex-info \"Boom!\" {})))))"))) + (yield))) + +(use-fixtures :once generated-tests-fixture) + (use-fixtures :each fixtures/tempdir) (def ^:dynamic *nrepl-port* @@ -112,7 +139,10 @@ (client-send! client{:id 2 :op "describe"}) (let [{:keys [ops versions status]} (client-recv! client)] (is (= ["done"] status)) - (is (= {:clone {} :close {} :complete {} :describe {} :eldoc {} :eval {} :info {} :load-file {}} ops)) + (is (= {:test {} :test-all {} :retest {} :clone {} :close {} + :complete {} :describe {} :eldoc {} :eval {} :info {} + :load-file {}} + ops)) (let [{:keys [basilisp python]} versions] (is (contains? basilisp :version-string)) (is (contains? python :version-string))))))) @@ -272,29 +302,58 @@ {:id @id* :ns "user" :value "1"} {:id @id* :ns "user" :status ["done"]}) (client-send! client {:id (id-inc!) :op "complete" :prefix "clojure.test/"}) - (is (= {:id @id* :status ["done"] - :completions [{:type "var" :ns "basilisp.test" :candidate "clojure.test/*test-failures*"} - {:type "var" :ns "basilisp.test" :candidate "clojure.test/*test-name*"} - {:type "var" :ns "basilisp.test" :candidate "clojure.test/*test-section*"} - {:type "macro" :ns "basilisp.test" :candidate "clojure.test/are"} - {:type "macro" :ns "basilisp.test" :candidate "clojure.test/deftest"} - {:type "var" :ns "basilisp.test" :candidate "clojure.test/gen-assert"} - {:type "macro" :ns "basilisp.test" :candidate "clojure.test/is"} - {:type "macro" :ns "basilisp.test" :candidate "clojure.test/testing"} - {:type "var" :ns "basilisp.test" :candidate "clojure.test/use-fixtures"}]} - (client-recv! client))) + (let [{:keys [status id completions]} (client-recv! client) + has-completion? (set completions)] + (is (= id @id*)) + (is (= status ["done"])) + (are [expected] (has-completion? expected) + {:type "var" :candidate "clojure.test/*test-failures*" :ns "basilisp.test"} + {:type "var" :candidate "clojure.test/*test-name*" :ns "basilisp.test"} + {:type "var" :candidate "clojure.test/*test-section*" :ns "basilisp.test"} + {:type "var" :candidate "clojure.test/*test-var*" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/->SimpleReport" :ns "basilisp.test"} + {:type "var" :candidate "clojure.test/Report" :ns "basilisp.test"} + {:type "macro" :candidate "clojure.test/are" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/assert!" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/collect-tests" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/compile-report" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/continue?" :ns "basilisp.test"} + {:type "macro" :candidate "clojure.test/deftest" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/error!" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/failures" :ns "basilisp.test"} + {:type "var" :candidate "clojure.test/gen-assert" :ns "basilisp.test"} + {:type "macro" :candidate "clojure.test/is" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/map->SimpleReport" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/pass!" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/report-begin" :ns "basilisp.test"} + {:type "function" :candidate "clojure.test/report-end" :ns "basilisp.test"})) + (client-send! client {:id (id-inc!) :op "complete" :prefix "test/"}) - (is (= {:id @id* :status ["done"] - :completions [{:type "var" :ns "basilisp.test" :candidate "test/*test-failures*"} - {:type "var" :ns "basilisp.test" :candidate "test/*test-name*"} - {:type "var" :ns "basilisp.test" :candidate "test/*test-section*"} - {:type "macro" :ns "basilisp.test" :candidate "test/are"} - {:type "macro" :ns "basilisp.test" :candidate "test/deftest"} - {:type "var" :ns "basilisp.test" :candidate "test/gen-assert"} - {:type "macro" :ns "basilisp.test" :candidate "test/is"} - {:type "macro" :ns "basilisp.test" :candidate "test/testing"} - {:type "var" :ns "basilisp.test" :candidate "test/use-fixtures"}]} - (client-recv! client)))))))) + (let [{:keys [status id completions]} (client-recv! client) + has-completion? (set completions)] + (is (= id @id*)) + (is (= status ["done"])) + (are [expected] (has-completion? expected) + {:type "var" :candidate "test/*test-failures*" :ns "basilisp.test"} + {:type "var" :candidate "test/*test-name*" :ns "basilisp.test"} + {:type "var" :candidate "test/*test-section*" :ns "basilisp.test"} + {:type "var" :candidate "test/*test-var*" :ns "basilisp.test"} + {:type "function" :candidate "test/->SimpleReport" :ns "basilisp.test"} + {:type "var" :candidate "test/Report" :ns "basilisp.test"} + {:type "macro" :candidate "test/are" :ns "basilisp.test"} + {:type "function" :candidate "test/assert!" :ns "basilisp.test"} + {:type "function" :candidate "test/collect-tests" :ns "basilisp.test"} + {:type "function" :candidate "test/compile-report" :ns "basilisp.test"} + {:type "function" :candidate "test/continue?" :ns "basilisp.test"} + {:type "macro" :candidate "test/deftest" :ns "basilisp.test"} + {:type "function" :candidate "test/error!" :ns "basilisp.test"} + {:type "function" :candidate "test/failures" :ns "basilisp.test"} + {:type "var" :candidate "test/gen-assert" :ns "basilisp.test"} + {:type "macro" :candidate "test/is" :ns "basilisp.test"} + {:type "function" :candidate "test/map->SimpleReport" :ns "basilisp.test"} + {:type "function" :candidate "test/pass!" :ns "basilisp.test"} + {:type "function" :candidate "test/report-begin" :ns "basilisp.test"} + {:type "function" :candidate "test/report-end" :ns "basilisp.test"}))))))) (deftest nrepl-server-eval (testing "basic" @@ -697,3 +756,137 @@ :done) status (deref shutdown-thread 1000 :time-out)] (is (= :done status)))))))))) + +(defn elapsed-time? + [x] + (and (int? (:ms x nil)) (string? (:humanized x nil)))) + +(deftest nrepl-server-test + (with-server {} + (with-connect client + (let [id* (atom 0) + id-inc! #(swap! id* inc)] + (client-send! client + {:id (id-inc!) :op "eval" + :code "(require 'nrepl-server-test-generated.test1)"}) + (client-recv! client) + (is (= ["done"] (:status (client-recv! client)))) + + (testing "format" + (client-send! client {:id (id-inc!) + :op "test" + :ns "nrepl-server-test-generated.test1" + :tests nil}) + (let [{:as response + :keys [results + summary + elapsed-time + ns-elapsed-time + var-elapsed-time]} (client-recv! client)] + (is (= {:test 4 :ns 1 :error 1 :pass 2 :fail 1 :var 3} summary)) + (is (elapsed-time? elapsed-time)) + (is (= '(:nrepl-server-test-generated.test1) + (keys ns-elapsed-time))) + (is (every? elapsed-time? (vals ns-elapsed-time))) + (let [var-elapsed (get var-elapsed-time + :nrepl-server-test-generated.test1)] + (is (= #{:pass-test :fail-test :error-test} + (-> var-elapsed keys set))) + (is (every? (comp elapsed-time? :elapsed-time) + (vals var-elapsed))) + (let [ns-results (get results :nrepl-server-test-generated.test1) + error-result (:error-test ns-results)] + (is (re-find #".*nrepl_server_test_generated.+test1.lpy" + (-> error-result first :file))) + (let [lines (-> error-result first :error (.split "\n"))] + (is (re-find #"Traceback \(most recent call last\):" + (first lines))) + (is (re-find #"^ basilisp.lang.exception.ExceptionInfo: Boom! \{\}" + (last (butlast lines))))) + (is (= 1 (count error-result))) + (is (= {:fault "true" + :actual "basilisp.lang.exception.ExceptionInfo(Boom!, {})\n" + :ns "nrepl-server-test-generated.test1" + :message "Exception in test: basilisp.lang.exception.ExceptionInfo(Boom!, {})" + :expected "nil\n" + :index 0 + :type "error" + :var "error-test" + :line 4 + :context ""} + (dissoc (first error-result) :file :error))) + + (let [pass-result (:pass-test ns-results)] + (is (= [{:ns "nrepl-server-test-generated.test1" + :message "Test failure: (= 1 1)" + :index 0 + :expr "(= 1 1)" + :type "pass" + :var "pass-test" + :context "section"} + {:ns "nrepl-server-test-generated.test1" + :message "Test failure: (thrown? Exception (throw (ex-info \"Boom!\" {})))" + :index 1 + :expr "(thrown? Exception (throw (ex-info \"Boom!\" {})))" + :type "pass" + :var "pass-test" + :context ""}] + pass-result))) + + (let [fail-result (:fail-test ns-results)] + (is (= 1 (count fail-result))) + (is (re-find #".*nrepl_server_test_generated.+test1\.lpy" + (-> fail-result first :file))) + (is (= {:actual "1\n" + :ns "nrepl-server-test-generated.test1" + :message "Test failure: (= 0 1)" + :index 0 + :line 8 + :expected "0\n" + :expr "(= 0 1)" + :type "fail" + :var "fail-test" + :context ""} + (dissoc (first fail-result) :file))))))) + + (testing "specific tests" + (client-send! client {:id (id-inc!) + :op "test" + :ns "nrepl-server-test-generated.test1" + :tests ["pass-test" "fail-test"]}) + (let [{:keys [summary results]} (client-recv! client)] + (is (= 2 (:var summary))) + (is (= #{:pass-test :fail-test} + (-> results :nrepl-server-test-generated.test1 keys set))))) + + (testing "fail fast" + (client-send! client {:id (id-inc!) + :op "test" + :ns "nrepl-server-test-generated.test1" + :fail-fast "true" + :tests ["fail-test" "pass-test"]}) + (let [{:keys [summary results]} (client-recv! client)] + (is (= 1 (:var summary))) + (is (= '(:fail-test) + (-> results :nrepl-server-test-generated.test1 keys)))))))))) + +(deftest nrepl-server-retest + (with-server {} + (with-connect client + (let [id* (atom 0) + id-inc! #(swap! id* inc)] + (client-send! client + {:id (id-inc!) :op "eval" + :code "(require 'nrepl-server-test-generated.test1)"}) + (client-recv! client) + (is (= ["done"] (:status (client-recv! client)))) + (client-send! client {:id (id-inc!) + :op "test" + :ns "nrepl-server-test-generated.test1" + :tests nil}) + (is (= 3 (-> (client-recv! client) :summary :var))) + (client-send! client {:id (id-inc!), :op "retest"}) + (let [{:keys [results summary]} (client-recv! client)] + (is (= 2 (:var summary))) + (is (= #{:error-test :fail-test} + (-> results :nrepl-server-test-generated.test1 keys set)))))))) diff --git a/tests/basilisp/source_test.py b/tests/basilisp/source_test.py index 12c340311..d4a963a9d 100644 --- a/tests/basilisp/source_test.py +++ b/tests/basilisp/source_test.py @@ -23,13 +23,14 @@ def test_format_source_context(monkeypatch, source_file, source_file_path): textwrap.dedent( """ (ns source-test) - + (a) (let [a 5] (b)) """ ) ) + monkeypatch.setenv("TERM", "xterm") format_c = format_source_context(source_file_path, 2, end_line=4) assert [ " 1 | \x1b[37m\x1b[39;49;00m\n", diff --git a/tests/basilisp/test_core_fns.lpy b/tests/basilisp/test_core_fns.lpy index 903688f3b..43a5081d7 100644 --- a/tests/basilisp/test_core_fns.lpy +++ b/tests/basilisp/test_core_fns.lpy @@ -1614,8 +1614,8 @@ (is (= #py [#py [1 2 3] #py [:a :b :c]] (to-array-2d '([1 2 3] (:a :b :c))))) - (is (thrown? python/TypeError) - (to-array-2d [[1 2 3] :b]))) + (is (thrown? python/TypeError + (to-array-2d [[1 2 3] :b])))) (deftest into-array-test (testing "with no type" diff --git a/tests/basilisp/test_multifn.lpy b/tests/basilisp/test_multifn.lpy index c9c79bc74..07769a098 100644 --- a/tests/basilisp/test_multifn.lpy +++ b/tests/basilisp/test_multifn.lpy @@ -91,8 +91,8 @@ (is (= "operating system" (os-lineage {:os :os/windows})))) (testing "cannot establish conflicting preference" - (is (thrown? basilisp.lang.runtime/RuntimeException) - (prefer-method os-lineage :os/bsd :os/unix))))) + (is (thrown? basilisp.lang.runtime/RuntimeException + (prefer-method os-lineage :os/bsd :os/unix)))))) (defmulti args-test1 "test1" :x) (defmulti args-test2 {:test 2} :x) diff --git a/tests/basilisp/test_test.lpy b/tests/basilisp/test_test.lpy new file mode 100644 index 000000000..1db549277 --- /dev/null +++ b/tests/basilisp/test_test.lpy @@ -0,0 +1,503 @@ +(ns tests.basilisp.test-test + (:import os) + (:require [basilisp.test :refer :all] + [basilisp.test.fixtures :as fixtures])) + +(defn- make-temp-file + [path text] + (assert fixtures/*tempdir* "Missing tempdir fixture") + (let [file (.join os/path fixtures/*tempdir* path)] + (os/makedirs (.dirname os/path file) ** :exist_ok true) + (with-open [wtr (python/open file ** :mode "w")] + (.write wtr text)))) + +(defn- generated-tests-fixture + [] + (with-fixtures [fixtures/reset-path + fixtures/tempdir] + (.insert sys/path 0 fixtures/*tempdir*) + (make-temp-file + "test_test_generated/test1.lpy" + "(ns test-test-generated.test1 + (:require [basilisp.test :refer :all])) + + (def not-a-test true) + + (deftest error-test + (throw (ex-info \"Boom!\" {}))) + + (deftest fail-test + (is (= 0 1))) + + (deftest pass-test + (is (= 1 1)) + (is (thrown? Exception (throw (ex-info \"Boom!\" {})))))") + + (make-temp-file + "test_test_generated/test2.lpy" + "(ns test-test-generated.test2 + (:require [basilisp.test :refer :all])) + + (deftest message-test + (is (= 1 1) \"pass message\") + (is (= 1 0) \"fail message\")) + + (deftest section-test + (is (= 0 1)) + (testing \"section\" + (is (= 1 0)) + (is (= 1 1)) + (is (nil? nil) \"message\") + (is false \"message\")))") + + (make-temp-file + "test_test_generated/test3.lpy" + "(ns test-test-generated.test3 + (:require [basilisp.test :refer :all])) + + (def ^:dynamic *once* false) + (def ^:dynamic *each* false) + + (use-fixtures :each (fn [] (binding [*each* true] (yield)))) + (use-fixtures :once (fn [] (binding [*once* true] (yield)))) + + (deftest fixture-test + (is *once*) + (is *each*))") + + (make-temp-file + "test_test_generated/test4.lpy" + "(ns test-test-generated.test4 + (:require [basilisp.test :refer :all])) + + (use-fixtures :each (fn [] (throw (ex-info \"Boom!\" {})))) + + (deftest each-fixture-test + (is true))") + + (make-temp-file + "test_test_generated/test5.lpy" + "(ns test-test-generated.test5 + (:require [basilisp.test :refer :all])) + + (use-fixtures :once (fn [] (throw (ex-info \"Boom!\" {})))) + + (deftest once-fixture-test + (is true))") + + (require 'test-test-generated.test1 + 'test-test-generated.test2 + 'test-test-generated.test3 + 'test-test-generated.test4 + 'test-test-generated.test5) + (yield))) + +(use-fixtures :once generated-tests-fixture) + +(defn- before-after-fixture + [events] + (fn [] + (swap! events conj :before) + (yield) + (swap! events conj :after))) + +(defn- index-fixture + [events idx] + (fn [] + (swap! events conj idx) + (yield) + (swap! events conj idx))) + +(def ^:dynamic *state* nil) + +(deftest with-fixtures-test + (testing "setup and teardown" + (let [events (atom [])] + (with-fixtures [(before-after-fixture events)] + (swap! events conj :during)) + (is (= [:before :during :after] @events)))) + + (testing "teardown on exception" + (let [events (atom [])] + (try + (with-fixtures [(before-after-fixture events)] + (swap! events conj :during) + (throw (ex-info "Boom!" {}))) + (catch Exception _ nil)) + (is (= [:before :during :after] @events)))) + + (testing "teardown on fixture setup exception" + (let [events (atom [])] + (try + (with-fixtures [(before-after-fixture events) + #(throw (ex-info "Boom!" {}))] + (swap! events conj :during)) + (catch Exception _ nil)) + (is (= [:before :after] @events)))) + + (testing "teardown on fixture teardown exception" + (let [events (atom [])] + (try + (with-fixtures [(before-after-fixture events) + (fn [] + (yield) + (throw (ex-info "Boom!" {})))] + (swap! events conj :during)) + (catch Exception _ nil)) + (is (= [:before :during :after] @events)))) + + (testing "applied in order" + (let [events (atom nil)] + (with-fixtures [(index-fixture events 1) + (index-fixture events 2) + (index-fixture events 3)] + (swap! events conj 4)) + (is (= '(1 2 3 4 3 2 1) @events)))) + + (testing "nesting fixtures" + (with-fixtures [(fn [] + (with-fixtures [(fn [] + (binding [*state* 1] + (yield)))] + (yield)))] + (is (= 1 *state*))))) + +(deftest join-fixtures-test + (testing "applied in order" + (let [events (atom nil)] + (with-fixtures [(join-fixtures [(index-fixture events 1) + (index-fixture events 2) + (index-fixture events 3)])] + (swap! events conj 4)) + (is (= '(1 2 3 4 3 2 1) @events))))) + +(defrecord TestReport [events max-events assertions] + Report + (continue? [self] + (< (count events) max-events)) + (report-begin [self] + (->TestReport (conj events [:begin]) max-events assertions)) + (report-end [self] + (->TestReport (conj events [:end]) max-events assertions)) + (report-namespace-begin [self ns] + (->TestReport (conj events [:begin ns]) max-events assertions)) + (report-namespace-end [self ns asserts] + (->TestReport (conj events [:end ns]) + max-events + (assoc assertions ns asserts))) + (report-test-begin [self var] + (->TestReport (conj events [:begin var]) max-events assertions)) + (report-test-end [self var asserts] + (->TestReport (conj events [:end var]) + max-events + (assoc assertions var asserts)))) + +(deftest use-fixtures-test + (let [fixture-test (resolve 'test-test-generated.test3/fixture-test) + {:keys [assertions]} (compile-report (->TestReport nil ##Inf nil) + [(namespace fixture-test)]) + result (group-by :expr (get assertions fixture-test))] + (are [expr] (= :pass (-> (get result expr) first :type)) + '*once* + '*each*))) + +(deftest compile-report-test + (let [namespaces (map find-ns ['test-test-generated.test1 + 'test-test-generated.test2 + 'test-test-generated.test3 + 'test-test-generated.test4 + 'test-test-generated.test5]) + {:keys [events]} (compile-report (->TestReport nil ##Inf nil) + namespaces) + has-event? (set events) + events (reverse events)] + (testing "report begin/end" + (doseq [ns namespaces] + (is (has-event? [:begin ns])) + (is (has-event? [:end ns]))) + (doseq [test ['test-test-generated.test1/pass-test + 'test-test-generated.test1/fail-test + 'test-test-generated.test1/error-test + 'test-test-generated.test2/message-test]] + (let [var (resolve test)] + (is (has-event? [:begin var])) + (is (has-event? [:end var])))) + + (is (= [:begin] (first events)) + "report-begin not called before testing starts") + (is (= [:end] (last events)) + "report-end not called when testing ends") + (reduce (fn [state [op k]] + (cond + (= :end op) + (if (= state k) + (cond + (nil? k) :report-end + (var? k) (namespace k) + :else nil) + (reduced + (is (= state k) (str (cond + (nil? k) "report-end" + (var? k) "report-test-end" + :else "report-namespace-end") + "call out of order")))) + (nil? k) + (reduced (is false "report-begin call out of order")) + + (var? k) + (if (= state (namespace k)) + k + (reduced (is (= state (namespace k)) + "report-test-begin call out of order"))) + + :else + (if (nil? state) + k + (reduced (is (nil? state) + "report-namespace-begin call out of order"))))) + nil + (rest events))) + + (testing "continue" + (are [n] + (every? (comp #{:end} first) + (->> namespaces + (compile-report (->TestReport nil n nil)) + :events + reverse + (drop (inc n)))) + 1 2 4 6 7 10)))) + +(deftest assertions-test + (let [[test1 test2 test3 test4 + test5 :as namespaces] (map find-ns ['test-test-generated.test1 + 'test-test-generated.test2 + 'test-test-generated.test3 + 'test-test-generated.test4 + 'test-test-generated.test5]) + {:keys [assertions]} (compile-report (->TestReport nil ##Inf nil) + namespaces)] + + (testing "pass" + (let [pass-test (resolve 'test-test-generated.test1/pass-test) + test-name (-> pass-test name str) + [assert1 assert2] (get assertions pass-test)] + (is (= {:ns test1 + :var pass-test + :test-name test-name + :test-section nil + :expr '(= 1 1) + :line 13 + :message "Test failure: (= 1 1)" + :type :pass} + assert1)) + (is (= {:ns test1 + :var pass-test + :test-name test-name + :test-section nil + :expr '(thrown? Exception (throw (ex-info "Boom!" {}))) + :line 14 + :message "Test failure: (thrown? Exception (throw (ex-info \"Boom!\" {})))" + :type :pass} + assert2)))) + + (testing "fail" + (let [fail-test (resolve 'test-test-generated.test1/fail-test)] + (is (= {:ns test1 + :var fail-test + :test-name (-> fail-test name str) + :test-section nil + :expr '(= 0 1) + :expected 0 + :actual 1 + :line 10 + :message "Test failure: (= 0 1)" + :type :failure} + (first (get assertions fail-test)))))) + + (testing "error" + (let [error-test (resolve 'test-test-generated.test1/error-test) + assertion (first (get assertions error-test))] + (is (instance? Exception (:actual assertion))) + (is (= {:ns test1 + :var error-test + :test-name (-> error-test name str) + :test-section nil + :expected nil + :line 6 + :expr nil + :message "Exception in test: basilisp.lang.exception.ExceptionInfo(Boom!, {})" + :type :error} + (dissoc assertion :actual))))) + + (testing "error each fixture" + (let [test (resolve 'test-test-generated.test4/each-fixture-test) + assertion (first (get assertions test))] + (is (instance? Exception (:actual assertion))) + (is (= {:ns test4 + :var test + :test-name (-> test name str) + :test-section nil + :expected nil + :line 6 + :expr nil + :message "Exception in test fixture: basilisp.lang.exception.ExceptionInfo(Boom!, {})" + :type :error} + (dissoc assertion :actual))))) + + (testing "error once fixture" + (let [assertion (first (get assertions test5))] + (is (instance? Exception (:actual assertion))) + (is (= {:ns test5 + :var nil + :test-name nil + :test-section nil + :expected nil + :line nil + :expr nil + :message "Exception in namespace fixture: basilisp.lang.exception.ExceptionInfo(Boom!, {})" + :type :error} + (dissoc assertion :actual))))) + + (testing "section" + (let [section-test (resolve 'test-test-generated.test2/section-test) + assertion (nth (get assertions section-test) 3)] + (is (= "message" (:message assertion))) + (is (= "section" (:test-section assertion))))))) + +(defn- found-test? + [tests var] + (some #{var} (get tests (namespace var)))) + +(deftest collect-tests-test + (testing "all tests" + (let [tests (collect-tests (all-ns))] + (are [test] (found-test? tests (resolve test)) + 'test-test-generated.test1/pass-test + 'test-test-generated.test1/fail-test + 'test-test-generated.test1/error-test + 'test-test-generated.test2/message-test + 'test-test-generated.test2/section-test + 'collect-tests-test) + (is (not (found-test? tests + (resolve 'test-test-generated.test1/not-a-test)))))) + + (testing "namespace tests" + (let [tests (collect-tests [(find-ns 'test-test-generated.test1)])] + (are [test] (found-test? tests (resolve test)) + 'test-test-generated.test1/pass-test + 'test-test-generated.test1/fail-test + 'test-test-generated.test1/error-test) + (are [x] (not (found-test? tests (resolve x))) + 'test-test-generated.test1/not-a-test + 'test-test-generated.test2/message-test + 'test-test-generated.test2/section-test + 'collect-tests-test))) + + (testing "var tests" + (let [tests (collect-tests + [(resolve 'test-test-generated.test1/pass-test) + (resolve 'test-test-generated.test1/fail-test)])] + (are [test] (found-test? tests (resolve test)) + 'test-test-generated.test1/pass-test + 'test-test-generated.test1/fail-test) + (are [x] (not (found-test? tests (resolve x))) + 'test-test-generated.test1/not-a-test + 'test-test-generated.test1/error-test + 'test-test-generated.test2/message-test + 'test-test-generated.test2/section-test + 'collect-tests-test))) + + (testing "current namespace" + (let [tests (collect-tests nil)] + (are [test] (found-test? tests (resolve test)) + 'collect-tests-test) + (are [x] (not (found-test? tests (resolve x))) + 'test-test-generated.test1/not-a-test + 'test-test-generated.test1/pass-test + 'test-test-generated.test1/fail-test + 'test-test-generated.test1/error-test + 'test-test-generated.test2/message-test + 'test-test-generated.test2/section-test)))) + +(defn- has-lines? + "return true if the lines contain some match for the regexes in order" + [lines regexes] + (cond + (not regexes) true + (not lines) false + :else (recur (next lines) + (if (re-find (first regexes) (first lines)) + (next regexes) + regexes)))) + +(deftest simple-report-test + (let [[result stdout] (binding [*out* (io/StringIO)] + [(run-tests (find-ns 'test-test-generated.test1)) + (.split (.getvalue *out*) "\n")])] + (testing "summary" + (are [k n] (= n (get result k)) + :pass 2 + :failure 1 + :error 1 + :tests 3)) + + (testing "print failures" + (is (has-lines? + stdout + [#"Testing test-test-generated\.test1" + #"" + #"Ran 3 tests containing 3 assertions\." + #"1 failures, 1 errors\."])) + (is (has-lines? + stdout + [#"Testing test-test-generated\.test1" + #"" + #"ERROR in \(error-test\) \(.*test_test_generated.+test1\.lpy:6\)" + #"Exception in test: basilisp\.lang\.exception\.ExceptionInfo\(Boom\!, \{\}\)" + #"Traceback \(most recent call last\):" + #" File \".*basilisp.+test.lpy\", line \d+, in execute__STAR__.*" + #" File \".*test_test_generated.+test1.lpy\", line 7, in __error_test.*" + #" basilisp.lang.exception.ExceptionInfo: Boom! \{\}" + #"" + #"Ran 3 tests containing 3 assertions\." + #"1 failures, 1 errors\."])) + (is (has-lines? + stdout + [#"Testing test-test-generated\.test1" + #"" + #"FAIL in \(fail-test\) \(.*test_test_generated.+test1\.lpy:10\)" + #"Test failure: \(= 0 1\)" + #"expected: 0" + #" actual: 1" + #"" + #"Ran 3 tests containing 3 assertions\." + #"1 failures, 1 errors\."])))) + + (testing "print test context" + (let [stdout (.split (with-out-str (run-tests 'test-test-generated.test2)) + \newline)] + (is (has-lines? + stdout + [#"FAIL in \(message-test\) \(.*test_test_generated.+test2\.lpy:6\)" + #"fail message" + #"expected: 1" + #" actual: 0"])) + (is (has-lines? + stdout + [#"FAIL in \(section-test\) \(.*test_test_generated.+test2\.lpy:9\)" + #"Test failure: \(= 0 1\)" + #"expected: 0" + #" actual: 1" + #"" + #"FAIL in \(section-test\) \(.*test_test_generated.+test2\.lpy:11\)" + #"section" + #"Test failure: \(= 1 0\)" + #"expected: 1" + #" actual: 0" + #"" + #"FAIL in \(section-test\) \(.*test_test_generated.+test2\.lpy:14\)" + #"section" + #"message" + #"expected: \(not false\)" + #" actual: false"]))))) diff --git a/tests/basilisp/testrunner_test.py b/tests/basilisp/testrunner_test.py index 1939cfb2e..01237c118 100644 --- a/tests/basilisp/testrunner_test.py +++ b/tests/basilisp/testrunner_test.py @@ -143,7 +143,7 @@ def test_error_repr(self, run_result: pytest.RunResult): "ERROR in (assertion-test) (test_testrunner.lpy:14)", "", "Traceback (most recent call last):", - ' File "*test_testrunner.lpy", line 14, in assertion_test', + ' File "*test_testrunner.lpy", line 14, in __assertion_test_*', ' (is (throw (ex-info "Uncaught exception" {})))', "basilisp.lang.exception.ExceptionInfo: Uncaught exception {}", ] @@ -155,9 +155,12 @@ def test_error_repr(self, run_result: pytest.RunResult): run_result.stdout.fnmatch_lines( [ - "ERROR in (error-test) (test_testrunner.lpy)", + "ERROR in (error-test) (test_testrunner.lpy:34)", + "", "Traceback (most recent call last):", - ' File "*test_testrunner.lpy", line 35, in error_test', + ' File "*basilisp/test.lpy", line *, in execute__STAR__*', + " (try", + ' File "*test_testrunner.lpy", line 35, in __error_test_*', " (throw", "basilisp.lang.exception.ExceptionInfo: This test will count as an error. {}", ]