.gitignore

@@ -13,3 +13,4 @@ pom.xml.asc
 book.iml
 skirmish.iml
 full-stack
+fixtures

README.md

@@ -6,29 +6,36 @@ A stock exchange simulator.  The scope of this project is not yet defined but it
 allow end user(s?) to exchange securities via the common exchange rules.
 
 
-## Installation
+## Getting Started
-Okay, this should be easy but it is hard... 
+
+### Development
+
+In development, you need nothing else than [clojure](https://clojure.org/)
+and [leiningen](https://leiningen.org/).  The easiest way to install clojure is
+to make sure you have java (8 and above) and install leiningen.
+
+### Production
+
+You will need plenty of things, the majority of them are yet to define.
+But, a good kafka deployment will be required.  Probably a database,
+hopefully datomic.
 
 
 ### Configuration
 All configuration are set in the `resources/config.edn` file.
-Configuration uses [aero](https://github.com/juxt/aero).
+Configuration uses [aero](https://github.com/juxt/aero).  The configuration
+file should be self documented.
 
 ### Technologies
 
 #### Server
-* clojure
+* [clojure](https://clojure.org/) for programming language
-* kafka
+* [kafka](http://kafka.apache.org/) for message broker/database
 
 #### In the browser
-* https://aftertheflood.com/projects/sparks/
+* [clojure script](https://clojurescript.org/) for programming language
-* http://metricsgraphicsjs.org/
+* [sparks](https://aftertheflood.com/projects/sparks/) for trivial charts
-
+* [metricsgraphicsjs](http://metricsgraphicsjs.org/) for complex charts
-## Usage
-
-FIXME: explanation
-
-    $ java -jar book-0.1.0-standalone.jar [args]
 
 
 

docker-compose.yml

@@ -154,7 +154,7 @@ services:
 
   skirmish-api:
     image: skirmish
-    command: ["lein", "run", "webserver"]
+    command: ["lein", "run", "api-server"]
     depends_on:
       - kafka-topics-ui # basically, we need all kafka
     environment:
@@ -162,3 +162,14 @@ services:
       KAFKA_BOOTSTRAP_SERVERS: "kafka1:9092"
     ports:
        - "1602:1602"
+
+  skirmish-exchange-core:
+    image: skirmish
+    command: ["lein", "run", "exchange-core"]
+    depends_on:
+      - skirmish-api # basically, we need all kafka
+    environment:
+      PORT: 1602  # https://en.wikipedia.org/wiki/Euronext_Amsterdam
+      KAFKA_BOOTSTRAP_SERVERS: "kafka1:9092"
+    ports:
+       - "1602:1602"

project.clj

@@ -19,6 +19,7 @@
                  [slingshot "0.12.2"]
                  [spootnik/kinsky "0.1.22"]
                  [org.clojars.fossi/kafka-streams-clojure "0.1.0-SNAPSHOT"] ;; JEAH!
+                 [orchestra "2017.11.12-1"]
                  [cheshire "5.8.0"]]
   :java-source-paths ["src/java"]
   :main ^:skip-aot skirmish.core

resources/config.edn

@@ -1,20 +1,37 @@
 {
+ ;; Specify which middleware to use, it will default to :chan.
+ :middleware :kafka
+
+ ;; If set to true, orchestra will instrument all the spec'ed functions
+ :instrument-spec true
+
+ ;; all the kafka configuration goes bellow.
  :kafka
  {:bootstrap.servers #or [#env KAFKA_BOOTSTRAP_SERVERS "localhost:9092"]
   :topics-prefix "skirmish-01-"}
 
- :webserver ;; possible config from http-kit's run-server
+ ;; possible config from http-kit's run-server that will affect the api-server
- {:port #long #or [#env PORT 8080]
+ :api-server
+ {:port #long #or [#env API_SERVER_PORT 1602]
   :ip "0.0.0.0"
   :max-body 524288 ;; 512kb, small but enough
   :max-ws 524288}
 
- :securities ;; list of tradable securities
+ ;; possible config from http-kit's run-server that will affect the api-bookkeeper
+ :api-bookkeeper
+ {:port #long #or [#env API_SERVER_PORT 1603]
+  :ip "0.0.0.0"
+  :max-body 524288 ;; 512kb, small but enough
+  :max-ws 524288}
+
+ ;; list of tradable securities
+ :securities
  [{:sec-id :foo}
   {:sec-id :bar}
   {:sec-id :toto}
   {:sec-id :tata}]
 
+ ;; stupid parameters about benchmarking tools
  :benchmark {
              :number-of-requests-per-securities 30}}
 

src/skirmish/cli/benchmark.clj

@@ -94,7 +94,7 @@
   (println "status map: " (:status results))
   (pp-durations (:durations results)))
 
-(defn -main [config]
+(defn -main [config exchange-apis]
   (let [benchmark-config (:benchmark config)
         requests (-> benchmark-config :number-of-requests-per-securities (make-requests (:securities config)))
         orders (create-orders-channel (:securities config))

src/skirmish/cli/exchange.clj

@@ -1,7 +1,7 @@
 (ns skirmish.cli.exchange
-  (:require [skirmish.match :refer [sell?]]
+  (:require [skirmish.exchange.match :refer [sell?]]
     [skirmish.util :refer :all]
-    [skirmish.exchange :refer [empty-book create-exchange-for-security]]
+    [skirmish.exchange.exchange :refer [empty-book create-exchange-for-security]]
     [skirmish.order]  ;; for spec side effect
     [slingshot.slingshot :refer [try+ throw+]]
     [clojure.edn :as edn]

src/skirmish/core.clj

@@ -2,18 +2,46 @@
   (:require [skirmish.cli.core]
             [skirmish.rest.core]
             [skirmish.cli.benchmark]
-            [skirmish.kafka.exchange-simulator]
+            [skirmish.exchange.core]
-            [aero.core :refer (read-config)])
+            [skirmish.middleware.core :refer [create-exchange-apis-for]]
+            [aero.core :refer (read-config)]
+            [orchestra.spec.test :as st]
+            [clojure.core.async :as async])
   (:gen-class))
 
-(def apps {:webserver skirmish.rest.core/-main
+(def targets ;; a target is a function that blocks the thread and takes two
-           :benchmark skirmish.cli.benchmark/-main
+             ;; arguments:  the config map and the "exchange-api"
-           :kafka-exchange-simulator skirmish.kafka.exchange-simulator/-main})
+  {:api-server     skirmish.rest.core/api-main
+   :exchange-core  skirmish.exchange.core/-main ;; should be skirmish.exchange.core/api-main
+   :api-bookkeeper skirmish.rest.core/bookkeeper-main})
+   ;:benchmark      skirmish.cli.benchmark/-main})
+
+(defn run-all-targets-in-separate-threads [target-fns config exchange-apis]
+  (println "running  multiple targets. This is okay in dev but horrible in production")
+  (println target-fns)
+  (let [threads (map #(async/thread (% config exchange-apis)) target-fns)
+        finished-threads (async/merge threads)]
+    (loop [finished-thread (async/<!! finished-threads)]
+      (when (some? finished-thread)
+          (recur (async/<!! finished-threads))))))
+
+
+(defn run-all-targets [target-fns config]
+  (let [exchange-apis (create-exchange-apis-for config)]
+    (if (= (count target-fns) 1)
+      ((first target-fns) config exchange-apis)
+      (run-all-targets-in-separate-threads target-fns config exchange-apis))))
 
 (defn -main [& argv]
   (let [config (-> "config.edn" clojure.java.io/resource read-config)
-        target (-> argv first keyword)
+        xf (comp (map keyword) (map targets) (filter some?))
-        target-fn (get apps target)]
+        target-fns (sequence xf argv)]
-    (if target-fn
+    (if (true? (:instrument-spec config))
-      (apply target-fn [config])
+      (st/instrument))
-      (println "target" target "not found"))))
+    (if (seq target-fns)
+      (run-all-targets target-fns config)
+      (do
+        (println argv)
+        (println "target" target-fns "not found")
+        (println "all known targets are...")
+        (println (keys targets))))))

src/skirmish/exchange/core.clj

@@ -0,0 +1,22 @@
+(ns skirmish.exchange.core
+  (:require [skirmish.util :refer [block-thread]]
+            [skirmish.middleware.core :refer [create-exchange-apis-for]]
+            [skirmish.exchange.exchange :refer [create-exchange-for-security empty-book]]
+            [kinsky.async]
+            [kinsky.client]))
+
+(defn create-exchange-for [exchange-api]
+  (let [initial-book empty-book
+        new-order-publisher (:new-order-publisher exchange-api)
+        trades-submitter (:trades-submitter exchange-api)
+        orders-submitter (:orders-submitter exchange-api)]
+
+    (create-exchange-for-security initial-book
+                                  new-order-publisher
+                                  trades-submitter
+                                  orders-submitter)))
+
+(defn -main [config exchange-apis]
+  (doseq [{sec-id :sec-id} (:securities config)]
+    (create-exchange-for (get exchange-apis sec-id)))
+  (block-thread))

src/skirmish/exchange/exchange.clj

@@ -1,9 +1,10 @@
-(ns skirmish.exchange
+(ns skirmish.exchange.exchange
-  (:require [skirmish.match :refer [sell?
+  (:require [skirmish.exchange.match :refer [sell?
                                              create-buy-book
                                              create-sell-book
                                              match-order-in-book]]
             [skirmish.util :refer :all]
+            [clojure.spec.alpha :as s]
             [clojure.core.async :refer [chan go >! <! pub] :as async]))
 
 
@@ -13,6 +14,18 @@
                  :buy (create-buy-book)
                  :id 0})
 
+(defn get-reference-book [order book]
+  "The reference book is the book where the order will be fulfilled"
+  (if (sell? order)
+    (:buy book)
+    (:sell book)))
+
+(defn get-same-type-order [order book]
+  "the same type order book is the book where the order will be added if not fulfilled"
+  (if (sell? order)
+    (:sell book)
+    (:buy book)))
+
 ;;  Bellow are all the functions to match the received order in the correct book
 (defn process-abstract [order reference-book same-type-book]
   (let [[trades new-reference-book leftover-order] (match-order-in-book reference-book order)
@@ -44,19 +57,19 @@
   (let [book-id (:book.order/book-id trade)
         f #(= book-id (:book.order/id %))
         ret-order (first (filter f book))
-        new-quantity (- (:quantity ret-order 10) (:book.order/quantity trade));; (- (:quantity ret-order) (:book.order/quantity trade))
+        new-quantity (- (:quantity ret-order) (:book.order/quantity trade));; (- (:quantity ret-order) (:book.order/quantity trade))
         totally-fulfilled? (= (:book.order/quantity trade) (:quantity ret-order))]
-    (into ret-order {:status         (if totally-fulfilled? :totally-fulfilled :partially-fulfilled)
+    (into ret-order {:book.order/status         (if totally-fulfilled? :totally-fulfilled :partially-fulfilled)
-                     :status-dual-id (:book.order/order-id trade)
+                     :book.order/status-dual-id (:book.order/order-id trade)
-                     :quantity new-quantity})));;;;
+                     :quantity new-quantity})))
 
 
 (defn get-intermediate-order [order trade]
   (let [new-order-quantity (- (:quantity order) (:book.order/quantity trade))
         totally-fulfilled? (= 0 new-order-quantity)
         dual-id (:book.order/book-id trade)]
-    (into order {:status (if totally-fulfilled? :totally-fulfilled :partially-fulfilled)
+    (into order {:book.order/status (if totally-fulfilled? :totally-fulfilled :partially-fulfilled)
-                 :status-dual-id dual-id
+                 :book.order/status-dual-id dual-id
                  :quantity new-order-quantity})))
 
 
@@ -68,33 +81,31 @@
 
 (defn get-processed-orders [order leftover-order trades book]
   "Computes all the intermediary orders that took place during
-  a the trade."
+  a the trade.
-  (let [ret [(assoc order :status :inserted)]
+  The interal logic is very complicated.  Probably too complicated."
+  (let [ret [(assoc order :book.order/status :received)]
         ret (into ret (get-processed-orders-in-trades order book trades))
         ret (if (seq leftover-order)
-              (conj ret (assoc leftover-order :status :inserted-into-book))
+              (conj ret (assoc leftover-order :book.order/status :inserted-into-book))
-              leftover-order)]
+              ret)]
     ret))
 
+(s/fdef get-processed-orders
+        :ret (s/* :book.order/order))
 
-;; bellow are the really interesting functions.
 
-(defn create-exchange-for-security [initial-book incoming-order-channel]
+;; bellow are the really interesting functions; aka public functions.
-  ;; output - trades channel, pub channel
+
-  ;;        - orders channel, pub channel
+(defn create-exchange-for-security [initial-book new-order-publisher trades-submitter orders-submitter]
-  (let [trades-channel (chan)
-        outcoming-orders-channel (chan)]
   (async/go
-      (loop [order (<! incoming-order-channel) book initial-book]
+    (loop [order (<! new-order-publisher) book initial-book]
-        (println "received an order!  " order)
       (let [[new-book new-order new-trades] (process-order book order)
-              orders (get-processed-orders order new-order new-trades book)
+            orders (get-processed-orders order new-order new-trades (get-reference-book order book))
-              orders-copy (async/onto-chan outcoming-orders-channel orders false)
+            orders-copy (async/onto-chan orders-submitter orders false)
-              trades-copy (async/onto-chan trades-channel new-trades false)]
+            trades-copy (async/onto-chan trades-submitter new-trades false)]
         (<! orders-copy)
         (<! trades-copy)
-          (recur (<! incoming-order-channel) new-book))))
+        (recur (<! new-order-publisher) new-book)))))
-    [(pub trades-channel :book.order/type) (pub outcoming-orders-channel :type)]))
 
 
 

src/skirmish/exchange/match.clj

@@ -1,4 +1,4 @@
-(ns skirmish.match
+(ns skirmish.exchange.match
   (:require [skirmish.util :refer [sorted-list-by]]
             [clojure.spec.alpha :as s]
             [clojure.edn :as edn]))

src/skirmish/kafka_exchange.clj

@@ -1,41 +0,0 @@
-(ns skirmish.kafka-exchange
-  (:require [kafka-streams-clojure.api :as api])
-  (:import [org.apache.kafka.clients.producer KafkaProducer ProducerRecord]
-           [org.apache.kafka.streams StreamsConfig KafkaStreams]
-           [org.apache.kafka.streams.kstream KStreamBuilder]))
-
-
-(defn -main []
-  (def xform (comp (filter (fn [[k v]] (string? v)))
-                   (map (fn [[k v]] [v k]))
-                   (filter (fn [[k v]] (= "foo" v)))))
-
-  (def builder (KStreamBuilder.))
-
-  (def kstream (-> builder
-                   (.stream (into-array String ["tset"]))
-                   (api/transduce-kstream xform)
-                   (.to "test")))
-
-  (def kafka-streams
-    (KafkaStreams. builder
-                   (StreamsConfig. {StreamsConfig/APPLICATION_ID_CONFIG    "test-app-id"
-                                    StreamsConfig/BOOTSTRAP_SERVERS_CONFIG "localhost:9092"
-                                    StreamsConfig/KEY_SERDE_CLASS_CONFIG   org.apache.kafka.common.serialization.Serdes$StringSerde
-                                    StreamsConfig/VALUE_SERDE_CLASS_CONFIG org.apache.kafka.common.serialization.Serdes$StringSerde})))
-  (.start kafka-streams)
-
-  (def producer (KafkaProducer. {"bootstrap.servers" "localhost:9092"
-                                 "acks"              "all"
-                                 "retries"           "0"
-                                 "key.serializer"    "org.apache.kafka.common.serialization.StringSerializer"
-                                 "value.serializer"  "org.apache.kafka.common.serialization.StringSerializer"}))
-
-  @(.send producer (ProducerRecord. "tset" "foo" "bar"))
-  ;; Observe message come across topic "test" via kafka-console-consumer
-
-  @(.send producer (ProducerRecord. "tset" "baz" "quux"))
-  ;; Observe message does not come across topic "test" via kafka-console-consumer
-
-  (.close producer)
-  (.close kafka-streams))

src/skirmish/middleware/core.clj

@@ -0,0 +1,27 @@
+(ns skirmish.middleware.core
+  (:require [clojure.core.async :as async]
+            [skirmish.exchange.exchange :refer [empty-book create-exchange-for-security]] ;; remove this import.
+            [kinsky.async]
+            [kinsky.client]))
+
+(defn create-chan-based-exchange-api-for [sec-id]
+  (let [new-order (async/chan)
+        trades (async/chan)
+        orders (async/chan)]
+    {:sec-id                 sec-id
+     :new-order-publisher    new-order  ;; anything published here will be processed as
+     :new-order-submitter    new-order
+     :trades-publisher       (async/pub trades :book.order/type)
+     :trades-submitter       trades
+     :orders-publisher       (async/pub orders :book.order/status)
+     :orders-submitter       orders}))
+
+(def api-creators {})
+
+(defn create-exchange-apis-for [config]
+  ;; todo: spec this function
+  (let [security-definitions (map :sec-id (:securities config))
+        create-exchange-api-for (get api-creators (:middleware config) create-chan-based-exchange-api-for)
+        f (fn [sec] [sec (create-exchange-api-for sec)])
+        ret (into {} (map f security-definitions))]
+    ret))

src/skirmish/order.clj

@@ -7,18 +7,19 @@
 ;;  :type    one of :sell :buy
 ;;  ::id     maybe an id
 
-(s/def :book.order/type #{:buy :sell})
+(def order-status #{:received  ;; maybe called acknowledged.
-(s/def :book.order/price pos-int?)
-(s/def :book.order/quantity pos-int?)
-(s/def :book.order/id    pos-int?)
-(s/def :book.order/status #{:inserted  ;; maybe called acknklowdlge.
                     :partially-matched
                     :totally-matched
                     :inserted-into-book
                     :partially-fulfilled
                     :totally-fulfilled
-                            :withdrawn}) ;; for instance the order has been canceled
+                    :withdrawn}) ;; for instance the order has been canceled)
 
+(s/def :book.order/type #{:buy :sell})
+(s/def :book.order/price nat-int?)
+(s/def :book.order/quantity nat-int?)
+(s/def :book.order/id    nat-int?)
+(s/def :book.order/status order-status)
 
 
 (s/def :book.order/status-dual-id :book.order/id)

src/skirmish/rest/bookkeeper.clj

@@ -0,0 +1,118 @@
+(ns skirmish.rest.bookkeeper
+  (:require [skirmish.exchange.exchange :refer [empty-book create-exchange-for-security]]
+            [org.httpkit.server :refer [run-server with-channel send! close on-close]]
+            [compojure.route :refer [not-found]]
+            [compojure.handler :refer [api]]
+            [compojure.core :refer :all]
+            [skirmish.order]  ;; for spec side effect
+            [skirmish.rest.util :refer [build-response build-response-body]]
+            [clojure.spec.alpha :as s]
+            [kinsky.client]
+            [slingshot.slingshot :refer [try+ throw+]]
+            [clojure.core.async :refer [chan go-loop close! <! sub]]))
+
+
+(def order-id (atom -1))
+
+
+(defn api-description [req]
+  (let [body {"/api/1/securities" "list all securities."
+              "/api/1/book"    "For all information about the book."}]
+    (build-response req body)))
+
+(defn subscribe-to-orders-publisher [exchange-api status-to-listen-on]
+  (let [ret (chan)
+        new-book-order-channel (:orders-publisher exchange-api)]
+    (doseq [status status-to-listen-on]
+      (sub new-book-order-channel status ret))
+    ret))
+
+
+(defn get-new-order-channel [exchange-apis sec-id status-to-listen-on]
+  (let [exchange (get exchange-apis sec-id)]
+    (subscribe-to-orders-publisher exchange status-to-listen-on)))
+
+
+(defn get-new-trades-channel [exchange-apis sec-id]
+  (let [exchange (get exchange-apis sec-id)
+        new-trades-channel (:trades-publisher exchange)
+        ret (chan)]
+    (sub new-trades-channel :sell ret)
+    (sub new-trades-channel :buy ret)
+    ret))
+
+
+(defn securities [exchange-apis]
+  (fn [request]
+    (build-response request {:securities (keys exchange-apis)})))
+
+
+(defn get-security [exchange-apis]
+  (fn [{{sec-id-str :sec-id} :route-params body :body :as request}]
+    (build-response request {:security {:sec-id sec-id-str}})))
+
+
+(defn chan-to-websocket [request input-chan ws-chan]
+  (on-close ws-chan (fn [status] (close! input-chan)))
+  (go-loop []
+    (when-let [e (<! input-chan)]
+      (send! ws-chan (build-response-body request e))
+      (recur))))
+
+
+(defn get-status-to-listen-on [{params :query-params}]
+  (let [param-status (:status params)
+        status (if (seq? param-status) param-status skirmish.order/order-status)]
+    (filter #(s/valid? :book.order/status %) status)))
+
+
+(defn listen-to-orders [exchange-apis]
+  (fn [{{sec-id-str :sec-id} :route-params :as request}]
+    (with-channel request ws-chan
+                  (let [status-to-listen-on (get-status-to-listen-on request)
+                        order-channel (get-new-order-channel exchange-apis (keyword sec-id-str) status-to-listen-on)]
+                    (chan-to-websocket request order-channel ws-chan)))))
+
+
+(defn listen-to-trades [exchange-apis]
+  (fn [{{sec-id-str :sec-id} :route-params :as request}]
+    (with-channel request ws-chan
+                  (let [order-channel (get-new-trades-channel exchange-apis (keyword sec-id-str))]
+                    (chan-to-websocket request order-channel ws-chan)))))
+
+
+(defn create-book [exchange-api]
+  (let [ret (atom {})
+        new-book-order-channel (subscribe-to-orders-publisher exchange-api skirmish.order/order-status)]
+    (go-loop [order (<! new-book-order-channel)]
+      (let [order-id (:book.order/id order)
+            to-remove (contains? #{:withdrawn :totally-fulfilled} (:book.order/status order))]
+        (println ret)
+        (swap! ret (fn [book]
+                     (if to-remove
+                       (dissoc book order-id)
+                       (assoc book order-id order)))))
+      (recur (<! new-book-order-channel)))
+    ret))
+
+(defn map-values [f m]
+  (into {} (for [[k v] m] [k (f v)])))
+
+(defn get-books [exchange-apis]
+  (let [books (map-values #(create-book %) exchange-apis)]
+    (fn [{{sec-id-str :sec-id} :route-params body :body :as request}]
+      (let [sec-id (keyword sec-id-str)]
+        (build-response request @(get books sec-id))))))
+
+
+(defn create-routes-for-bookkeeper [exchange-apis]
+  (let [get-book (get-books exchange-apis)]
+    (routes
+      (context "/api/1" []
+        (GET "/" [] api-description)
+        (GET "/securities" [] (securities exchange-apis))
+        (GET "/securities/:sec-id" [] (get-security exchange-apis))
+        (GET "/securities/:sec-id/trades" [] (listen-to-trades exchange-apis))
+        (GET "/securities/:sec-id/orders" [] (listen-to-orders exchange-apis))
+        (GET "/securities/:sec-id/book" [] get-book))
+      (not-found "not found"))))

src/skirmish/rest/core.clj

@@ -1,17 +1,23 @@
 (ns skirmish.rest.core
   (:require [org.httpkit.server :refer [run-server]]
             [compojure.handler :refer [api]]
-            [kinsky.client]
+            [skirmish.util :refer [block-thread]]
-            [skirmish.rest.exchange :refer [create-exchanges-api-for
+            [skirmish.middleware.core :refer [create-exchange-apis-for]]
-                                            create-routes]]))
+            [skirmish.rest.bookkeeper :refer [create-routes-for-bookkeeper]]
+            [skirmish.rest.exchange :refer [create-routes-for-exchange]]))
 
-(defn -main [config]
+
-  (let [webserver-config (:webserver config)
+(defn api-main [config exchange-apis]
-        exchange-apis (create-exchanges-api-for (:securities config))
+  (let [webserver-config (:api-server config)
-        kafka-producer (kinsky.client/producer {:bootstrap.servers (get-in config [:kafka :bootstrap.servers])}
+        api-routes (create-routes-for-exchange exchange-apis)]
-                                               (kinsky.client/keyword-serializer)
+    (println "Running server with bellow config")
-                                               (kinsky.client/edn-serializer))
+    (println webserver-config)
-        api-routes (create-routes exchange-apis kafka-producer)]
+    (run-server (api api-routes) webserver-config)))
+
+
+(defn bookkeeper-main [config exchange-apis]
+  (let [webserver-config (:api-bookkeeper config)
+        api-routes (create-routes-for-bookkeeper exchange-apis)]
     (println "Running server with bellow config")
     (println webserver-config)
     (run-server (api api-routes) webserver-config)))

src/skirmish/rest/exchange.clj

@@ -1,11 +1,10 @@
 (ns skirmish.rest.exchange
-  (:require [skirmish.exchange :refer [empty-book]]
+  (:require [skirmish.exchange.exchange :refer [empty-book create-exchange-for-security]]
             [org.httpkit.server :refer [run-server with-channel send! close on-close]]
             [compojure.route :refer [not-found]]
             [compojure.handler :refer [api]]
             [compojure.core :refer :all]
             [skirmish.order]  ;; for spec side effect
-            [skirmish.exchange :refer [create-exchange-for-security]]
             [skirmish.rest.util :refer [build-response build-response-body]]
             [clojure.spec.alpha :as s]
             [kinsky.client]
@@ -17,62 +16,6 @@
 (def order-id (atom -1))
 
 
-(defn api-description [req]
-  (let [body {"/api/1/securities" "list all securities."
-              "/api/1/book"    "For all information about the book."}]
-    (build-response req body)))
-
-
-;; the api should look like...
-;; trades ...
-;;    /:sec-id/trades/:trade-id
-;; book ...
-;;    /:sec-id/book/:order
-;;
-
-
-(defn create-exchange-api-for [sec-id]
-  (let [initial-book empty-book
-        incoming-order-channel (chan)
-        [trades-publisher new-book-order-publisher] (create-exchange-for-security initial-book incoming-order-channel)]
-    {:sec-id                 sec-id
-     :incoming-order-channel incoming-order-channel
-     :trades-publisher         trades-publisher
-     :new-book-order-publisher new-book-order-publisher}))
-
-(defn create-exchanges-api-for [security-definitions]
-  (let [f (fn [sec] [(:sec-id sec) (create-exchange-api-for (:sec-id sec))])]
-    (into {} (map f security-definitions))))
-
-
-(defn get-new-order-channel [exchange-apis sec-id]
-  (let [exchange (get exchange-apis sec-id)
-        new-book-order-channel (:new-book-order-publisher exchange)
-        ret (chan)]
-    (sub new-book-order-channel :sell ret)
-    (sub new-book-order-channel :buy ret)
-    ret))
-
-
-(defn get-new-trades-channel [exchange-apis sec-id]
-  (let [exchange (get exchange-apis sec-id)
-        new-trades-channel (:trades-publisher exchange)
-        ret (chan)]
-    (sub new-trades-channel :sell ret)
-    (sub new-trades-channel :buy ret)
-    ret))
-
-
-(defn securities [exchange-apis]
-  (fn [request]
-    (build-response request {:securities (keys exchange-apis)})))
-
-
-(defn get-security [exchange-apis]
-  (fn [{{sec-id-str :sec-id} :route-params body :body :as request}]
-    (build-response request {:security {:sec-id sec-id-str}})))
-
-
 (defn validate-order [asked-order sec-id]
   (let [now (System/currentTimeMillis)]
     (if (not (s/valid? :book.order/order asked-order))
@@ -98,12 +41,12 @@
     (throw+ {:error-type :4XX :status 404 :body {:error "Security not available."}})))
 
 
-(defn create-order [exchange-apis kafka-producer]
+(defn create-order [exchange-apis]
   (fn [{{sec-id-str :sec-id} :route-params body :body :as request}]
     (try+
       (let [asked-order (get-body body)
             exchange-api (get-exchange-api (keyword sec-id-str) exchange-apis)
-            incoming-order-channel (:incoming-order-channel exchange-api)
+            incoming-order-channel (:new-order-submitter exchange-api)
             order (validate-order asked-order (:sec-id exchange-api))]
         (>!! incoming-order-channel order)
         (build-response request order))
@@ -111,32 +54,9 @@
       (catch [:error-type :5XX] e (build-response request e)))))
 
 
-(defn chan-to-websocket [request input-chan ws-chan]
+(defn create-routes-for-exchange [exchange-apis]
-  (on-close ws-chan (fn [status] (close! input-chan)))
-  (go-loop []
-    (when-let [e (<! input-chan)]
-      (send! ws-chan (build-response-body request e))
-      (recur))))
-
-(defn listen-to-orders [exchange-apis]
-  (fn [{{sec-id-str :sec-id} :route-params :as request}]
-    (with-channel request ws-chan
-      (let [order-channel (get-new-order-channel exchange-apis (keyword sec-id-str))]
-        (chan-to-websocket request order-channel ws-chan)))))
-
-(defn listen-to-trades [exchange-apis]
-  (fn [{{sec-id-str :sec-id} :route-params :as request}]
-    (with-channel request ws-chan
-                  (let [order-channel (get-new-trades-channel exchange-apis (keyword sec-id-str))]
-                    (chan-to-websocket request order-channel ws-chan)))))
-
-(defn create-routes [exchange-apis kafka-producer]
   (routes
     (context "/api/1" []
-      (GET "/" [] api-description)
+      (POST "/securities/:sec-id" [] (create-order exchange-apis)))
-      (GET "/securities" [] (securities exchange-apis))
-      (GET "/securities/:sec-id" [] (get-security exchange-apis))
-      (POST "/securities/:sec-id" [] (create-order exchange-apis kafka-producer))
-      (GET "/securities/:sec-id/trades" [] (listen-to-trades exchange-apis))
-      (GET "/securities/:sec-id/orders" [] (listen-to-orders exchange-apis)))
     (not-found "not found")))
+

src/skirmish/rest/marketdata.clj

@@ -1 +0,0 @@
-(ns skirmish.rest.marketdata)

src/skirmish/util.clj

@@ -1,4 +1,5 @@
 (ns skirmish.util
+  (:require [clojure.core.async :as async])
   (:import [skirmish.PersistentSortedList]))
 
 (defn sorted-list [& keys]
@@ -7,3 +8,6 @@
 (defn sorted-list-by [comparator & keys]
   (skirmish.PersistentSortedList/create comparator (seq keys)))
 
+(defn block-thread []
+  "Block the current thread until chuck norris gets weaker."
+  (async/<!! (async/chan)))

test/skirmish/exchange/exchange_test.clj

@@ -1,11 +1,14 @@
-(ns skirmish.exchange-test
+(ns skirmish.exchange.exchange-test
   (:import [skirmish.PersistentSortedList])
   (:require [midje.sweet :refer :all]
-            [skirmish.exchange :refer :all]
+            [orchestra.spec.test :as st]
-            [skirmish.match :refer [create-buy-book
+            [skirmish.exchange.exchange :refer :all]
+            [skirmish.exchange.match :refer [create-buy-book
                                              create-sell-book
                                              match-order-in-book]]))
 
+(st/instrument)
+
 (def PersistentSortedList? (comp #(= % skirmish.PersistentSortedList) type))
 
 (facts "the exchange as an whole"
@@ -70,13 +73,21 @@
                book-4 => expt-book-4)))
 
 (facts "get-processed-orders"
-       (fact "ss"
+       (fact "bug numero uno"
              (let [sell-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 116}
                                               {:type :buy :quantity 8 :price 13 :book.order/id 117})
                    order {:type :sell :quantity 21 :price 13 :book.order/id 118}
                    [trades new-book leftover-order] (match-order-in-book sell-book order)
                    processed-order (get-processed-orders order leftover-order trades sell-book)]
-               processed-order => seq)));;;
+               processed-order => seq));;;
+       (fact "bug numero dos"
+             (let [order   {:type :sell, :quantity 10, :price 2, :book.order/id 1}
+                   leftover-order nil
+                   trades [#:book.order{:type :buy, :quantity 10, :price 3, :order-id 1, :book-id 0}]
+                   book  {:sell (create-sell-book)
+                          :buy (create-buy-book {:type :buy, :quantity 15, :price 3, :book.order/id 0})}
+                   ret (get-processed-orders order leftover-order trades (:buy book))]
+                (count ret) => 3)))
 
 
 

test/skirmish/exchange/match_test.clj

@@ -0,0 +1,116 @@
+(ns skirmish.exchange.match-test
+  (:require [midje.sweet :refer :all]
+            [skirmish.exchange.match :refer :all]))
+
+
+(def PersistentSortedList? (comp #(= % skirmish.PersistentSortedList) type))
+
+
+(facts "matching buy order in sell book"
+  (fact "Simplest case I."
+        (let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 101})
+              order {:type :buy :quantity 10 :price 2 :book.order/id 102}
+              [trades new-book] (match-order-in-book sell-book order)]
+          new-book => empty?
+          trades => [#:book.order{:quantity 10 :price 2 :book-id 101 :order-id 102 :type :sell}]))
+
+  (fact "Simplest case II."
+        (let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 103})
+              order {:type :buy :quantity 10 :price 3 :book.order/id 104}
+              [trades new-book] (match-order-in-book sell-book order)]
+          new-book => empty?
+          trades => [#:book.order{:quantity 10 :price 2 :book-id 103 :order-id 104 :type :sell}]))
+  (fact "Simplest case III."
+        (let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 105})
+              order {:type :buy :quantity 8 :price 2 :book.order/id 106}
+              [trades new-book] (match-order-in-book sell-book order)]
+          new-book => [{:type :sell :quantity 2 :price 2 :book.order/id 105}]
+          trades => [#:book.order{:quantity 8 :price 2 :book-id 105 :order-id 106 :type :sell}]))
+  (fact "Simplest case IV."
+        (let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 107} {:type :sell :quantity 8 :price 13 :book.order/id 108})
+              order {:type :buy :quantity 10 :price 13 :book.order/id 109}
+              [trades new-book] (match-order-in-book sell-book order)]
+          new-book => empty?
+          trades => [#:book.order{:quantity 2 :price 12 :book-id 107 :order-id 109 :type :sell}
+                     #:book.order{:quantity 8 :price 13 :book-id 108 :order-id 109 :type :sell}]))
+  (fact "Simplest case V."
+        (let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 113} {:type :sell :quantity 8 :price 13 :book.order/id 114})
+              order {:type :buy :quantity 9 :price 13 :book.order/id 115}
+              [trades new-book] (match-order-in-book sell-book order)]
+          new-book => (create-sell-book {:type :sell :quantity 1 :price 13 :book.order/id 114})
+          trades => [#:book.order{:quantity 2 :price 12 :book-id 113 :order-id 115 :type :sell}
+                     #:book.order{:quantity 7 :price 13 :book-id 114 :order-id 115 :type :sell}]))
+  (fact "Simplest case VI."
+        (let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 116} {:type :sell :quantity 8 :price 13 :book.order/id 117})
+              order {:type :buy :quantity 21 :price 13 :book.order/id 118}
+              [trades new-book leftover-order] (match-order-in-book sell-book order)]
+          new-book => empty?
+          trades => [#:book.order{:quantity 2 :price 12 :book-id 116 :order-id 118 :type :sell}
+                     #:book.order{:quantity 8 :price 13 :book-id 117 :order-id 118 :type :sell}]
+          leftover-order => {:type :buy :quantity 11 :price 13 :book.order/id 118}))
+  (fact "Simplest case VII."
+        (let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 116} {:type :sell :quantity 8 :price 13 :book.order/id 117})
+              order {:type :buy :quantity 21 :price 10 :book.order/id 118}
+              [trades new-book leftover-order] (match-order-in-book sell-book order)]
+          new-book => sell-book
+          trades => empty?
+          leftover-order => order)))
+
+
+(facts "matching sell order in buy book"
+  (fact "Simplest case I."
+        (let [buy-book (create-buy-book {:type :buy :quantity 10 :price 2 :book.order/id 101})
+              order {:type :sell :quantity 10 :price 2 :book.order/id 102}
+              [trades new-book] (match-order-in-book buy-book order)]
+          new-book => empty?
+          trades => [#:book.order{:quantity 10 :price 2 :book-id 101 :order-id 102 :type :buy}]))
+
+  (fact "Simplest case II."
+        (let [buy-book (create-buy-book {:type :buy :quantity 10 :price 4 :book.order/id 103})
+              order {:type :sell :quantity 10 :price 3 :book.order/id 104}
+              [trades new-book] (match-order-in-book buy-book order)]
+          new-book => []
+          trades => [#:book.order{:quantity 10 :price 4 :book-id 103 :order-id 104 :type :buy}]))
+
+  (fact "Simplest case III."
+        (let [buy-book (create-buy-book {:type :buy :quantity 10 :price 2 :book.order/id 105})
+              order {:type :sell :quantity 8 :price 2 :book.order/id 106}
+              [trades new-book] (match-order-in-book buy-book order)]
+          new-book => [{:type :buy :quantity 2 :price 2 :book.order/id 105}]
+          trades => [#:book.order{:quantity 8 :price 2 :book-id 105 :order-id 106 :type :buy}]))
+
+  (fact "Simplest case IV."
+        (let [buy-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 107}
+                                        {:type :buy :quantity 8 :price 13 :book.order/id 108})
+              order {:type :sell :quantity 10 :price 13 :book.order/id 109}
+              [trades new-book] (match-order-in-book buy-book order)]
+          new-book => []
+          trades => [#:book.order{:quantity 2 :price 14 :book-id 107 :order-id 109 :type :buy}
+                     #:book.order{:quantity 8 :price 13 :book-id 108 :order-id 109 :type :buy}]))
+
+  (fact "Simplest case V."
+        (let [buy-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 113}
+                                        {:type :buy :quantity 8 :price 13 :book.order/id 114})
+              order {:type :sell :quantity 9 :price 13 :book.order/id 115}
+              [trades new-book] (match-order-in-book buy-book order)]
+          new-book => (create-buy-book {:type :buy :quantity 1 :price 13 :book.order/id 114})
+          trades => [#:book.order{:quantity 2 :price 14 :book-id 113 :order-id 115 :type :buy}
+                     #:book.order{:quantity 7 :price 13 :book-id 114 :order-id 115 :type :buy}]))
+
+  (fact "Simplest case VI."
+        (let [sell-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 116}
+                                         {:type :buy :quantity 8 :price 13 :book.order/id 117})
+              order {:type :sell :quantity 21 :price 13 :book.order/id 118}
+              [trades new-book leftover-order] (match-order-in-book sell-book order)]
+          new-book => empty?
+          trades => [#:book.order{:quantity 2 :price 14 :book-id 116 :order-id 118 :type :buy}
+                     #:book.order{:quantity 8 :price 13 :book-id 117 :order-id 118 :type :buy}]
+          leftover-order => {:type :sell :quantity 11 :price 13 :book.order/id 118})))
+
+(facts "We should write tests for each bug found"
+  (fact "book type should always be book.PersistentSortedList"
+    (let [book (create-buy-book)
+          order {:type :sell :quantity 21 :price 13 :book.order/id 118}
+          [_ new-book _] (match-order-in-book book order)]
+      (count new-book) => 0
+      (type new-book) => skirmish.PersistentSortedList)))

test/skirmish/match_test.clj

@@ -1,124 +0,0 @@
-(ns skirmish.match-test
-  (:require [midje.sweet :refer :all]
-            [skirmish.match :refer :all]))
-
-(def PersistentSortedList? (comp #(= % skirmish.PersistentSortedList) type))
-
-
-(facts "matching buy order in sell book"
-  (fact "Simplest case I."
-    (let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 101})
-          order {:type :buy :quantity 10 :price 2 :book.order/id 102}
-          [trades new-book] (match-order-in-book sell-book order)]
-      new-book => empty?
-      trades => [#:book.order{:quantity 10 :price 2 :book-id 101 :order-id 102 :type :sell}]))
-
-  (fact "Simplest case II."
-        (let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 103})
-              order {:type :buy :quantity 10 :price 3 :book.order/id 104}
-              [trades new-book] (match-order-in-book sell-book order)]
-          new-book => empty?
-          trades => [#:book.order{:quantity 10 :price 2 :book-id 103 :order-id 104 :type :sell}]))
-  (fact "Simplest case III."
-        (let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 105})
-              order {:type :buy :quantity 8 :price 2 :book.order/id 106}
-              [trades new-book] (match-order-in-book sell-book order)]
-          new-book => [{:type :sell :quantity 2 :price 2 :book.order/id 105}]
-          trades => [#:book.order{:quantity 8 :price 2 :book-id 105 :order-id 106 :type :sell}]))
-  (fact "Simplest case IV."
-        (let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 107} {:type :sell :quantity 8 :price 13 :book.order/id 108})
-              order {:type :buy :quantity 10 :price 13 :book.order/id 109}
-              [trades new-book] (match-order-in-book sell-book order)]
-          new-book => empty?
-          trades => [#:book.order{:quantity 2 :price 12 :book-id 107 :order-id 109 :type :sell}
-                     #:book.order{:quantity 8 :price 13 :book-id 108 :order-id 109 :type :sell}]))
-  (fact "Simplest case V."
-        (let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 113} {:type :sell :quantity 8 :price 13 :book.order/id 114})
-              order {:type :buy :quantity 9 :price 13 :book.order/id 115}
-              [trades new-book] (match-order-in-book sell-book order)]
-          new-book => (create-sell-book {:type :sell :quantity 1 :price 13 :book.order/id 114})
-          trades => [#:book.order{:quantity 2 :price 12 :book-id 113 :order-id 115 :type :sell}
-                     #:book.order{:quantity 7 :price 13 :book-id 114 :order-id 115 :type :sell}]))
-  (fact "Simplest case VI."
-        (let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 116} {:type :sell :quantity 8 :price 13 :book.order/id 117})
-              order {:type :buy :quantity 21 :price 13 :book.order/id 118}
-              [trades new-book leftover-order] (match-order-in-book sell-book order)]
-          new-book => empty?
-          trades => [#:book.order{:quantity 2 :price 12 :book-id 116 :order-id 118 :type :sell}
-                     #:book.order{:quantity 8 :price 13 :book-id 117 :order-id 118 :type :sell}]
-          leftover-order => {:type :buy :quantity 11 :price 13 :book.order/id 118}))
-  (fact "Simplest case VII."
-        (let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 116} {:type :sell :quantity 8 :price 13 :book.order/id 117})
-              order {:type :buy :quantity 21 :price 10 :book.order/id 118}
-              [trades new-book leftover-order] (match-order-in-book sell-book order)]
-          new-book => sell-book
-          trades => empty?
-          leftover-order => order)))
-
-
-(facts "matching sell order in buy book"
-  (fact "Simplest case I."
-    (let [buy-book (create-buy-book {:type :buy :quantity 10 :price 2 :book.order/id 101})
-          order {:type :sell :quantity 10 :price 2 :book.order/id 102}
-          [trades new-book] (match-order-in-book buy-book order)]
-      new-book => empty?
-      trades => [#:book.order{:quantity 10 :price 2 :book-id 101 :order-id 102 :type :buy}]))
-
-  (fact "Simplest case II."
-    (let [buy-book (create-buy-book {:type :buy :quantity 10 :price 4 :book.order/id 103})
-          order {:type :sell :quantity 10 :price 3 :book.order/id 104}
-          [trades new-book] (match-order-in-book buy-book order)]
-      new-book => []
-      trades => [#:book.order{:quantity 10 :price 4 :book-id 103 :order-id 104 :type :buy}]))
-
-  (fact "Simplest case III."
-    (let [buy-book (create-buy-book {:type :buy :quantity 10 :price 2 :book.order/id 105})
-          order {:type :sell :quantity 8 :price 2 :book.order/id 106}
-          [trades new-book] (match-order-in-book buy-book order)]
-      new-book => [{:type :buy :quantity 2 :price 2 :book.order/id 105}]
-      trades => [#:book.order{:quantity 8 :price 2 :book-id 105 :order-id 106 :type :buy}]))
-
-  (fact "Simplest case IV."
-    (let [buy-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 107}
-                                    {:type :buy :quantity 8 :price 13 :book.order/id 108})
-          order {:type :sell :quantity 10 :price 13 :book.order/id 109}
-          [trades new-book] (match-order-in-book buy-book order)]
-      new-book => []
-      trades => [#:book.order{:quantity 2 :price 14 :book-id 107 :order-id 109 :type :buy}
-                 #:book.order{:quantity 8 :price 13 :book-id 108 :order-id 109 :type :buy}]))
-
-  (fact "Simplest case V."
-    (let [buy-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 113}
-                                    {:type :buy :quantity 8 :price 13 :book.order/id 114})
-          order {:type :sell :quantity 9 :price 13 :book.order/id 115}
-          [trades new-book] (match-order-in-book buy-book order)]
-      new-book => (create-buy-book {:type :buy :quantity 1 :price 13 :book.order/id 114})
-      trades => [#:book.order{:quantity 2 :price 14 :book-id 113 :order-id 115 :type :buy}
-                 #:book.order{:quantity 7 :price 13 :book-id 114 :order-id 115 :type :buy}]))
-
-  (fact "Simplest case VI."
-    (let [sell-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 116}
-                                     {:type :buy :quantity 8 :price 13 :book.order/id 117})
-          order {:type :sell :quantity 21 :price 13 :book.order/id 118}
-          [trades new-book leftover-order] (match-order-in-book sell-book order)]
-      new-book => empty?
-      trades => [#:book.order{:quantity 2 :price 14 :book-id 116 :order-id 118 :type :buy}
-                 #:book.order{:quantity 8 :price 13 :book-id 117 :order-id 118 :type :buy}]
-      leftover-order => {:type :sell :quantity 11 :price 13 :book.order/id 118})))
-
-(facts "We should write tests for each bug found"
-       (fact "book type should always be book.PersistentSortedList"
-             (let [book (create-buy-book)
-                   order {:type :sell :quantity 21 :price 13 :book.order/id 118}
-                   [_ new-book _] (match-order-in-book book order)]
-               (count new-book) => 0
-               (type new-book) => skirmish.PersistentSortedList)))
-       ;(fact "should not fuck up"
-       ;      (let [order-1 {:type :buy :quantity 4 :price 4 :book.order/id 5}
-       ;            order-2 {:type :sell :quantity 2 :price 3 :book.order/id 6}
-       ;            book (create-buy-book order-1)
-       ;            expt-trade #:book.order{}
-       ;            [trades new-book new-order] (match-order-in-book book order-2)]
-       ;        trades => 1
-       ;        new-order => nil?
-       ;        new-book => {})))

test/skirmish/middleware/test_core.clj

@@ -0,0 +1,13 @@
+(ns skirmish.middleware.test-core
+  (:require [midje.sweet :refer :all]
+            [skirmish.middleware.core :refer :all]))
+
+(defn all? [xf]
+  #(every? true? (sequence xf %)))
+
+(facts "we should have a async/chan base middleware for developing."
+  (fact "It should exchanges for each security in the config"
+    (let [config {:securities [{:sec-id :molly} {:sec-id :pepper}]}
+          exchanges (create-exchange-apis-for config)]
+      (keys exchanges) => [:molly :pepper]
+      (vals exchanges) => #(every? map? %))))