didier/skirmish
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'middleware' into 'master'

Browse Source 
Middleware

See merge request didiercrunch/skirmish!1
This commit was merged in pull request #1.
This commit is contained in:
didier amyot committed 2018-07-15 22:28:11 +00:00
2 parents 7fbb3484a1 27b4fd5727
commit 130a8b948e
23 files changed
+493 -345
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+1
View File
@@ -13,3 +13,4 @@ pom.xml.asc
book.iml
skirmish.iml
full-stack
fixtures
README.md
+20 -13
View File
@@ -6,29 +6,36 @@ A stock exchange simulator. The scope of this project is not yet defined but it
allow end user(s?) to exchange securities via the common exchange rules.
## Installation
Okay, this should be easy but it is hard...
## Getting Started
### Development
In development, you need nothing else than [clojure](https://clojure.org/)
and [leiningen](https://leiningen.org/). The easiest way to install clojure is
to make sure you have java (8 and above) and install leiningen.
### Production
You will need plenty of things, the majority of them are yet to define.
But, a good kafka deployment will be required. Probably a database,
hopefully datomic.
### Configuration
All configuration are set in the `resources/config.edn` file.
Configuration uses [aero](https://github.com/juxt/aero).
Configuration uses [aero](https://github.com/juxt/aero). The configuration
file should be self documented.
### Technologies
#### Server
* clojure
* kafka
* [clojure](https://clojure.org/) for programming language
* [kafka](http://kafka.apache.org/) for message broker/database
#### In the browser
* https://aftertheflood.com/projects/sparks/
* http://metricsgraphicsjs.org/
## Usage
FIXME: explanation
$ java -jar book-0.1.0-standalone.jar [args]
* [clojure script](https://clojurescript.org/) for programming language
* [sparks](https://aftertheflood.com/projects/sparks/) for trivial charts
* [metricsgraphicsjs](http://metricsgraphicsjs.org/) for complex charts
docker-compose.yml
+12 -1
View File
@@ -154,7 +154,7 @@ services:
skirmish-api:
image: skirmish
command: ["lein", "run", "webserver"]
command: ["lein", "run", "api-server"]
depends_on:
- kafka-topics-ui # basically, we need all kafka
environment:
@@ -162,3 +162,14 @@ services:
KAFKA_BOOTSTRAP_SERVERS: "kafka1:9092"
ports:
- "1602:1602"
skirmish-exchange-core:
image: skirmish
command: ["lein", "run", "exchange-core"]
depends_on:
- skirmish-api # basically, we need all kafka
environment:
PORT: 1602 # https://en.wikipedia.org/wiki/Euronext_Amsterdam
KAFKA_BOOTSTRAP_SERVERS: "kafka1:9092"
ports:
- "1602:1602"
project.clj
+1
View File
@@ -19,6 +19,7 @@
[slingshot "0.12.2"]
[spootnik/kinsky "0.1.22"]
[org.clojars.fossi/kafka-streams-clojure "0.1.0-SNAPSHOT"] ;; JEAH!
[orchestra "2017.11.12-1"]
[cheshire "5.8.0"]]
:java-source-paths ["src/java"]
:main ^:skip-aot skirmish.core
resources/config.edn
+20 -3
View File
@@ -1,20 +1,37 @@
{
;; Specify which middleware to use, it will default to :chan.
:middleware :kafka
;; If set to true, orchestra will instrument all the spec'ed functions
:instrument-spec true
;; all the kafka configuration goes bellow.
:kafka
{:bootstrap.servers #or [#env KAFKA_BOOTSTRAP_SERVERS "localhost:9092"]
:topics-prefix "skirmish-01-"}
:webserver ;; possible config from http-kit's run-server
{:port #long #or [#env PORT 8080]
;; possible config from http-kit's run-server that will affect the api-server
:api-server
{:port #long #or [#env API_SERVER_PORT 1602]
:ip "0.0.0.0"
:max-body 524288 ;; 512kb, small but enough
:max-ws 524288}
:securities ;; list of tradable securities
;; possible config from http-kit's run-server that will affect the api-bookkeeper
:api-bookkeeper
{:port #long #or [#env API_SERVER_PORT 1603]
:ip "0.0.0.0"
:max-body 524288 ;; 512kb, small but enough
:max-ws 524288}
;; list of tradable securities
:securities
[{:sec-id :foo}
{:sec-id :bar}
{:sec-id :toto}
{:sec-id :tata}]
;; stupid parameters about benchmarking tools
:benchmark {
:number-of-requests-per-securities 30}}
src/skirmish/cli/benchmark.clj
+1 -1
View File
@@ -94,7 +94,7 @@
(println "status map: " (:status results))
(pp-durations (:durations results)))
(defn -main [config]
(defn -main [config exchange-apis]
(let [benchmark-config (:benchmark config)
requests (-> benchmark-config :number-of-requests-per-securities (make-requests (:securities config)))
orders (create-orders-channel (:securities config))
src/skirmish/cli/exchange.clj
+2 -2
View File
@@ -1,7 +1,7 @@
(ns skirmish.cli.exchange
(:require [skirmish.match :refer [sell?]]
(:require [skirmish.exchange.match :refer [sell?]]
[skirmish.util :refer :all]
[skirmish.exchange :refer [empty-book create-exchange-for-security]]
[skirmish.exchange.exchange :refer [empty-book create-exchange-for-security]]
[skirmish.order] ;; for spec side effect
[slingshot.slingshot :refer [try+ throw+]]
[clojure.edn :as edn]
src/skirmish/core.clj
+38 -10
View File
@@ -2,18 +2,46 @@
(:require [skirmish.cli.core]
[skirmish.rest.core]
[skirmish.cli.benchmark]
[skirmish.kafka.exchange-simulator]
[aero.core :refer (read-config)])
[skirmish.exchange.core]
[skirmish.middleware.core :refer [create-exchange-apis-for]]
[aero.core :refer (read-config)]
[orchestra.spec.test :as st]
[clojure.core.async :as async])
(:gen-class))
(def apps {:webserver skirmish.rest.core/-main
:benchmark skirmish.cli.benchmark/-main
:kafka-exchange-simulator skirmish.kafka.exchange-simulator/-main})
(def targets ;; a target is a function that blocks the thread and takes two
;; arguments: the config map and the "exchange-api"
{:api-server skirmish.rest.core/api-main
:exchange-core skirmish.exchange.core/-main ;; should be skirmish.exchange.core/api-main
:api-bookkeeper skirmish.rest.core/bookkeeper-main})
;:benchmark skirmish.cli.benchmark/-main})
(defn run-all-targets-in-separate-threads [target-fns config exchange-apis]
(println "running multiple targets. This is okay in dev but horrible in production")
(println target-fns)
(let [threads (map #(async/thread (% config exchange-apis)) target-fns)
finished-threads (async/merge threads)]
(loop [finished-thread (async/<!! finished-threads)]
(when (some? finished-thread)
(recur (async/<!! finished-threads))))))
(defn run-all-targets [target-fns config]
(let [exchange-apis (create-exchange-apis-for config)]
(if (= (count target-fns) 1)
((first target-fns) config exchange-apis)
(run-all-targets-in-separate-threads target-fns config exchange-apis))))
(defn -main [& argv]
(let [config (-> "config.edn" clojure.java.io/resource read-config)
target (-> argv first keyword)
target-fn (get apps target)]
(if target-fn
(apply target-fn [config])
(println "target" target "not found"))))
xf (comp (map keyword) (map targets) (filter some?))
target-fns (sequence xf argv)]
(if (true? (:instrument-spec config))
(st/instrument))
(if (seq target-fns)
(run-all-targets target-fns config)
(do
(println argv)
(println "target" target-fns "not found")
(println "all known targets are...")
(println (keys targets))))))
src/skirmish/exchange/core.clj
+22
View File
@@ -0,0 +1,22 @@
(ns skirmish.exchange.core
(:require [skirmish.util :refer [block-thread]]
[skirmish.middleware.core :refer [create-exchange-apis-for]]
[skirmish.exchange.exchange :refer [create-exchange-for-security empty-book]]
[kinsky.async]
[kinsky.client]))
(defn create-exchange-for [exchange-api]
(let [initial-book empty-book
new-order-publisher (:new-order-publisher exchange-api)
trades-submitter (:trades-submitter exchange-api)
orders-submitter (:orders-submitter exchange-api)]
(create-exchange-for-security initial-book
new-order-publisher
trades-submitter
orders-submitter)))
(defn -main [config exchange-apis]
(doseq [{sec-id :sec-id} (:securities config)]
(create-exchange-for (get exchange-apis sec-id)))
(block-thread))
src/skirmish/exchange.clj → src/skirmish/exchange/exchange.clj
+44 -33
View File
@@ -1,9 +1,10 @@
(ns skirmish.exchange
(:require [skirmish.match :refer [sell?
create-buy-book
create-sell-book
match-order-in-book]]
(ns skirmish.exchange.exchange
(:require [skirmish.exchange.match :refer [sell?
create-buy-book
create-sell-book
match-order-in-book]]
[skirmish.util :refer :all]
[clojure.spec.alpha :as s]
[clojure.core.async :refer [chan go >! <! pub] :as async]))
@@ -13,6 +14,18 @@
:buy (create-buy-book)
:id 0})
(defn get-reference-book [order book]
"The reference book is the book where the order will be fulfilled"
(if (sell? order)
(:buy book)
(:sell book)))
(defn get-same-type-order [order book]
"the same type order book is the book where the order will be added if not fulfilled"
(if (sell? order)
(:sell book)
(:buy book)))
;; Bellow are all the functions to match the received order in the correct book
(defn process-abstract [order reference-book same-type-book]
(let [[trades new-reference-book leftover-order] (match-order-in-book reference-book order)
@@ -44,23 +57,23 @@
(let [book-id (:book.order/book-id trade)
f #(= book-id (:book.order/id %))
ret-order (first (filter f book))
new-quantity (- (:quantity ret-order 10) (:book.order/quantity trade));; (- (:quantity ret-order) (:book.order/quantity trade))
new-quantity (- (:quantity ret-order) (:book.order/quantity trade));; (- (:quantity ret-order) (:book.order/quantity trade))
totally-fulfilled? (= (:book.order/quantity trade) (:quantity ret-order))]
(into ret-order {:status (if totally-fulfilled? :totally-fulfilled :partially-fulfilled)
:status-dual-id (:book.order/order-id trade)
:quantity new-quantity})));;;;
(into ret-order {:book.order/status (if totally-fulfilled? :totally-fulfilled :partially-fulfilled)
:book.order/status-dual-id (:book.order/order-id trade)
:quantity new-quantity})))
(defn get-intermediate-order [order trade]
(let [new-order-quantity (- (:quantity order) (:book.order/quantity trade))
totally-fulfilled? (= 0 new-order-quantity)
dual-id (:book.order/book-id trade)]
(into order {:status (if totally-fulfilled? :totally-fulfilled :partially-fulfilled)
:status-dual-id dual-id
(into order {:book.order/status (if totally-fulfilled? :totally-fulfilled :partially-fulfilled)
:book.order/status-dual-id dual-id
:quantity new-order-quantity})))
(defn get-processed-orders-in-trades [order book trades]
(defn get-processed-orders-in-trades[order book trades]
(let [trades-orders (sequence (map #(retrieve-new-book-order-from-trade % book)) trades)
processed-orders (rest (reductions get-intermediate-order order trades))]
(interleave trades-orders processed-orders)))
@@ -68,33 +81,31 @@
(defn get-processed-orders [order leftover-order trades book]
"Computes all the intermediary orders that took place during
a the trade."
(let [ret [(assoc order :status :inserted)]
a the trade.
The interal logic is very complicated. Probably too complicated."
(let [ret [(assoc order :book.order/status :received)]
ret (into ret (get-processed-orders-in-trades order book trades))
ret (if (seq leftover-order)
(conj ret (assoc leftover-order :status :inserted-into-book))
leftover-order)]
(conj ret (assoc leftover-order :book.order/status :inserted-into-book))
ret)]
ret))
(s/fdef get-processed-orders
:ret (s/* :book.order/order))
;; bellow are the really interesting functions.
(defn create-exchange-for-security [initial-book incoming-order-channel]
;; output - trades channel, pub channel
;; - orders channel, pub channel
(let [trades-channel (chan)
outcoming-orders-channel (chan)]
(async/go
(loop [order (<! incoming-order-channel) book initial-book]
(println "received an order! " order)
(let [[new-book new-order new-trades] (process-order book order)
orders (get-processed-orders order new-order new-trades book)
orders-copy (async/onto-chan outcoming-orders-channel orders false)
trades-copy (async/onto-chan trades-channel new-trades false)]
(<! orders-copy)
(<! trades-copy)
(recur (<! incoming-order-channel) new-book))))
[(pub trades-channel :book.order/type) (pub outcoming-orders-channel :type)]))
;; bellow are the really interesting functions; aka public functions.
(defn create-exchange-for-security [initial-book new-order-publisher trades-submitter orders-submitter]
(async/go
(loop [order (<! new-order-publisher) book initial-book]
(let [[new-book new-order new-trades] (process-order book order)
orders (get-processed-orders order new-order new-trades (get-reference-book order book))
orders-copy (async/onto-chan orders-submitter orders false)
trades-copy (async/onto-chan trades-submitter new-trades false)]
(<! orders-copy)
(<! trades-copy)
(recur (<! new-order-publisher) new-book)))))
src/skirmish/match.clj → src/skirmish/exchange/match.clj
+1 -1
View File
@@ -1,4 +1,4 @@
(ns skirmish.match
(ns skirmish.exchange.match
(:require [skirmish.util :refer [sorted-list-by]]
[clojure.spec.alpha :as s]
[clojure.edn :as edn]))
src/skirmish/kafka_exchange.clj
-41
View File
@@ -1,41 +0,0 @@
(ns skirmish.kafka-exchange
(:require [kafka-streams-clojure.api :as api])
(:import [org.apache.kafka.clients.producer KafkaProducer ProducerRecord]
[org.apache.kafka.streams StreamsConfig KafkaStreams]
[org.apache.kafka.streams.kstream KStreamBuilder]))
(defn -main []
(def xform (comp (filter (fn [[k v]] (string? v)))
(map (fn [[k v]] [v k]))
(filter (fn [[k v]] (= "foo" v)))))
(def builder (KStreamBuilder.))
(def kstream (-> builder
(.stream (into-array String ["tset"]))
(api/transduce-kstream xform)
(.to "test")))
(def kafka-streams
(KafkaStreams. builder
(StreamsConfig. {StreamsConfig/APPLICATION_ID_CONFIG "test-app-id"
StreamsConfig/BOOTSTRAP_SERVERS_CONFIG "localhost:9092"
StreamsConfig/KEY_SERDE_CLASS_CONFIG org.apache.kafka.common.serialization.Serdes$StringSerde
StreamsConfig/VALUE_SERDE_CLASS_CONFIG org.apache.kafka.common.serialization.Serdes$StringSerde})))
(.start kafka-streams)
(def producer (KafkaProducer. {"bootstrap.servers" "localhost:9092"
"acks" "all"
"retries" "0"
"key.serializer" "org.apache.kafka.common.serialization.StringSerializer"
"value.serializer" "org.apache.kafka.common.serialization.StringSerializer"}))
@(.send producer (ProducerRecord. "tset" "foo" "bar"))
;; Observe message come across topic "test" via kafka-console-consumer
@(.send producer (ProducerRecord. "tset" "baz" "quux"))
;; Observe message does not come across topic "test" via kafka-console-consumer
(.close producer)
(.close kafka-streams))
src/skirmish/middleware/core.clj
+27
View File
@@ -0,0 +1,27 @@
(ns skirmish.middleware.core
(:require [clojure.core.async :as async]
[skirmish.exchange.exchange :refer [empty-book create-exchange-for-security]] ;; remove this import.
[kinsky.async]
[kinsky.client]))
(defn create-chan-based-exchange-api-for [sec-id]
(let [new-order (async/chan)
trades (async/chan)
orders (async/chan)]
{:sec-id sec-id
:new-order-publisher new-order ;; anything published here will be processed as
:new-order-submitter new-order
:trades-publisher (async/pub trades :book.order/type)
:trades-submitter trades
:orders-publisher (async/pub orders :book.order/status)
:orders-submitter orders}))
(def api-creators {})
(defn create-exchange-apis-for [config]
;; todo: spec this function
(let [security-definitions (map :sec-id (:securities config))
create-exchange-api-for (get api-creators (:middleware config) create-chan-based-exchange-api-for)
f (fn [sec] [sec (create-exchange-api-for sec)])
ret (into {} (map f security-definitions))]
ret))
src/skirmish/order.clj
+12 -11
View File
@@ -7,18 +7,19 @@
;; :type one of :sell :buy
;; ::id maybe an id
(s/def :book.order/type #{:buy :sell})
(s/def :book.order/price pos-int?)
(s/def :book.order/quantity pos-int?)
(s/def :book.order/id pos-int?)
(s/def :book.order/status #{:inserted ;; maybe called acknklowdlge.
:partially-matched
:totally-matched
:inserted-into-book
:partially-fulfilled
:totally-fulfilled
:withdrawn}) ;; for instance the order has been canceled
(def order-status #{:received ;; maybe called acknowledged.
:partially-matched
:totally-matched
:inserted-into-book
:partially-fulfilled
:totally-fulfilled
:withdrawn}) ;; for instance the order has been canceled)
(s/def :book.order/type #{:buy :sell})
(s/def :book.order/price nat-int?)
(s/def :book.order/quantity nat-int?)
(s/def :book.order/id nat-int?)
(s/def :book.order/status order-status)
(s/def :book.order/status-dual-id :book.order/id)
src/skirmish/rest/bookkeeper.clj
+118
View File
@@ -0,0 +1,118 @@
(ns skirmish.rest.bookkeeper
(:require [skirmish.exchange.exchange :refer [empty-book create-exchange-for-security]]
[org.httpkit.server :refer [run-server with-channel send! close on-close]]
[compojure.route :refer [not-found]]
[compojure.handler :refer [api]]
[compojure.core :refer :all]
[skirmish.order] ;; for spec side effect
[skirmish.rest.util :refer [build-response build-response-body]]
[clojure.spec.alpha :as s]
[kinsky.client]
[slingshot.slingshot :refer [try+ throw+]]
[clojure.core.async :refer [chan go-loop close! <! sub]]))
(def order-id (atom -1))
(defn api-description [req]
(let [body {"/api/1/securities" "list all securities."
"/api/1/book" "For all information about the book."}]
(build-response req body)))
(defn subscribe-to-orders-publisher [exchange-api status-to-listen-on]
(let [ret (chan)
new-book-order-channel (:orders-publisher exchange-api)]
(doseq [status status-to-listen-on]
(sub new-book-order-channel status ret))
ret))
(defn get-new-order-channel [exchange-apis sec-id status-to-listen-on]
(let [exchange (get exchange-apis sec-id)]
(subscribe-to-orders-publisher exchange status-to-listen-on)))
(defn get-new-trades-channel [exchange-apis sec-id]
(let [exchange (get exchange-apis sec-id)
new-trades-channel (:trades-publisher exchange)
ret (chan)]
(sub new-trades-channel :sell ret)
(sub new-trades-channel :buy ret)
ret))
(defn securities [exchange-apis]
(fn [request]
(build-response request {:securities (keys exchange-apis)})))
(defn get-security [exchange-apis]
(fn [{{sec-id-str :sec-id} :route-params body :body :as request}]
(build-response request {:security {:sec-id sec-id-str}})))
(defn chan-to-websocket [request input-chan ws-chan]
(on-close ws-chan (fn [status] (close! input-chan)))
(go-loop []
(when-let [e (<! input-chan)]
(send! ws-chan (build-response-body request e))
(recur))))
(defn get-status-to-listen-on [{params :query-params}]
(let [param-status (:status params)
status (if (seq? param-status) param-status skirmish.order/order-status)]
(filter #(s/valid? :book.order/status %) status)))
(defn listen-to-orders [exchange-apis]
(fn [{{sec-id-str :sec-id} :route-params :as request}]
(with-channel request ws-chan
(let [status-to-listen-on (get-status-to-listen-on request)
order-channel (get-new-order-channel exchange-apis (keyword sec-id-str) status-to-listen-on)]
(chan-to-websocket request order-channel ws-chan)))))
(defn listen-to-trades [exchange-apis]
(fn [{{sec-id-str :sec-id} :route-params :as request}]
(with-channel request ws-chan
(let [order-channel (get-new-trades-channel exchange-apis (keyword sec-id-str))]
(chan-to-websocket request order-channel ws-chan)))))
(defn create-book [exchange-api]
(let [ret (atom {})
new-book-order-channel (subscribe-to-orders-publisher exchange-api skirmish.order/order-status)]
(go-loop [order (<! new-book-order-channel)]
(let [order-id (:book.order/id order)
to-remove (contains? #{:withdrawn :totally-fulfilled} (:book.order/status order))]
(println ret)
(swap! ret (fn [book]
(if to-remove
(dissoc book order-id)
(assoc book order-id order)))))
(recur (<! new-book-order-channel)))
ret))
(defn map-values [f m]
(into {} (for [[k v] m] [k (f v)])))
(defn get-books [exchange-apis]
(let [books (map-values #(create-book %) exchange-apis)]
(fn [{{sec-id-str :sec-id} :route-params body :body :as request}]
(let [sec-id (keyword sec-id-str)]
(build-response request @(get books sec-id))))))
(defn create-routes-for-bookkeeper [exchange-apis]
(let [get-book (get-books exchange-apis)]
(routes
(context "/api/1" []
(GET "/" [] api-description)
(GET "/securities" [] (securities exchange-apis))
(GET "/securities/:sec-id" [] (get-security exchange-apis))
(GET "/securities/:sec-id/trades" [] (listen-to-trades exchange-apis))
(GET "/securities/:sec-id/orders" [] (listen-to-orders exchange-apis))
(GET "/securities/:sec-id/book" [] get-book))
(not-found "not found"))))
src/skirmish/rest/core.clj
+17 -11
View File
@@ -1,17 +1,23 @@
(ns skirmish.rest.core
(:require [org.httpkit.server :refer [run-server]]
[compojure.handler :refer [api]]
[kinsky.client]
[skirmish.rest.exchange :refer [create-exchanges-api-for
create-routes]]))
[skirmish.util :refer [block-thread]]
[skirmish.middleware.core :refer [create-exchange-apis-for]]
[skirmish.rest.bookkeeper :refer [create-routes-for-bookkeeper]]
[skirmish.rest.exchange :refer [create-routes-for-exchange]]))
(defn -main [config]
(let [webserver-config (:webserver config)
exchange-apis (create-exchanges-api-for (:securities config))
kafka-producer (kinsky.client/producer {:bootstrap.servers (get-in config [:kafka :bootstrap.servers])}
(kinsky.client/keyword-serializer)
(kinsky.client/edn-serializer))
api-routes (create-routes exchange-apis kafka-producer)]
(defn api-main [config exchange-apis]
(let [webserver-config (:api-server config)
api-routes (create-routes-for-exchange exchange-apis)]
(println "Running server with bellow config")
(println webserver-config)
(run-server (api api-routes) webserver-config)))
(run-server (api api-routes) webserver-config)))
(defn bookkeeper-main [config exchange-apis]
(let [webserver-config (:api-bookkeeper config)
api-routes (create-routes-for-bookkeeper exchange-apis)]
(println "Running server with bellow config")
(println webserver-config)
(run-server (api api-routes) webserver-config)))
src/skirmish/rest/exchange.clj
+6 -86
View File
@@ -1,11 +1,10 @@
(ns skirmish.rest.exchange
(:require [skirmish.exchange :refer [empty-book]]
(:require [skirmish.exchange.exchange :refer [empty-book create-exchange-for-security]]
[org.httpkit.server :refer [run-server with-channel send! close on-close]]
[compojure.route :refer [not-found]]
[compojure.handler :refer [api]]
[compojure.core :refer :all]
[skirmish.order] ;; for spec side effect
[skirmish.exchange :refer [create-exchange-for-security]]
[skirmish.rest.util :refer [build-response build-response-body]]
[clojure.spec.alpha :as s]
[kinsky.client]
@@ -17,62 +16,6 @@
(def order-id (atom -1))
(defn api-description [req]
(let [body {"/api/1/securities" "list all securities."
"/api/1/book" "For all information about the book."}]
(build-response req body)))
;; the api should look like...
;; trades ...
;; /:sec-id/trades/:trade-id
;; book ...
;; /:sec-id/book/:order
;;
(defn create-exchange-api-for [sec-id]
(let [initial-book empty-book
incoming-order-channel (chan)
[trades-publisher new-book-order-publisher] (create-exchange-for-security initial-book incoming-order-channel)]
{:sec-id sec-id
:incoming-order-channel incoming-order-channel
:trades-publisher trades-publisher
:new-book-order-publisher new-book-order-publisher}))
(defn create-exchanges-api-for [security-definitions]
(let [f (fn [sec] [(:sec-id sec) (create-exchange-api-for (:sec-id sec))])]
(into {} (map f security-definitions))))
(defn get-new-order-channel [exchange-apis sec-id]
(let [exchange (get exchange-apis sec-id)
new-book-order-channel (:new-book-order-publisher exchange)
ret (chan)]
(sub new-book-order-channel :sell ret)
(sub new-book-order-channel :buy ret)
ret))
(defn get-new-trades-channel [exchange-apis sec-id]
(let [exchange (get exchange-apis sec-id)
new-trades-channel (:trades-publisher exchange)
ret (chan)]
(sub new-trades-channel :sell ret)
(sub new-trades-channel :buy ret)
ret))
(defn securities [exchange-apis]
(fn [request]
(build-response request {:securities (keys exchange-apis)})))
(defn get-security [exchange-apis]
(fn [{{sec-id-str :sec-id} :route-params body :body :as request}]
(build-response request {:security {:sec-id sec-id-str}})))
(defn validate-order [asked-order sec-id]
(let [now (System/currentTimeMillis)]
(if (not (s/valid? :book.order/order asked-order))
@@ -98,12 +41,12 @@
(throw+ {:error-type :4XX :status 404 :body {:error "Security not available."}})))
(defn create-order [exchange-apis kafka-producer]
(defn create-order [exchange-apis]
(fn [{{sec-id-str :sec-id} :route-params body :body :as request}]
(try+
(let [asked-order (get-body body)
exchange-api (get-exchange-api (keyword sec-id-str) exchange-apis)
incoming-order-channel (:incoming-order-channel exchange-api)
incoming-order-channel (:new-order-submitter exchange-api)
order (validate-order asked-order (:sec-id exchange-api))]
(>!! incoming-order-channel order)
(build-response request order))
@@ -111,32 +54,9 @@
(catch [:error-type :5XX] e (build-response request e)))))
(defn chan-to-websocket [request input-chan ws-chan]
(on-close ws-chan (fn [status] (close! input-chan)))
(go-loop []
(when-let [e (<! input-chan)]
(send! ws-chan (build-response-body request e))
(recur))))
(defn listen-to-orders [exchange-apis]
(fn [{{sec-id-str :sec-id} :route-params :as request}]
(with-channel request ws-chan
(let [order-channel (get-new-order-channel exchange-apis (keyword sec-id-str))]
(chan-to-websocket request order-channel ws-chan)))))
(defn listen-to-trades [exchange-apis]
(fn [{{sec-id-str :sec-id} :route-params :as request}]
(with-channel request ws-chan
(let [order-channel (get-new-trades-channel exchange-apis (keyword sec-id-str))]
(chan-to-websocket request order-channel ws-chan)))))
(defn create-routes [exchange-apis kafka-producer]
(defn create-routes-for-exchange [exchange-apis]
(routes
(context "/api/1" []
(GET "/" [] api-description)
(GET "/securities" [] (securities exchange-apis))
(GET "/securities/:sec-id" [] (get-security exchange-apis))
(POST "/securities/:sec-id" [] (create-order exchange-apis kafka-producer))
(GET "/securities/:sec-id/trades" [] (listen-to-trades exchange-apis))
(GET "/securities/:sec-id/orders" [] (listen-to-orders exchange-apis)))
(POST "/securities/:sec-id" [] (create-order exchange-apis)))
(not-found "not found")))
src/skirmish/rest/marketdata.clj
-1
View File
@@ -1 +0,0 @@
(ns skirmish.rest.marketdata)
src/skirmish/util.clj
+4
View File
@@ -1,4 +1,5 @@
(ns skirmish.util
(:require [clojure.core.async :as async])
(:import [skirmish.PersistentSortedList]))
(defn sorted-list [& keys]
@@ -7,3 +8,6 @@
(defn sorted-list-by [comparator & keys]
(skirmish.PersistentSortedList/create comparator (seq keys)))
(defn block-thread []
"Block the current thread until chuck norris gets weaker."
(async/<!! (async/chan)))
test/skirmish/exchange_test.clj → test/skirmish/exchange/exchange_test.clj
+18 -7
View File
@@ -1,10 +1,13 @@
(ns skirmish.exchange-test
(ns skirmish.exchange.exchange-test
(:import [skirmish.PersistentSortedList])
(:require [midje.sweet :refer :all]
[skirmish.exchange :refer :all]
[skirmish.match :refer [create-buy-book
create-sell-book
match-order-in-book]]))
[orchestra.spec.test :as st]
[skirmish.exchange.exchange :refer :all]
[skirmish.exchange.match :refer [create-buy-book
create-sell-book
match-order-in-book]]))
(st/instrument)
(def PersistentSortedList? (comp #(= % skirmish.PersistentSortedList) type))
@@ -70,13 +73,21 @@
book-4 => expt-book-4)))
(facts "get-processed-orders"
(fact "ss"
(fact "bug numero uno"
(let [sell-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 116}
{:type :buy :quantity 8 :price 13 :book.order/id 117})
order {:type :sell :quantity 21 :price 13 :book.order/id 118}
[trades new-book leftover-order] (match-order-in-book sell-book order)
processed-order (get-processed-orders order leftover-order trades sell-book)]
processed-order => seq)));;;
processed-order => seq));;;
(fact "bug numero dos"
(let [order {:type :sell, :quantity 10, :price 2, :book.order/id 1}
leftover-order nil
trades [#:book.order{:type :buy, :quantity 10, :price 3, :order-id 1, :book-id 0}]
book {:sell (create-sell-book)
:buy (create-buy-book {:type :buy, :quantity 15, :price 3, :book.order/id 0})}
ret (get-processed-orders order leftover-order trades (:buy book))]
(count ret) => 3)))
test/skirmish/exchange/match_test.clj
+116
View File
@@ -0,0 +1,116 @@
(ns skirmish.exchange.match-test
(:require [midje.sweet :refer :all]
[skirmish.exchange.match :refer :all]))
(def PersistentSortedList? (comp #(= % skirmish.PersistentSortedList) type))
(facts "matching buy order in sell book"
(fact "Simplest case I."
(let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 101})
order {:type :buy :quantity 10 :price 2 :book.order/id 102}
[trades new-book] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 10 :price 2 :book-id 101 :order-id 102 :type :sell}]))
(fact "Simplest case II."
(let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 103})
order {:type :buy :quantity 10 :price 3 :book.order/id 104}
[trades new-book] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 10 :price 2 :book-id 103 :order-id 104 :type :sell}]))
(fact "Simplest case III."
(let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 105})
order {:type :buy :quantity 8 :price 2 :book.order/id 106}
[trades new-book] (match-order-in-book sell-book order)]
new-book => [{:type :sell :quantity 2 :price 2 :book.order/id 105}]
trades => [#:book.order{:quantity 8 :price 2 :book-id 105 :order-id 106 :type :sell}]))
(fact "Simplest case IV."
(let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 107} {:type :sell :quantity 8 :price 13 :book.order/id 108})
order {:type :buy :quantity 10 :price 13 :book.order/id 109}
[trades new-book] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 2 :price 12 :book-id 107 :order-id 109 :type :sell}
#:book.order{:quantity 8 :price 13 :book-id 108 :order-id 109 :type :sell}]))
(fact "Simplest case V."
(let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 113} {:type :sell :quantity 8 :price 13 :book.order/id 114})
order {:type :buy :quantity 9 :price 13 :book.order/id 115}
[trades new-book] (match-order-in-book sell-book order)]
new-book => (create-sell-book {:type :sell :quantity 1 :price 13 :book.order/id 114})
trades => [#:book.order{:quantity 2 :price 12 :book-id 113 :order-id 115 :type :sell}
#:book.order{:quantity 7 :price 13 :book-id 114 :order-id 115 :type :sell}]))
(fact "Simplest case VI."
(let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 116} {:type :sell :quantity 8 :price 13 :book.order/id 117})
order {:type :buy :quantity 21 :price 13 :book.order/id 118}
[trades new-book leftover-order] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 2 :price 12 :book-id 116 :order-id 118 :type :sell}
#:book.order{:quantity 8 :price 13 :book-id 117 :order-id 118 :type :sell}]
leftover-order => {:type :buy :quantity 11 :price 13 :book.order/id 118}))
(fact "Simplest case VII."
(let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 116} {:type :sell :quantity 8 :price 13 :book.order/id 117})
order {:type :buy :quantity 21 :price 10 :book.order/id 118}
[trades new-book leftover-order] (match-order-in-book sell-book order)]
new-book => sell-book
trades => empty?
leftover-order => order)))
(facts "matching sell order in buy book"
(fact "Simplest case I."
(let [buy-book (create-buy-book {:type :buy :quantity 10 :price 2 :book.order/id 101})
order {:type :sell :quantity 10 :price 2 :book.order/id 102}
[trades new-book] (match-order-in-book buy-book order)]
new-book => empty?
trades => [#:book.order{:quantity 10 :price 2 :book-id 101 :order-id 102 :type :buy}]))
(fact "Simplest case II."
(let [buy-book (create-buy-book {:type :buy :quantity 10 :price 4 :book.order/id 103})
order {:type :sell :quantity 10 :price 3 :book.order/id 104}
[trades new-book] (match-order-in-book buy-book order)]
new-book => []
trades => [#:book.order{:quantity 10 :price 4 :book-id 103 :order-id 104 :type :buy}]))
(fact "Simplest case III."
(let [buy-book (create-buy-book {:type :buy :quantity 10 :price 2 :book.order/id 105})
order {:type :sell :quantity 8 :price 2 :book.order/id 106}
[trades new-book] (match-order-in-book buy-book order)]
new-book => [{:type :buy :quantity 2 :price 2 :book.order/id 105}]
trades => [#:book.order{:quantity 8 :price 2 :book-id 105 :order-id 106 :type :buy}]))
(fact "Simplest case IV."
(let [buy-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 107}
{:type :buy :quantity 8 :price 13 :book.order/id 108})
order {:type :sell :quantity 10 :price 13 :book.order/id 109}
[trades new-book] (match-order-in-book buy-book order)]
new-book => []
trades => [#:book.order{:quantity 2 :price 14 :book-id 107 :order-id 109 :type :buy}
#:book.order{:quantity 8 :price 13 :book-id 108 :order-id 109 :type :buy}]))
(fact "Simplest case V."
(let [buy-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 113}
{:type :buy :quantity 8 :price 13 :book.order/id 114})
order {:type :sell :quantity 9 :price 13 :book.order/id 115}
[trades new-book] (match-order-in-book buy-book order)]
new-book => (create-buy-book {:type :buy :quantity 1 :price 13 :book.order/id 114})
trades => [#:book.order{:quantity 2 :price 14 :book-id 113 :order-id 115 :type :buy}
#:book.order{:quantity 7 :price 13 :book-id 114 :order-id 115 :type :buy}]))
(fact "Simplest case VI."
(let [sell-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 116}
{:type :buy :quantity 8 :price 13 :book.order/id 117})
order {:type :sell :quantity 21 :price 13 :book.order/id 118}
[trades new-book leftover-order] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 2 :price 14 :book-id 116 :order-id 118 :type :buy}
#:book.order{:quantity 8 :price 13 :book-id 117 :order-id 118 :type :buy}]
leftover-order => {:type :sell :quantity 11 :price 13 :book.order/id 118})))
(facts "We should write tests for each bug found"
(fact "book type should always be book.PersistentSortedList"
(let [book (create-buy-book)
order {:type :sell :quantity 21 :price 13 :book.order/id 118}
[_ new-book _] (match-order-in-book book order)]
(count new-book) => 0
(type new-book) => skirmish.PersistentSortedList)))
test/skirmish/match_test.clj
-124
View File
@@ -1,124 +0,0 @@
(ns skirmish.match-test
(:require [midje.sweet :refer :all]
[skirmish.match :refer :all]))
(def PersistentSortedList? (comp #(= % skirmish.PersistentSortedList) type))
(facts "matching buy order in sell book"
(fact "Simplest case I."
(let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 101})
order {:type :buy :quantity 10 :price 2 :book.order/id 102}
[trades new-book] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 10 :price 2 :book-id 101 :order-id 102 :type :sell}]))
(fact "Simplest case II."
(let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 103})
order {:type :buy :quantity 10 :price 3 :book.order/id 104}
[trades new-book] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 10 :price 2 :book-id 103 :order-id 104 :type :sell}]))
(fact "Simplest case III."
(let [sell-book (create-sell-book {:type :sell :quantity 10 :price 2 :book.order/id 105})
order {:type :buy :quantity 8 :price 2 :book.order/id 106}
[trades new-book] (match-order-in-book sell-book order)]
new-book => [{:type :sell :quantity 2 :price 2 :book.order/id 105}]
trades => [#:book.order{:quantity 8 :price 2 :book-id 105 :order-id 106 :type :sell}]))
(fact "Simplest case IV."
(let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 107} {:type :sell :quantity 8 :price 13 :book.order/id 108})
order {:type :buy :quantity 10 :price 13 :book.order/id 109}
[trades new-book] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 2 :price 12 :book-id 107 :order-id 109 :type :sell}
#:book.order{:quantity 8 :price 13 :book-id 108 :order-id 109 :type :sell}]))
(fact "Simplest case V."
(let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 113} {:type :sell :quantity 8 :price 13 :book.order/id 114})
order {:type :buy :quantity 9 :price 13 :book.order/id 115}
[trades new-book] (match-order-in-book sell-book order)]
new-book => (create-sell-book {:type :sell :quantity 1 :price 13 :book.order/id 114})
trades => [#:book.order{:quantity 2 :price 12 :book-id 113 :order-id 115 :type :sell}
#:book.order{:quantity 7 :price 13 :book-id 114 :order-id 115 :type :sell}]))
(fact "Simplest case VI."
(let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 116} {:type :sell :quantity 8 :price 13 :book.order/id 117})
order {:type :buy :quantity 21 :price 13 :book.order/id 118}
[trades new-book leftover-order] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 2 :price 12 :book-id 116 :order-id 118 :type :sell}
#:book.order{:quantity 8 :price 13 :book-id 117 :order-id 118 :type :sell}]
leftover-order => {:type :buy :quantity 11 :price 13 :book.order/id 118}))
(fact "Simplest case VII."
(let [sell-book (create-sell-book {:type :sell :quantity 2 :price 12 :book.order/id 116} {:type :sell :quantity 8 :price 13 :book.order/id 117})
order {:type :buy :quantity 21 :price 10 :book.order/id 118}
[trades new-book leftover-order] (match-order-in-book sell-book order)]
new-book => sell-book
trades => empty?
leftover-order => order)))
(facts "matching sell order in buy book"
(fact "Simplest case I."
(let [buy-book (create-buy-book {:type :buy :quantity 10 :price 2 :book.order/id 101})
order {:type :sell :quantity 10 :price 2 :book.order/id 102}
[trades new-book] (match-order-in-book buy-book order)]
new-book => empty?
trades => [#:book.order{:quantity 10 :price 2 :book-id 101 :order-id 102 :type :buy}]))
(fact "Simplest case II."
(let [buy-book (create-buy-book {:type :buy :quantity 10 :price 4 :book.order/id 103})
order {:type :sell :quantity 10 :price 3 :book.order/id 104}
[trades new-book] (match-order-in-book buy-book order)]
new-book => []
trades => [#:book.order{:quantity 10 :price 4 :book-id 103 :order-id 104 :type :buy}]))
(fact "Simplest case III."
(let [buy-book (create-buy-book {:type :buy :quantity 10 :price 2 :book.order/id 105})
order {:type :sell :quantity 8 :price 2 :book.order/id 106}
[trades new-book] (match-order-in-book buy-book order)]
new-book => [{:type :buy :quantity 2 :price 2 :book.order/id 105}]
trades => [#:book.order{:quantity 8 :price 2 :book-id 105 :order-id 106 :type :buy}]))
(fact "Simplest case IV."
(let [buy-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 107}
{:type :buy :quantity 8 :price 13 :book.order/id 108})
order {:type :sell :quantity 10 :price 13 :book.order/id 109}
[trades new-book] (match-order-in-book buy-book order)]
new-book => []
trades => [#:book.order{:quantity 2 :price 14 :book-id 107 :order-id 109 :type :buy}
#:book.order{:quantity 8 :price 13 :book-id 108 :order-id 109 :type :buy}]))
(fact "Simplest case V."
(let [buy-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 113}
{:type :buy :quantity 8 :price 13 :book.order/id 114})
order {:type :sell :quantity 9 :price 13 :book.order/id 115}
[trades new-book] (match-order-in-book buy-book order)]
new-book => (create-buy-book {:type :buy :quantity 1 :price 13 :book.order/id 114})
trades => [#:book.order{:quantity 2 :price 14 :book-id 113 :order-id 115 :type :buy}
#:book.order{:quantity 7 :price 13 :book-id 114 :order-id 115 :type :buy}]))
(fact "Simplest case VI."
(let [sell-book (create-buy-book {:type :buy :quantity 2 :price 14 :book.order/id 116}
{:type :buy :quantity 8 :price 13 :book.order/id 117})
order {:type :sell :quantity 21 :price 13 :book.order/id 118}
[trades new-book leftover-order] (match-order-in-book sell-book order)]
new-book => empty?
trades => [#:book.order{:quantity 2 :price 14 :book-id 116 :order-id 118 :type :buy}
#:book.order{:quantity 8 :price 13 :book-id 117 :order-id 118 :type :buy}]
leftover-order => {:type :sell :quantity 11 :price 13 :book.order/id 118})))
(facts "We should write tests for each bug found"
(fact "book type should always be book.PersistentSortedList"
(let [book (create-buy-book)
order {:type :sell :quantity 21 :price 13 :book.order/id 118}
[_ new-book _] (match-order-in-book book order)]
(count new-book) => 0
(type new-book) => skirmish.PersistentSortedList)))
;(fact "should not fuck up"
; (let [order-1 {:type :buy :quantity 4 :price 4 :book.order/id 5}
; order-2 {:type :sell :quantity 2 :price 3 :book.order/id 6}
; book (create-buy-book order-1)
; expt-trade #:book.order{}
; [trades new-book new-order] (match-order-in-book book order-2)]
; trades => 1
; new-order => nil?
; new-book => {})))
test/skirmish/middleware/test_core.clj
+13
View File
@@ -0,0 +1,13 @@
(ns skirmish.middleware.test-core
(:require [midje.sweet :refer :all]
[skirmish.middleware.core :refer :all]))
(defn all? [xf]
#(every? true? (sequence xf %)))
(facts "we should have a async/chan base middleware for developing."
(fact "It should exchanges for each security in the config"
(let [config {:securities [{:sec-id :molly} {:sec-id :pepper}]}
exchanges (create-exchange-apis-for config)]
(keys exchanges) => [:molly :pepper]
(vals exchanges) => #(every? map? %))))