From b81606fd27f6f01374130627411289849a268eac Mon Sep 17 00:00:00 2001 From: "didier.amyot" Date: Sat, 24 May 2014 12:51:25 -0400 Subject: [PATCH] now as kick ass histogram to represent the numeric fields! --- api.go | 65 +++++++++++++++++ client/coffee/controllers/keyStatCtrl.coffee | 34 ++++++--- main.go | 1 + params.yml | 4 ++ stats.go | 74 ++++++++++---------- stats_test.go | 40 ++--------- 6 files changed, 137 insertions(+), 81 deletions(-) diff --git a/api.go b/api.go index 7a0ae61..778b309 100644 --- a/api.go +++ b/api.go @@ -158,6 +158,33 @@ func (this *VultureBackend) GetStat(key string) (interface{}, error) { return <-out, nil } +func (this *VultureBackend) GetHistogram(key string, min, max float64, numberOfBins int) (interface{}, error) { + hm := &HistogramMaker{min, max, numberOfBins} + query := bson.M{key: bson.M{"$exists": true}} + iter := this.Collection.Find(query).Select(bson.M{key: 1}).Iter() + result := make(map[string]interface{}) + + in := make(chan float64) + out := make(chan *Histogram) + go hm.MakeHistogram(in, out) + + for iter.Next(&result) { + if val, ok := result[key]; ok { + if fval, isFloat := val.(float64); isFloat { + in <- fval + } else if ival, isInt := val.(int); isInt { + in <- float64(ival) + } + } + } + close(in) + if err := iter.Close(); err != nil { + return nil, err + } + return <-out, nil + +} + func (this *VultureBackend) GetAllDocuments(query interface{}) (interface{}, error) { var result []map[string]interface{} @@ -282,3 +309,41 @@ func getStats(w http.ResponseWriter, request *http.Request) (interface{}, error) } return vb.GetStat(key) } + +func getHistogram(w http.ResponseWriter, request *http.Request) (interface{}, error) { + vb, err := GetVultureBackend(request) + vars := mux.Vars(request) + smin, ok := vars["min"] + if !ok { + return nil, errors.New("min field not specified") + } + min, err := strconv.ParseFloat(smin, 64) + if err != nil { + return nil, errors.New("min value is not an float64") + } + + smax, ok := vars["max"] + if !ok { + return nil, errors.New("max field not specified") + } + max, err := strconv.ParseFloat(smax, 64) + if err != nil { + return nil, errors.New("max value is not an float64") + } + sNumberOfBins, ok := vars["number_of_bins"] + if !ok { + return nil, errors.New("number_of_bins field not specified") + } + numberOfBins, err := strconv.Atoi(sNumberOfBins) + if err != nil { + return nil, errors.New("numberOfBins value is not an int") + } + + key, ok := vars["key"] + if !ok { + return nil, errors.New("key field not specified") + } + + return vb.GetHistogram(key, min, max, numberOfBins) + +} diff --git a/client/coffee/controllers/keyStatCtrl.coffee b/client/coffee/controllers/keyStatCtrl.coffee index 04dd716..158f148 100644 --- a/client/coffee/controllers/keyStatCtrl.coffee +++ b/client/coffee/controllers/keyStatCtrl.coffee @@ -1,5 +1,6 @@ root = this; + humanValueOfStatisticalAbreviation = mean: "mean" std: "standard deviation" @@ -8,6 +9,11 @@ humanValueOfStatisticalAbreviation = max: "maximum" n: "number of documents" +findHistogramMin = (stats) -> + return Math.max(stats.min, stats.mean - 3 * stats.std) + +findHistogramMax = (stats) -> + return Math.min(stats.max, stats.mean + 3 * stats.std) getAxisLabel = (min, stepSize, numberOfBins) -> @@ -17,13 +23,9 @@ root.controllers.controller('keyStatCtrl', ['$scope', '$routeParams', 'util', ($ $scope.name = $routeParams.key $scope.stats = [] - url = "/api/#{ $routeParams.server }/#{ $routeParams.database }/#{ $routeParams.collection }" - url = "#{ url }/stats/#{$routeParams.key}" + base_url = "/api/#{ $routeParams.server }/#{ $routeParams.database }/#{ $routeParams.collection }" + stat_url = "#{ base_url }/stats/#{$routeParams.key}" - util.get(url).then((res) -> - for key, val of res.data - $scope.stats.push({name: humanValueOfStatisticalAbreviation[key], value: val }) - ) $scope.chartConfig = options: @@ -34,16 +36,30 @@ root.controllers.controller('keyStatCtrl', ['$scope', '$routeParams', 'util', ($ groupPadding: 0 pointPadding: 0 borderWidth: 0 - xAxis: - categories: ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + xAxis: + categories: [] title: text: 'Distribution' loading: false series: [{ - data: [10, 15, 12, 8, 7, 4, 12, 12, 9, 11], + data: [], name: $routeParams.key color: '#008CBA' }] + + + util.get(stat_url).then((res) -> + for key, val of res.data + $scope.stats.push({name: humanValueOfStatisticalAbreviation[key], value: val }) + numberOfBins = 50 + min = findHistogramMin(res.data) + max = findHistogramMax(res.data) + histogram_url = "#{ base_url }/histogram/#{$routeParams.key}/#{min}/#{max}/#{numberOfBins}" + return util.get(histogram_url) + ).then((res) -> + $scope.chartConfig.series[0].data = res.data.values + $scope.chartConfig.xAxis.categories = getAxisLabel(res.data.min, res.data.step_size, res.values.length) + ) ]) diff --git a/main.go b/main.go index 50ea622..271a815 100644 --- a/main.go +++ b/main.go @@ -92,6 +92,7 @@ func serveApi(router *mux.Router) { router.Handle("/{server}/{database}/{collection}/idx/{index:[0-9]+}", ApiHandler(getDocumentByIndex)) router.Handle("/{server}/{database}/{collection}/_id/{id}", ApiHandler(getDocumentById)) router.Handle("/{server}/{database}/{collection}/stats/{key}", ApiHandler(getStats)) + router.Handle("/{server}/{database}/{collection}/histogram/{key}/{min}/{max}/{number_of_bins}", ApiHandler(getHistogram)) router.Handle("/{server}/{database}/{collection}/all", ApiHandler(getAllDocuments)) router.Handle("/{server}/{database}/{collection}/query/{query}/idx/{index:[0-9]+}", ApiHandler(getDocumentByQueryAndIndex)) router.Handle("/{server}/{database}/{collection}/query/{query}/all", ApiHandler(getAllDocuments)) diff --git a/params.yml b/params.yml index c64fa44..223f374 100644 --- a/params.yml +++ b/params.yml @@ -21,3 +21,7 @@ MongoServers: - name: mongolab url: mongodb://ustraca:9OApiiV9En85bBFGfVyYKH4C@ds049288.mongolab.com:49288/ustraca + + - + name: miaou + url: 192.168.1.10:27017 diff --git a/stats.go b/stats.go index e098a62..6a12d8b 100644 --- a/stats.go +++ b/stats.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "math" ) @@ -17,6 +18,43 @@ type Histogram struct { Values []int `json:"values"` } +type HistogramMaker struct { + Min float64 + Max float64 + NumberOfBins int +} + +func (this *HistogramMaker) findSuitableBinForDatum(min, step, val float64) int { + var i int + var act float64 = min + for i = 0; act <= val; act += step { + i++ + } + return i - 1 + +} + +func (this *HistogramMaker) MakeHistogram(dataChannel chan float64, outputChannel chan *Histogram) { + + step := (this.Max - this.Min) / float64(this.NumberOfBins) + hist := make([]int, this.NumberOfBins) + + for datum := range dataChannel { + if datum < this.Min || datum > this.Max { + continue + } + i := this.findSuitableBinForDatum(this.Min, step, datum) + if i >= len(hist) || i < 0 { + log.Println("what the fuck?!", this.Min, step, datum, i) + } else { + hist[i]++ + } + } + + outputChannel <- &Histogram{this.Min, step, hist} + +} + type Stats struct { Mean float64 `json:"mean"` Var float64 `json:"var"` @@ -26,42 +64,6 @@ type Stats struct { N int `json:"n"` } -func (this *Stats) findMinHistogramValue() float64 { - return math.Max(this.Min, this.Mean-3*this.Std) -} - -func (this *Stats) findMaxHistogramValue() float64 { - return math.Min(this.Max, this.Mean+3*this.Std) -} - -func (this *Stats) findSuitableBinForDatum(min, step, val float64) int { - var i int - var act float64 = min - for i = 0; act < val; act += step { - i++ - } - return i - 1 - -} - -func (this *Stats) MakeHistogram(numberOfBins int, dataChannel chan float64, outputChannel chan *Histogram) { - min := this.findMinHistogramValue() - max := this.findMaxHistogramValue() - step := (max - min) / float64(numberOfBins) - hist := make([]int, numberOfBins) - - for datum := range dataChannel { - if datum < min || datum > max { - continue - } - i := this.findSuitableBinForDatum(min, step, datum) - hist[i]++ - } - - outputChannel <- &Histogram{min, step, hist} - -} - type StatAggregator struct { sum float64 sumOfTheSquares float64 diff --git a/stats_test.go b/stats_test.go index a7b045a..10c015b 100644 --- a/stats_test.go +++ b/stats_test.go @@ -62,43 +62,8 @@ func TestGetFinalStats(t *testing.T) { } } -func TestFindMinHistogramValue(t *testing.T) { - stats := &Stats{ - Mean: 10, - Std: 2, - Min: 1, - } - - if stats.findMinHistogramValue() != 4 { - t.Fail() - } - - stats.Min = 7 - if stats.findMinHistogramValue() != 7 { - t.Fail() - } - -} - -func TestFindMaxHistogramValue(t *testing.T) { - stats := &Stats{ - Mean: 10, - Std: 2, - Max: 22, - } - - if stats.findMaxHistogramValue() != 16 { - t.Fail() - } - - stats.Max = 14 - if stats.findMaxHistogramValue() != 14 { - t.Fail() - } -} - func TestFindSuitableBinForDatum(t *testing.T) { - stats := new(Stats) + stats := new(HistogramMaker) if bin := stats.findSuitableBinForDatum(-4, 1, -3.2); bin != 0 { t.Error("wrong bin: ", bin) } @@ -110,5 +75,8 @@ func TestFindSuitableBinForDatum(t *testing.T) { if bin := stats.findSuitableBinForDatum(-4, 1, 10.1); bin != 14 { t.Error("wrong bin: ", bin) } + if bin := stats.findSuitableBinForDatum(1, 50, 1); bin != 0 { + t.Error("wrong bin: ", bin) + } }