From 0c8e5ed2946a3ab74b518df69b7af35008dff2b4 Mon Sep 17 00:00:00 2001 From: didier amyot Date: Mon, 6 Jul 2015 10:43:33 -0400 Subject: [PATCH] changed ids to base64 format. --- doorman.go | 28 +++++-- doorman_test.go | 109 +++++++++++++++++++------ httpsubscriber/httpsubscriber.go | 6 +- nanomsgsubscriber/nanomsgsubscriber.go | 3 +- nsqsubscriber/nsqsubscriber.go | 5 +- subscribe.go | 3 +- subscriber/subscriber.go | 17 ++-- 7 files changed, 115 insertions(+), 56 deletions(-) diff --git a/doorman.go b/doorman.go index 2bd178a..cd399d9 100644 --- a/doorman.go +++ b/doorman.go @@ -1,6 +1,8 @@ package doorman import ( + "encoding/base64" + "encoding/binary" "encoding/json" "errors" "log" @@ -13,7 +15,6 @@ import ( "github.com/dchest/siphash" "github.com/didiercrunch/doorman/shared" - "gopkg.in/mgo.v2/bson" ) var ONE *big.Rat = big.NewRat(1, 1) @@ -34,17 +35,21 @@ func IsEqual(f1, f2 *big.Rat) bool { } type Doorman struct { - Id bson.ObjectId // the id of the doorman + Id string // the id of the doorman LastChangeTimestamp int64 // an always increasing int that represent the last time the doorman has beed updated Probabilities []*big.Rat // The probability of each cases. The sum of probabilities needs to be one wg sync.WaitGroup // waitgroup for goroutine safety } -func New(id bson.ObjectId, Probabilities []*big.Rat) *Doorman { +func New(id string, probabilities []*big.Rat) (*Doorman, error) { wab := &Doorman{} - wab.Id = id - wab.Probabilities = Probabilities - return wab + if bid, err := base64.URLEncoding.DecodeString(id); err != nil { + return nil, err + } else if len(bid) != 16 { + return nil, errors.New("id must be base64 encoded of a 16 bytes array") + } + wab.Probabilities = probabilities + return wab, wab.Validate() } func (w *Doorman) Length() int { @@ -52,7 +57,7 @@ func (w *Doorman) Length() int { } func (w *Doorman) UpdateHard(baseURL string) error { - r, err := http.Get(baseURL + "/" + w.Id.Hex()) + r, err := http.Get(baseURL + "/" + w.Id) if err != nil { return err } @@ -69,7 +74,7 @@ func (w *Doorman) Update(wu *shared.DoormanUpdater) error { if wu.Timestamp <= w.LastChangeTimestamp { return nil } - if wu.Id != w.Id.Hex() { + if w.Id != wu.Id { return errors.New("bad doorman id") } w.wg.Add(1) @@ -144,6 +149,13 @@ func (w *Doorman) GetCaseFromString(data string) uint { return w.GetCaseFromData([]byte(data)) } +// not yet in spect +func (w *Doorman) getCaseFromInt(data int) uint { + b := make([]byte, 10) + binary.LittleEndian.PutUint64(b, uint64(data)) + return w.GetCaseFromData(b) +} + func (w *Doorman) GetRandomCase() uint { r := rand.Float64() return w.GetCase(new(big.Rat).SetFloat64(r)) diff --git a/doorman_test.go b/doorman_test.go index 5061d41..40a99d9 100644 --- a/doorman_test.go +++ b/doorman_test.go @@ -1,6 +1,7 @@ package doorman import ( + "encoding/base64" "encoding/json" "errors" "fmt" @@ -9,30 +10,58 @@ import ( "net/http" "net/http/httptest" "reflect" + "strconv" "testing" "github.com/didiercrunch/doorman/shared" - "gopkg.in/mgo.v2/bson" ) +const oid = "507f1f77bcf86cd799439011" + var epsilon = 0.0 + var ZERO = big.NewRat(0, 1) +func newDoorman(probabilities []*big.Rat) *Doorman { + id := base64.URLEncoding.EncodeToString(make([]byte, 16)) + if ret, err := New(id, probabilities); err != nil { + panic(err) + } else { + return ret + } +} + func assertIsEqual(t *testing.T, expected, received *big.Rat) { if expected.Cmp(received) != 0 { t.Error("received", received, "but expected", expected) } } -func getProbs(probs ...float64) []*big.Rat { +func getProbs(probs ...string) []*big.Rat { ret := make([]*big.Rat, len(probs)) + var ok bool for i, p := range probs { - ret[i] = new(big.Rat).SetFloat64(p) + if ret[i], ok = new(big.Rat).SetString(p); !ok { + panic(p) + } } return ret } -var oid = "507f1f77bcf86cd799439011" +func TestNew(t *testing.T) { + notBase64 := "this is not url encoded base 64 /" + if _, err := New(notBase64, getProbs("1/4", "3/4")); err == nil { + t.Error() + } + base64OfAndArrayOfLength15 := "MTIzNDU2Nzg5MDEyMzQ1" + if _, err := New(base64OfAndArrayOfLength15, getProbs("1/4", "3/4")); err == nil { + t.Error() + } + goodId := "MTIzNDU2Nzg5MDEyMzQ1Ng==" + if _, err := New(goodId, getProbs("1/4", "1/4")); err == nil { + t.Error() + } +} func TestIsEqual(t *testing.T) { if !IsEqual(big.NewRat(2, 2), big.NewRat(1, 1)) { @@ -49,24 +78,24 @@ func TestValidate(t *testing.T) { t.Error() } - w = &Doorman{Probabilities: getProbs(0.5, 0.75)} + w = &Doorman{Probabilities: getProbs("2/4", "3/4")} if w.Validate().Error() != "The sum of probabilities is not one" { t.Error() } - w = &Doorman{Probabilities: getProbs(0.25, 0.75)} + w = &Doorman{Probabilities: getProbs("1/4", "3/4")} if w.Validate() != nil { t.Error() } - w = &Doorman{Probabilities: getProbs(0.2500000000002, 0.75)} + w = &Doorman{Probabilities: getProbs("100000001/400000000", "3/4")} if w.Validate() == nil { t.Error("even very small diff should be significative") } } func TestGetCase(t *testing.T) { - w := New(bson.NewObjectId(), getProbs(0.25, 0.5, 0.25)) + w := newDoorman(getProbs("1/4", "2/4", "1/4")) if c := w.GetCase(ZERO); c != 0 { t.Error("expected 0 but received", c) } @@ -77,7 +106,7 @@ func TestGetCase(t *testing.T) { } func TestGetCaseCoroutineSafety(t *testing.T) { - w := New(bson.NewObjectId(), getProbs(0.25, 0.5, 0.25)) + w := newDoorman(getProbs("1/4", "2/4", "1/4")) i := 0 w.wg.Add(1) go func() { @@ -111,8 +140,7 @@ func TestHash(t *testing.T) { } func TestUpdateTimestamp(t *testing.T) { - - w := &Doorman{LastChangeTimestamp: 10, Id: bson.ObjectIdHex(oid)} + w := &Doorman{LastChangeTimestamp: 10, Id: oid} m := &shared.DoormanUpdater{Timestamp: 9} w.Update(m) if w.LastChangeTimestamp != 10 { @@ -127,24 +155,24 @@ func TestUpdateTimestamp(t *testing.T) { } func TestUpdateProbabilities(t *testing.T) { - w := &Doorman{LastChangeTimestamp: 0, Id: bson.ObjectIdHex(oid)} - m := &shared.DoormanUpdater{Timestamp: 0, Probabilities: getProbs(0.5, 0.5), Id: oid} + w := &Doorman{LastChangeTimestamp: 0, Id: oid} + m := &shared.DoormanUpdater{Timestamp: 0, Probabilities: getProbs("1/2", "1/2"), Id: oid} w.Update(m) if len(w.Probabilities) != 0 { t.Error() } - m = &shared.DoormanUpdater{Timestamp: 2, Probabilities: getProbs(0.5, 0.5), Id: oid} + m = &shared.DoormanUpdater{Timestamp: 2, Probabilities: getProbs("1/2", "1/2"), Id: oid} if err := w.Update(m); err != nil { t.Error(err) - } else if !reflect.DeepEqual(w.Probabilities, getProbs(0.5, 0.5)) { + } else if !reflect.DeepEqual(w.Probabilities, getProbs("1/2", "1/2")) { t.Error("bad prob") } - m = &shared.DoormanUpdater{Timestamp: 3, Probabilities: getProbs(0.5, 0.25), Id: oid} + m = &shared.DoormanUpdater{Timestamp: 3, Probabilities: getProbs("1/2", "1/4"), Id: oid} if err := w.Update(m); err == nil { t.Error("should received an error") - } else if !reflect.DeepEqual(w.Probabilities, getProbs(0.5, 0.5)) { + } else if !reflect.DeepEqual(w.Probabilities, getProbs("1/2", "1/2")) { t.Error("bad prob") } @@ -160,9 +188,10 @@ func randomBytes(n int) []byte { func TestUpdateHard(t *testing.T) { - wab := New(bson.ObjectIdHex(oid), getProbs()) + wab := newDoorman(getProbs("1/3", "1/3", "1/3")) + wab.Id = oid createMessage := func() string { - msg := &shared.DoormanUpdater{Timestamp: 2, Probabilities: getProbs(0.5, 0.25, 0.25), Id: oid} + msg := &shared.DoormanUpdater{Timestamp: 2, Probabilities: getProbs("1/2", "1/4", "1/4"), Id: oid} if ret, err := json.Marshal(msg); err != nil { panic(err) } else { @@ -170,7 +199,7 @@ func TestUpdateHard(t *testing.T) { } } handler := func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/"+wab.Id.Hex() { + if r.URL.Path != "/"+wab.Id { t.Error("bad url", r.URL) } fmt.Fprint(w, createMessage()) @@ -181,8 +210,8 @@ func TestUpdateHard(t *testing.T) { if err := wab.UpdateHard(ts.URL); err != nil { t.Error(err) } - if wab.Length() != 3 { - t.Error() + if l := wab.Length(); l != 3 { + t.Error(l) } } @@ -203,7 +232,7 @@ func IsExtremeBinomialResult(x int, n, p float64) error { func TestGetRandomCase(t *testing.T) { p := 0.5 n := 10000 - wab := New(bson.NewObjectId(), getProbs(p, 1-p)) + wab := newDoorman(getProbs("1/2", "1/2")) var sum = 0 for i := 0; i < n; i++ { sum += int(wab.GetRandomCase()) @@ -215,21 +244,49 @@ func TestGetRandomCase(t *testing.T) { } func TestGetCaseFromString(t *testing.T) { - w := New(bson.NewObjectId(), getProbs(0.25, 0.5, 0.25)) + w := newDoorman(getProbs("1/4", "1/2", "1/4")) expts := map[string]uint{ - "j'aime la ratoutille": 2, + "Հայաստան..": 2, + "საქართველო": 1, + "Azərbaycan..": 0, } for data, expt := range expts { if res := w.GetCaseFromString(data); res != expt { t.Error("incorrect results for: ", data, res) } } +} +func TestConsistencyOfDoormenWhenProbabilityChanges(t *testing.T) { + w := "dddddddddddddddddddddd" + for i := 1; i < 100; i++ { + doorman := newDoorman(getProbs(strconv.Itoa(i)+"/100", strconv.Itoa(100-i)+"/100")) + if doorman.GetCaseFromString(w) != 0 { + t.Error() + } + } +} + +// bellow test is not yet in doorman's specification. It is not clear how +//we want to hash numbers because of the many number types in many programming +// language +func _TestGetCaseFromInt(t *testing.T) { + w := newDoorman(getProbs("1/4", "1/2", "1/4")) + expts := map[int]uint{ + 2: 2, + 13: 2, + 195: 1, + } + for data, expt := range expts { + if res := w.getCaseFromInt(data); res != expt { + t.Error("incorrect results for: ", data, res) + } + } } func BenchmarkGetCaseFromData(b *testing.B) { var data = randomBytes(1024 * 1024) - w := New(bson.NewObjectId(), getProbs(0.1, 0.4, 0.4, 0.05, 0.05)) + w := newDoorman(getProbs("10/100", "40/100", "40/100", "5/100", "5/100")) for i := 0; i < b.N; i++ { w.GetCaseFromData(data) } diff --git a/httpsubscriber/httpsubscriber.go b/httpsubscriber/httpsubscriber.go index eaeb15c..25514c8 100644 --- a/httpsubscriber/httpsubscriber.go +++ b/httpsubscriber/httpsubscriber.go @@ -3,12 +3,10 @@ package httpsubscriber import ( "encoding/json" "errors" + "github.com/didiercrunch/doorman/shared" "log" "net/http" "time" - - "github.com/didiercrunch/doorman/shared" - "gopkg.in/mgo.v2/bson" ) type HttpSubscriber struct { @@ -39,7 +37,7 @@ func (s *HttpSubscriber) callUpdateHandlerFunction(f shared.UpdateHandlerFunc, d } } -func (s *HttpSubscriber) Subscribe(abtestId bson.ObjectId, update shared.UpdateHandlerFunc) error { +func (s *HttpSubscriber) Subscribe(abtestId string, update shared.UpdateHandlerFunc) error { go func() { c := time.Tick(s.HartBeat) for _ = range c { diff --git a/nanomsgsubscriber/nanomsgsubscriber.go b/nanomsgsubscriber/nanomsgsubscriber.go index f1abbac..72844de 100644 --- a/nanomsgsubscriber/nanomsgsubscriber.go +++ b/nanomsgsubscriber/nanomsgsubscriber.go @@ -10,7 +10,6 @@ import ( "github.com/gdamore/mangos/protocol/sub" "github.com/gdamore/mangos/transport/ipc" "github.com/gdamore/mangos/transport/tcp" - "gopkg.in/mgo.v2/bson" ) type NanoMsgSubscriber struct { @@ -26,7 +25,7 @@ func (s *NanoMsgSubscriber) callUpdateHandlerFunction(f shared.UpdateHandlerFunc } } -func (s *NanoMsgSubscriber) Subscribe(abtestId bson.ObjectId, update shared.UpdateHandlerFunc) error { +func (s *NanoMsgSubscriber) Subscribe(abtestId string, update shared.UpdateHandlerFunc) error { var sock mangos.Socket var err error var msg []byte diff --git a/nsqsubscriber/nsqsubscriber.go b/nsqsubscriber/nsqsubscriber.go index e3d5485..44c3c8e 100644 --- a/nsqsubscriber/nsqsubscriber.go +++ b/nsqsubscriber/nsqsubscriber.go @@ -6,7 +6,6 @@ import ( "code.google.com/p/go-uuid/uuid" "github.com/bitly/go-nsq" "github.com/didiercrunch/doorman/shared" - "gopkg.in/mgo.v2/bson" ) var UUID = uuid.New() @@ -26,9 +25,9 @@ func toNSQHandlerFunc(update shared.UpdateHandlerFunc) nsq.HandlerFunc { } } -func (sub *NSQSubscriber) Subscribe(doormanId bson.ObjectId, update shared.UpdateHandlerFunc) error { +func (sub *NSQSubscriber) Subscribe(doormanId string, update shared.UpdateHandlerFunc) error { config := nsq.NewConfig() - q, err := nsq.NewConsumer(doormanId.Hex(), UUID, config) + q, err := nsq.NewConsumer(doormanId, UUID, config) if err != nil { return err } diff --git a/subscribe.go b/subscribe.go index 3263b05..97e45ab 100644 --- a/subscribe.go +++ b/subscribe.go @@ -5,11 +5,10 @@ import ( "github.com/didiercrunch/doorman/nsqsubscriber" "github.com/didiercrunch/doorman/shared" "github.com/didiercrunch/doorman/subscriber" - "gopkg.in/mgo.v2/bson" ) type Subscriber interface { - Subscribe(doormanId bson.ObjectId, update shared.UpdateHandlerFunc) error + Subscribe(doormanId string, update shared.UpdateHandlerFunc) error } func (w *Doorman) subscribe(sub Subscriber) error { diff --git a/subscriber/subscriber.go b/subscriber/subscriber.go index 1524c33..ff13017 100644 --- a/subscriber/subscriber.go +++ b/subscriber/subscriber.go @@ -3,11 +3,9 @@ package subscriber import ( "encoding/json" "errors" - "fmt" "github.com/didiercrunch/doorman/httpsubscriber" "github.com/didiercrunch/doorman/nanomsgsubscriber" "github.com/didiercrunch/doorman/shared" - "gopkg.in/mgo.v2/bson" "net/http" "time" ) @@ -17,7 +15,7 @@ type Subscriber struct { } type subscriber interface { - Subscribe(abtestId bson.ObjectId, update shared.UpdateHandlerFunc) error + Subscribe(abtestId string, update shared.UpdateHandlerFunc) error } type ServerSpecification struct { @@ -29,7 +27,6 @@ type ServerSpecification struct { func (s *Subscriber) getServerSpecification() (*ServerSpecification, error) { url := s.URL + "/api/server" - fmt.Println(">>>", url) resp, err := http.Get(url) if err != nil { return nil, err @@ -41,24 +38,22 @@ func (s *Subscriber) getServerSpecification() (*ServerSpecification, error) { ret := new(ServerSpecification) decoder := json.NewDecoder(resp.Body) err = decoder.Decode(ret) - fmt.Println("ok!") return ret, err } -func (sub *Subscriber) GetSubsciber(serverSpec *ServerSpecification, doormanId bson.ObjectId) subscriber { +func (sub *Subscriber) GetSubsciber(serverSpec *ServerSpecification, doormanId string) subscriber { switch serverSpec.MessageQueue { case "nanomsg": return &nanomsgsubscriber.NanoMsgSubscriber{serverSpec.NanoMsg["url"]} } - fmt.Println("shit!", serverSpec) return &httpsubscriber.HttpSubscriber{sub.getDoormanStatusUrl(doormanId), time.Second * 5} } -func (sub *Subscriber) getDoormanStatusUrl(doormanId bson.ObjectId) string { - return sub.URL + "/api/doormen/" + doormanId.Hex() + "/status" +func (sub *Subscriber) getDoormanStatusUrl(doormanId string) string { + return sub.URL + "/api/doormen/" + doormanId + "/status" } -func (sub *Subscriber) SetInitialState(doormanId bson.ObjectId, update shared.UpdateHandlerFunc) error { +func (sub *Subscriber) SetInitialState(doormanId string, update shared.UpdateHandlerFunc) error { httpSUbscriber := &httpsubscriber.HttpSubscriber{Url: sub.getDoormanStatusUrl(doormanId)} if du, err := httpSUbscriber.GetDoormanUpdater(); err != nil { return err @@ -72,7 +67,7 @@ func startHartBeat() error { return nil } -func (sub *Subscriber) Subscribe(doormanId bson.ObjectId, update shared.UpdateHandlerFunc) error { +func (sub *Subscriber) Subscribe(doormanId string, update shared.UpdateHandlerFunc) error { spec, err := sub.getServerSpecification() if err != nil { return err