From 40130e3c1343648a499f79ecea2f3ab89a9cb1c3 Mon Sep 17 00:00:00 2001 From: didier amyot Date: Sun, 7 Dec 2014 20:07:39 -0800 Subject: [PATCH] refactoring and fixing bug. --- twitter.go | 10 +- twitterApi.go | 25 +++-- twitterApi_test.go | 186 ++++++++++++++++++++++++++++++++- twitter_test.go | 186 +-------------------------------- joiner.go => util.go | 0 joiner_test.go => util_test.go | 0 6 files changed, 205 insertions(+), 202 deletions(-) rename joiner.go => util.go (100%) rename joiner_test.go => util_test.go (100%) diff --git a/twitter.go b/twitter.go index 2f010de..fc8a3ee 100644 --- a/twitter.go +++ b/twitter.go @@ -7,7 +7,7 @@ import ( type FollowerGetter interface { GetFollowerByCursor(screenName, cursor string) <-chan *FollowerList - GetGetFollowerIdsByCursor(screenName, cursor string) <-chan *FollowerIDList + GetFollowerIdsByCursor(screenName, cursor string) <-chan *FollowerIDList GetScreenNameOfUsersByIds(ids []uint64) <-chan string } @@ -32,7 +32,7 @@ func GetFollowerIds(followerGetter FollowerGetter, screenName string) <-chan uin go func() { nextCursor := "-1" for nextCursor != "0" && nextCursor != "" { - followers := <-followerGetter.GetGetFollowerIdsByCursor(screenName, nextCursor) + followers := <-followerGetter.GetFollowerIdsByCursor(screenName, nextCursor) nextCursor = followers.NextCursor for _, follower := range followers.Followers { followerC <- follower @@ -43,7 +43,7 @@ func GetFollowerIds(followerGetter FollowerGetter, screenName string) <-chan uin return followerC } -func GetTwitterScreenNameByIds(followerGetter FollowerGetter, idsC <-chan uint64) <-chan string { +func GetScreenNameByIds(followerGetter FollowerGetter, idsC <-chan uint64) <-chan string { screenNameC := make(chan string) var wg sync.WaitGroup @@ -81,8 +81,8 @@ func GetFollowerIdsOfBothAccounts(followerGetter FollowerGetter, screenName1, sc func main() { token := "AAAAAAAAAAAAAAAAAAAAAPwfcQAAAAAAzkou%2FHjJNJmwdepeRq0c%2Bi3Nx6o%3DXofLt7SVvc99ulETLRA3yS2lYo8smfc6tACxEYsLUmGsrNbc9J" t := NewTwitterApi(TWITTER_API_URL, token) - ff := Intersection(GetFollowerIds(t, "Shopify"), GetFollowerIds(t, "ShopifyPicks")) - for screenName := range GetTwitterScreenNameByIds(t, ff) { + followers := Intersection(GetFollowerIds(t, "Shopify"), GetFollowerIds(t, "ShopifyPicks")) + for screenName := range GetScreenNameByIds(t, followers) { fmt.Println(screenName) } } diff --git a/twitterApi.go b/twitterApi.go index 9d48e67..9548e7d 100644 --- a/twitterApi.go +++ b/twitterApi.go @@ -33,7 +33,14 @@ func (t *TwitterApi) encodeParams(params map[string]string) string { q.Set(k, v) } return q.Encode() +} +func (t *TwitterApi) asCommaSeparatedString(lst []uint64) string { + lstAsString := make([]string, len(lst)) + for i, n := range lst { + lstAsString[i] = fmt.Sprintf("%v", n) + } + return strings.Join(lstAsString, ",") } func (t *TwitterApi) createGetPathAndParams(apiEndpoint string, params map[string]string) string { @@ -70,7 +77,7 @@ func (t *TwitterApi) Post(path_ string, body string) (resp *http.Response, err e return nil, err } req.Header.Set("Authorization", "Bearer "+t.AccessToken) - req.Header.Set("content-type", "application/json; charset=utf-8") + req.Header.Set("content-type", "application/x-www-form-urlencoded") return http.DefaultClient.Do(req) } @@ -146,7 +153,7 @@ func (t *TwitterApi) GetFollowerByCursor(screenName, cursor string) <-chan *Foll return followerListC } -func (t *TwitterApi) GetGetFollowerIdsByCursor(screenName, cursor string) <-chan *FollowerIDList { +func (t *TwitterApi) GetFollowerIdsByCursor(screenName, cursor string) <-chan *FollowerIDList { followerListC := make(chan *FollowerIDList) go func() { @@ -158,7 +165,7 @@ func (t *TwitterApi) GetGetFollowerIdsByCursor(screenName, cursor string) <-chan } else if twitterErr, ok := err.(*TwitterErr); ok && twitterErr.Status == 429 { log.Println("api limit reached, need to sleep") time.Sleep(5 * 60 * time.Second) - followerListC <- <-t.GetGetFollowerIdsByCursor(screenName, cursor) + followerListC <- <-t.GetFollowerIdsByCursor(screenName, cursor) } else { log.Println(err) followerListC <- nil @@ -175,9 +182,9 @@ func (t *TwitterApi) GetScreenNameOfUsersByIds(ids []uint64) <-chan string { screenNameC := make(chan string) go func() { path := "/users/lookup.json" - params := map[string]string{"include_entities": "false", "user_id": t.asCommaSeparatedString(ids)} + params := map[string]string{"user_id": t.asCommaSeparatedString(ids)} users := make([]*User, 0, len(ids)) - if err := t.PostAndDeserialize(path, params, users); err == nil { + if err := t.PostAndDeserialize(path, params, &users); err == nil { for _, user := range users { screenNameC <- user.ScreenName } @@ -194,11 +201,3 @@ func (t *TwitterApi) GetScreenNameOfUsersByIds(ids []uint64) <-chan string { }() return screenNameC } - -func (t *TwitterApi) asCommaSeparatedString(lst []uint64) string { - lstAsString := make([]string, len(lst)) - for i, n := range lst { - lstAsString[i] = fmt.Sprintf("%v", n) - } - return strings.Join(lstAsString, ",") -} diff --git a/twitterApi_test.go b/twitterApi_test.go index 87a753f..83b4b45 100644 --- a/twitterApi_test.go +++ b/twitterApi_test.go @@ -1,6 +1,13 @@ package main -import "testing" +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) func TestcreateGetPathAndParams(t *testing.T) { tw := NewTwitterApi("twitter.com", "access_token") @@ -19,3 +26,180 @@ func TestGetBase64EncodedBearerTokenCredentials(t *testing.T) { t.Error("expected: ", expt, "\nbut received:", res) } } + +func TestAsCommaSeparatedString(t *testing.T) { + tw := new(TwitterApi) + if s := tw.asCommaSeparatedString([]uint64{10765432100123456789, 78}); s != "10765432100123456789,78" { + t.Error(s) + } +} + +func TestGet(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + t.Error("bad request verb") + } + if auth := r.Header.Get("Authorization"); auth != "Bearer access_token" { + t.Error("bad authorization in header :", auth) + } + if contentType := r.Header.Get("Content-Type"); contentType != "application/json; charset=utf-8" { + t.Error("bad content-type in header ", contentType) + } + })) + defer ts.Close() + tw := NewTwitterApi(ts.URL, "access_token") + + if res, err := tw.Get("/"); err != nil { + t.Error(err) + } else { + ioutil.ReadAll(res.Body) + } +} + +func TestPost(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + t.Error("bad request verb") + } + if auth := r.Header.Get("Authorization"); auth != "Bearer access_token" { + t.Error("bad authorization in header :", auth) + } + if contentType := r.Header.Get("Content-Type"); contentType != "application/x-www-form-urlencoded" { + t.Error("bad content-type in header ", contentType) + } + if body, err := ioutil.ReadAll(r.Body); err != nil { + t.Error(err) + } else if string(body) != `{"data": "is_hugue"}` { + t.Error("bad body", string(body)) + } + })) + defer ts.Close() + tw := NewTwitterApi(ts.URL, "access_token") + + if res, err := tw.Post("/", `{"data": "is_hugue"}`); err != nil { + t.Error(err) + } else { + ioutil.ReadAll(res.Body) + } +} + +func TestGetAndDeserializeError(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + fmt.Fprint(w, `{"error": "bar"}`) + })) + defer ts.Close() + tw := NewTwitterApi(ts.URL, "access_token") + m := make(map[string]string) + if err := tw.GetAndDeserialize("/foo", map[string]string{}, &m); err == nil { + t.Error("shoud have an error message here") + } else if err.Error() != `{"error": "bar"}` { + t.Error("error message should propagate here") + } else if terr, ok := err.(*TwitterErr); !ok { + t.Error("the error should be a twitter error") + } else if terr.Status != 404 { + t.Error("bad status") + } +} + +func TestGetAndDeserialize(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("foo") != "bar" { + t.Error("bad params") + } + fmt.Fprint(w, `{"foo": "bar"}`) + })) + defer ts.Close() + tw := NewTwitterApi(ts.URL, "access_token") + m := make(map[string]string) + if err := tw.GetAndDeserialize("/foo", map[string]string{"foo": "bar"}, &m); err != nil { + t.Error(err) + } + if m["foo"] != "bar" { + t.Fail() + } +} + +func TestPostAndDeserialize(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if body, err := ioutil.ReadAll(r.Body); err != nil { + t.Error(err) + } else if string(body) != `val=7` { + t.Error("bad body") + } + fmt.Fprint(w, `{"foo": "bar"}`) + })) + defer ts.Close() + tw := NewTwitterApi(ts.URL, "access_token") + m := make(map[string]string) + + if err := tw.PostAndDeserialize("/foo", map[string]string{"val": "7"}, &m); err != nil { + t.Error(err) + } + if m["foo"] != "bar" { + t.Fail() + } +} + +func TestGetTwitterIdByScreenName(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if url_ := r.URL.String(); url_ != "/users/lookup.json?include_entities=id&screen_name=bobLeChef" { + t.Error("bad url :", url_) + } + fmt.Fprint(w, `[{"id": 1492}]`) + + })) + defer ts.Close() + tw := NewTwitterApi(ts.URL, "access_token") + + if id, err := tw.GetTwitterIdByScreenName("bobLeChef"); err != nil { + t.Error(err) + } else if id != 1492 { + t.Error("found bad id") + } +} + +func TestGetFollowerByCursor(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if url_ := r.URL.String(); url_ != "/followers/list.json?count=200&cursor=89&screen_name=bobLeChef&skip_status=true" { + t.Error("bad url :", url_) + } + fmt.Fprint(w, `{"users": [{"id": 1492, "screen_name": "bob_le_chef"}], "next_cursor_str": "1793"}`) + })) + defer ts.Close() + tw := NewTwitterApi(ts.URL, "access_token") + + followersListC := tw.GetFollowerByCursor("bobLeChef", "89") + followers := <-followersListC + + if followers.NextCursor != "1793" { + t.Error("bad cursor") + } else if len(followers.Followers) != 1 { + t.Error("bad number of followers") + } else if user := followers.Followers[0]; !reflect.DeepEqual(user, &User{"bob_le_chef", 1492}) { + t.Error("bad user", user) + } +} + +func TestGetFollowerIdsByCursor(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if url_ := r.URL.String(); url_ != "/followers/ids.json?count=5000&cursor=89&screen_name=bobLeChef" { + t.Error("bad url :", url_) + } + fmt.Fprint(w, `{"ids": [1492], "next_cursor_str": "1793"}`) + })) + defer ts.Close() + tw := NewTwitterApi(ts.URL, "access_token") + + followersListC := tw.GetFollowerIdsByCursor("bobLeChef", "89") + + followers := <-followersListC + + if followers.NextCursor != "1793" { + t.Error("bad cursor") + } else if len(followers.Followers) != 1 { + t.Error("bad number of followers", len(followers.Followers)) + } else if id := followers.Followers[0]; id != 1492 { + t.Error("bad id") + } +} diff --git a/twitter_test.go b/twitter_test.go index f20fb84..fe59951 100644 --- a/twitter_test.go +++ b/twitter_test.go @@ -2,183 +2,10 @@ package main import ( "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" "reflect" "testing" ) -func TestGet(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { - t.Error("bad request verb") - } - if auth := r.Header.Get("Authorization"); auth != "Bearer access_token" { - t.Error("bad authorization in header :", auth) - } - if contentType := r.Header.Get("Content-Type"); contentType != "application/json; charset=utf-8" { - t.Error("bad content-type in header ", contentType) - } - })) - defer ts.Close() - tw := NewTwitterApi(ts.URL, "access_token") - - if res, err := tw.Get("/"); err != nil { - t.Error(err) - } else { - ioutil.ReadAll(res.Body) - } -} - -func TestPost(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { - t.Error("bad request verb") - } - if auth := r.Header.Get("Authorization"); auth != "Bearer access_token" { - t.Error("bad authorization in header :", auth) - } - if contentType := r.Header.Get("Content-Type"); contentType != "application/json; charset=utf-8" { - t.Error("bad content-type in header ", contentType) - } - if body, err := ioutil.ReadAll(r.Body); err != nil { - t.Error(err) - } else if string(body) != `{"data": "is_hugue"}` { - t.Error("bad body", string(body)) - } - })) - defer ts.Close() - tw := NewTwitterApi(ts.URL, "access_token") - - if res, err := tw.Post("/", `{"data": "is_hugue"}`); err != nil { - t.Error(err) - } else { - ioutil.ReadAll(res.Body) - } -} - -func TestGetAndDeserializeError(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(404) - fmt.Fprint(w, `{"error": "bar"}`) - })) - defer ts.Close() - tw := NewTwitterApi(ts.URL, "access_token") - m := make(map[string]string) - if err := tw.GetAndDeserialize("/foo", map[string]string{}, &m); err == nil { - t.Error("shoud have an error message here") - } else if err.Error() != `{"error": "bar"}` { - t.Error("error message should propagate here") - } else if terr, ok := err.(*TwitterErr); !ok { - t.Error("the error should be a twitter error") - } else if terr.Status != 404 { - t.Error("bad status") - } -} - -func TestGetAndDeserialize(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("foo") != "bar" { - t.Error("bad params") - } - fmt.Fprint(w, `{"foo": "bar"}`) - })) - defer ts.Close() - tw := NewTwitterApi(ts.URL, "access_token") - m := make(map[string]string) - if err := tw.GetAndDeserialize("/foo", map[string]string{"foo": "bar"}, &m); err != nil { - t.Error(err) - } - if m["foo"] != "bar" { - t.Fail() - } -} - -func TestPostAndDeserialize(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if body, err := ioutil.ReadAll(r.Body); err != nil { - t.Error(err) - } else if string(body) != `val=7` { - t.Error("bad body") - } - fmt.Fprint(w, `{"foo": "bar"}`) - })) - defer ts.Close() - tw := NewTwitterApi(ts.URL, "access_token") - m := make(map[string]string) - - if err := tw.PostAndDeserialize("/foo", map[string]string{"val": "7"}, &m); err != nil { - t.Error(err) - } - if m["foo"] != "bar" { - t.Fail() - } -} - -func TestGetTwitterIdByScreenName(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if url_ := r.URL.String(); url_ != "/users/lookup.json?include_entities=id&screen_name=bobLeChef" { - t.Error("bad url :", url_) - } - fmt.Fprint(w, `[{"id": 1492}]`) - - })) - defer ts.Close() - tw := NewTwitterApi(ts.URL, "access_token") - - if id, err := tw.GetTwitterIdByScreenName("bobLeChef"); err != nil { - t.Error(err) - } else if id != 1492 { - t.Error("found bad id") - } -} - -func TestGetFollowerByCursor(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if url_ := r.URL.String(); url_ != "/followers/list.json?count=200&cursor=89&screen_name=bobLeChef&skip_status=true" { - t.Error("bad url :", url_) - } - fmt.Fprint(w, `{"users": [{"id": 1492, "screen_name": "bob_le_chef"}], "next_cursor_str": "1793"}`) - })) - defer ts.Close() - tw := NewTwitterApi(ts.URL, "access_token") - - followersListC := tw.GetFollowerByCursor("bobLeChef", "89") - followers := <-followersListC - - if followers.NextCursor != "1793" { - t.Error("bad cursor") - } else if len(followers.Followers) != 1 { - t.Error("bad number of followers") - } else if user := followers.Followers[0]; !reflect.DeepEqual(user, &User{"bob_le_chef", 1492}) { - t.Error("bad user", user) - } -} - -func TestGetFollowerIdsByCursor(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if url_ := r.URL.String(); url_ != "/followers/ids.json?count=5000&cursor=89&screen_name=bobLeChef" { - t.Error("bad url :", url_) - } - fmt.Fprint(w, `{"ids": [1492], "next_cursor_str": "1793"}`) - })) - defer ts.Close() - tw := NewTwitterApi(ts.URL, "access_token") - - followersListC := tw.GetGetFollowerIdsByCursor("bobLeChef", "89") - - followers := <-followersListC - - if followers.NextCursor != "1793" { - t.Error("bad cursor") - } else if len(followers.Followers) != 1 { - t.Error("bad number of followers", len(followers.Followers)) - } else if id := followers.Followers[0]; id != 1492 { - t.Error("bad id") - } -} - type MockFollowerGetter struct { T *testing.T } @@ -203,7 +30,7 @@ func (m *MockFollowerGetter) GetFollowerByCursor(screenName, cursor string) <-ch return followerListC } -func (m *MockFollowerGetter) GetGetFollowerIdsByCursor(screenName, cursor string) <-chan *FollowerIDList { +func (m *MockFollowerGetter) GetFollowerIdsByCursor(screenName, cursor string) <-chan *FollowerIDList { followerListC := make(chan *FollowerIDList) go func() { switch cursor { @@ -265,13 +92,6 @@ func TestGetFollowerIds(t *testing.T) { } } -func TestAsCommaSeparatedString(t *testing.T) { - tw := new(TwitterApi) - if s := tw.asCommaSeparatedString([]uint64{10765432100123456789, 78}); s != "10765432100123456789,78" { - t.Error(s) - } -} - func uint64Range(n int) []uint64 { ret := make([]uint64, n) for i := 0; i < n; i++ { @@ -280,11 +100,11 @@ func uint64Range(n int) []uint64 { return ret } -func TestGetTwitterScreenNameByIds(t *testing.T) { +func TestGetScreenNameByIds(t *testing.T) { fg := &MockFollowerGetter{t} ids := uint64Range(233) ret := make(map[string]bool) - for name := range GetTwitterScreenNameByIds(fg, makeUInt64Channel(ids...)) { + for name := range GetScreenNameByIds(fg, makeUInt64Channel(ids...)) { ret[name] = true } if len(ret) != 233 { diff --git a/joiner.go b/util.go similarity index 100% rename from joiner.go rename to util.go diff --git a/joiner_test.go b/util_test.go similarity index 100% rename from joiner_test.go rename to util_test.go