From 3273079ffd12699b0984b27520bbbf43041f77d3 Mon Sep 17 00:00:00 2001 From: Ian Clark Date: Tue, 22 Nov 2016 17:09:02 -0600 Subject: [PATCH 1/5] Issue #150 CHANGE: - Moved all redis stuff into `./server/peerStore` - peerStore/redisStore.go Now creates connectiosn on-the-fly. This is a degradation for performance and shouldn't be necessary. However, I'm lazy and Redis is going to be trashed for Olivia. In the future, a better solution would be to send all redis operations to a central store and have that store process via one connection. --- reaper/reaper.go | 6 +-- server/announce.go | 15 ++++--- server/announce_response.go | 5 ++- server/peerStore/peerstore.go | 9 ++++ server/{redis.go => peerStore/redisStore.go} | 46 +++++++++++--------- server/{ => peerStore}/redis_test.go | 2 +- server/server.go | 11 ++--- 7 files changed, 56 insertions(+), 38 deletions(-) create mode 100644 server/peerStore/peerstore.go rename server/{redis.go => peerStore/redisStore.go} (76%) rename server/{ => peerStore}/redis_test.go (99%) diff --git a/reaper/reaper.go b/reaper/reaper.go index 6e228b3..e265724 100644 --- a/reaper/reaper.go +++ b/reaper/reaper.go @@ -3,7 +3,7 @@ package reaper import ( "fmt" "github.com/GrappigPanda/notorious/database" - "github.com/GrappigPanda/notorious/server" + "github.com/GrappigPanda/notorious/server/peerStore" "gopkg.in/redis.v3" "strconv" "strings" @@ -72,7 +72,7 @@ func StartReapingScheduler(waitTime time.Duration) { go func() { for { // Handle any other cleanup or Notorious-related functions - c := server.OpenClient() + c := peerStore.OpenClient() _, err := c.Ping().Result() if err != nil { panic("No Redis instance detected. If deploying without Docker, install redis-server") @@ -86,7 +86,7 @@ func StartReapingScheduler(waitTime time.Duration) { x, err := db.GetWhitelistedTorrents() for x.Next() { x.Scan(infoHash, name, addedBy, dateAdded) - server.CreateNewTorrentKey(c, *infoHash) + peerStore.CreateNewTorrentKey(*infoHash) } // Start the actual peer reaper. diff --git a/server/announce.go b/server/announce.go index d671e60..3b14889 100644 --- a/server/announce.go +++ b/server/announce.go @@ -6,6 +6,7 @@ import ( "net/url" "strconv" "strings" + "github.com/GrappigPanda/notorious/server/peerStore" ) func (a *announceData) parseAnnounceData(req *http.Request) (err error) { @@ -56,7 +57,7 @@ func (a *announceData) parseAnnounceData(req *http.Request) (err error) { a.event = "started" } - a.requestContext.redisClient = OpenClient() + a.requestContext.redisClient = peerStore.OpenClient() return } @@ -103,8 +104,8 @@ func (a *announceData) StartedEventHandler() (err error) { ipport = fmt.Sprintf("%s:%d", a.ip, a.port) } - RedisSetKeyVal(a.requestContext.redisClient, keymember, ipport) - if RedisSetKeyIfNotExists(a.requestContext.redisClient, keymember, ipport) { + peerStore.RedisSetKeyVal(a.requestContext.redisClient, keymember, ipport) + if peerStore.RedisSetKeyIfNotExists(a.requestContext.redisClient, keymember, ipport) { fmt.Printf("Adding host %s to %s\n", ipport, keymember) } @@ -139,7 +140,7 @@ func (a *announceData) CompletedEventHandler() { keymember := fmt.Sprintf("%s:complete", a.info_hash) // TODO(ian): DRY! ipport := fmt.Sprintf("%s:%s", a.ip, a.port) - if RedisSetKeyIfNotExists(a.requestContext.redisClient, keymember, ipport) { + if peerStore.RedisSetKeyIfNotExists(a.requestContext.redisClient, keymember, ipport) { fmt.Printf("Adding host %s to %s:complete\n", ipport, a.info_hash) } } @@ -150,15 +151,15 @@ func (a *announceData) removeFromKVStorage(subkey string) { keymember := fmt.Sprintf("%s:%s", a.info_hash, subkey) fmt.Printf("Removing host %s from %v\n", ipport, keymember) - RedisRemoveKeysValue(a.requestContext.redisClient, keymember, ipport) + peerStore.RedisRemoveKeysValue(a.requestContext.redisClient, keymember, ipport) } func (a *announceData) infoHashExists() bool { - return RedisGetBoolKeyVal(a.requestContext.redisClient, a.info_hash) + return peerStore.RedisGetBoolKeyVal(a.info_hash) } func (a *announceData) createInfoHashKey() { - CreateNewTorrentKey(a.requestContext.redisClient, a.info_hash) + peerStore.CreateNewTorrentKey(a.info_hash) } // ParseInfoHash parses the encoded info hash. Such a simple solution for a diff --git a/server/announce_response.go b/server/announce_response.go index 221d3d4..cf0a7f9 100644 --- a/server/announce_response.go +++ b/server/announce_response.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/GrappigPanda/notorious/bencode" "github.com/GrappigPanda/notorious/database" + "github.com/GrappigPanda/notorious/server/peerStore" "net" "strconv" "strings" @@ -70,8 +71,8 @@ func formatResponseData(ips []string, data *announceData) string { // string that we respond with. func EncodeResponse(ipport []string, data *announceData) (resp string) { ret := "" - completeCount := len(RedisGetKeyVal(data, data.info_hash)) - incompleteCount := len(RedisGetKeyVal(data, data.info_hash)) + completeCount := len(peerStore.RedisGetKeyVal(data.info_hash)) + incompleteCount := len(peerStore.RedisGetKeyVal(data.info_hash)) ret += bencode.EncodeKV("complete", bencode.EncodeInt(completeCount)) ret += bencode.EncodeKV("incomplete", bencode.EncodeInt(incompleteCount)) diff --git a/server/peerStore/peerstore.go b/server/peerStore/peerstore.go new file mode 100644 index 0000000..a684098 --- /dev/null +++ b/server/peerStore/peerstore.go @@ -0,0 +1,9 @@ +package peerStore + +type PeerStore interface { + SetKeyIfNotExists(string, string) bool + SetKV(string, string) bool + RemoveKV(string, string) + KeyExists(string) bool + GetKeyVal(string) []string +} diff --git a/server/redis.go b/server/peerStore/redisStore.go similarity index 76% rename from server/redis.go rename to server/peerStore/redisStore.go index a71d009..957d420 100644 --- a/server/redis.go +++ b/server/peerStore/redisStore.go @@ -1,4 +1,4 @@ -package server +package peerStore import ( "bytes" @@ -24,16 +24,16 @@ func OpenClient() (client *redis.Client) { } // RedisSetIPMember sets a key as a member of an infohash and sets a timeout. -func RedisSetIPMember(data *announceData) (retval int) { - c := data.requestContext.redisClient +func RedisSetIPMember(infoHash, ipPort string) (retval int) { + c := OpenClient() - keymember := concatenateKeyMember(data.info_hash, "ip") + keymember := concatenateKeyMember(infoHash, "ip") currTime := int64(time.Now().UTC().AddDate(0, 0, 1).Unix()) - ipPort := fmt.Sprintf("%s:%v", createIPPortPair(data), currTime) + key := fmt.Sprintf("%s:%v", ipPort, currTime) - if err := c.SAdd(keymember, ipPort).Err(); err != nil { + if err := c.SAdd(keymember, key).Err(); err != nil { retval = 0 panic("Failed to add key") @@ -61,30 +61,34 @@ func RedisSetKeyVal(c *redis.Client, keymember string, value string) { } // RedisGetKeyVal Lookup a peer in the specified infohash at `key` -func RedisGetKeyVal(data *announceData, key string) []string { +func RedisGetKeyVal(key string) []string { + c := OpenClient() + // RedisGetKeyVal retrieves a value from the Redis store by looking up the // provided key. If the key does not yet exist, we create the key in the KV // storage or if the value is empty, we add the current requester to the // list. keymember := concatenateKeyMember(key, "complete") - val, err := data.requestContext.redisClient.SMembers(keymember).Result() + val, err := c.SMembers(keymember).Result() if err != nil { // Fail because the key doesn't exist in the KV storage. - CreateNewTorrentKey(data.requestContext.redisClient, keymember) + CreateNewTorrentKey(keymember) } return val } // RedisGetAllPeers fetches all peers from the info_hash at `key` -func RedisGetAllPeers(data *announceData, key string) []string { +func RedisGetAllPeers(key string) []string { + c := OpenClient() + keymember := concatenateKeyMember(key, "complete") - val, err := data.requestContext.redisClient.SRandMemberN(keymember, 30).Result() + val, err := c.SRandMemberN(keymember, 30).Result() if err != nil { // Fail because the key doesn't exist in the KV storage. - CreateNewTorrentKey(data.requestContext.redisClient, keymember) + CreateNewTorrentKey(keymember) } if len(val) == 30 { @@ -93,7 +97,7 @@ func RedisGetAllPeers(data *announceData, key string) []string { keymember = concatenateKeyMember(key, "incomplete") - val2, err := data.requestContext.redisClient.SRandMemberN(keymember, int64(30-len(val))).Result() + val2, err := c.SRandMemberN(keymember, int64(30-len(val))).Result() if err != nil { panic("Failed to get incomplete peers for") } else { @@ -120,15 +124,16 @@ func RedisGetCount(c *redis.Client, info_hash string, member string) (retval int } // RedisGetBoolKeyVal Checks if a `key` exists -func RedisGetBoolKeyVal(client *redis.Client, key string) bool { - ret, _ := client.Exists(key).Result() +func RedisGetBoolKeyVal(key string) bool { + c := OpenClient() + ret, _ := c.Exists(key).Result() return ret } // RedisSetKeyIfNotExists Set a key if it doesn't exist. func RedisSetKeyIfNotExists(c *redis.Client, keymember string, value string) (rv bool) { - rv = RedisGetBoolKeyVal(c, keymember) + rv = RedisGetBoolKeyVal(keymember) if !rv { RedisSetKeyVal(c, keymember, value) } @@ -145,8 +150,9 @@ func RedisRemoveKeysValue(c *redis.Client, key string, value string) { // CreateNewTorrentKey creates a new key. By default, it adds a member // ":ip". I don't think this ought to ever be generalized, as I just want // Redis to function in one specific way in notorious. -func CreateNewTorrentKey(client *redis.Client, key string) { - client.SAdd(key, "complete", "incomplete") +func CreateNewTorrentKey(key string) { + c := OpenClient() + c.SAdd(key, "complete", "incomplete") } @@ -164,6 +170,6 @@ func concatenateKeyMember(key string, member string) string { // createIPPortPair creates a string formatted ("%s:%s", value.ip, // value.port) looking like so: "127.0.0.1:6886" and returns this value. -func createIPPortPair(value *announceData) string { - return fmt.Sprintf("%v:%v", value.ip, value.port) +func createIPPortPair(ip, port string) string { + return fmt.Sprintf("%v:%v", ip, port) } diff --git a/server/redis_test.go b/server/peerStore/redis_test.go similarity index 99% rename from server/redis_test.go rename to server/peerStore/redis_test.go index fdf2a34..968655e 100644 --- a/server/redis_test.go +++ b/server/peerStore/redis_test.go @@ -1,4 +1,4 @@ -package server +package peerStore import ( "testing" diff --git a/server/server.go b/server/server.go index 57e31b8..a0650d1 100644 --- a/server/server.go +++ b/server/server.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/GrappigPanda/notorious/config" "github.com/GrappigPanda/notorious/database" + "github.com/GrappigPanda/notorious/server/peerStore" "net/http" ) @@ -11,16 +12,16 @@ import ( var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"} func worker(data *announceData) []string { - if RedisGetBoolKeyVal(data.requestContext.redisClient, data.info_hash) { - x := RedisGetKeyVal(data, data.info_hash) + if peerStore.RedisGetBoolKeyVal(data.info_hash) { + x := peerStore.RedisGetKeyVal(data.info_hash) - RedisSetIPMember(data) + peerStore.RedisSetIPMember(data.info_hash, fmt.Sprintf("%s:%s", data.ip, data.port)) return x } - CreateNewTorrentKey(data.requestContext.redisClient, data.info_hash) + peerStore.CreateNewTorrentKey(data.info_hash) return worker(data) } func (app *applicationContext) handleStatsTracking(data *announceData) { @@ -75,7 +76,7 @@ func (app *applicationContext) requestHandler(w http.ResponseWriter, req *http.R if data.event == "started" || data.event == "completed" { worker(data) - x := RedisGetAllPeers(data, data.info_hash) + x := peerStore.RedisGetAllPeers(data.info_hash) if len(x) > 0 { response := formatResponseData(x, data) From bc80fecb676bb48366bfa05ea2356db497d4d013 Mon Sep 17 00:00:00 2001 From: Ian Clark Date: Tue, 22 Nov 2016 17:19:08 -0600 Subject: [PATCH 2/5] Issue #150 CHANGE: - server/peerStore/redis_test.go Importing `server` package, as we need the definitions from it. --- server/peerStore/redis_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/peerStore/redis_test.go b/server/peerStore/redis_test.go index 968655e..3906f16 100644 --- a/server/peerStore/redis_test.go +++ b/server/peerStore/redis_test.go @@ -2,6 +2,7 @@ package peerStore import ( "testing" + "github.com/GrappigPanda/notorious/server" ) var reqContext = requestAppContext{ From a879c23b09848a7911b1cfd8ecfd187f68bf114c Mon Sep 17 00:00:00 2001 From: Ian Clark Date: Thu, 24 Nov 2016 23:13:23 -0600 Subject: [PATCH 3/5] Issue #150 Major refactor of server/annoucne -> announce/announce.go --- {server => announce}/announce.go | 98 +++--- .../announce_test.go | 2 +- announce/definitions.go | 46 +++ server/announce_response.go | 11 +- server/announce_test.go | 50 --- server/definitions.go | 82 ----- server/peerStore/redis_test.go | 309 ------------------ server/server.go | 81 +++-- 8 files changed, 162 insertions(+), 517 deletions(-) rename {server => announce}/announce.go (55%) rename server/server_test.go => announce/announce_test.go (99%) create mode 100644 announce/definitions.go delete mode 100644 server/announce_test.go delete mode 100644 server/definitions.go delete mode 100644 server/peerStore/redis_test.go diff --git a/server/announce.go b/announce/announce.go similarity index 55% rename from server/announce.go rename to announce/announce.go index 3b14889..c04a077 100644 --- a/server/announce.go +++ b/announce/announce.go @@ -1,4 +1,4 @@ -package server +package announce import ( "fmt" @@ -9,55 +9,61 @@ import ( "github.com/GrappigPanda/notorious/server/peerStore" ) -func (a *announceData) parseAnnounceData(req *http.Request) (err error) { +func (a *AnnounceData) ParseAnnounceData(req *http.Request) (err error) { query := req.URL.Query() - a.info_hash = ParseInfoHash(query.Get("info_hash")) - if a.info_hash == "" { - err = fmt.Errorf("No info_hash provided.") + + a.RequestContext = requestAppContext{ + dbConn: nil, + Whitelist: false, + } + + a.InfoHash = ParseInfoHash(query.Get("InfoHash")) + if a.InfoHash == "" { + err = fmt.Errorf("No InfoHash provided.") return } if strings.Contains(req.RemoteAddr, ":") { - a.ip = strings.Split(req.RemoteAddr, ":")[0] + a.IP = strings.Split(req.RemoteAddr, ":")[0] } else { - a.ip = query.Get(req.RemoteAddr) + a.IP = query.Get(req.RemoteAddr) } - a.peer_id = query.Get("peer_id") + a.PeerID = query.Get("peer_id") - a.port, err = GetInt(query, "port") + a.Port, err = GetInt(query, "port") if err != nil { return fmt.Errorf("Failed to get port") } - a.downloaded, err = GetInt(query, "downloaded") + a.Downloaded, err = GetInt(query, "downloaded") if err != nil { err = fmt.Errorf("Failed to get downloaded byte count.") return } - a.uploaded, err = GetInt(query, "uploaded") + a.Uploaded, err = GetInt(query, "uploaded") if err != nil { err = fmt.Errorf("Failed to get uploaded byte count.") return } - a.left, err = GetInt(query, "left") + a.Left, err = GetInt(query, "left") if err != nil { err = fmt.Errorf("Failed to get remaining byte count.") return } - a.numwant, err = GetInt(query, "numwant") + a.Numwant, err = GetInt(query, "numwant") if err != nil { - a.numwant = 0 + a.Numwant = 0 } if x := query.Get("compact"); x != "" { - a.compact, err = strconv.ParseBool(x) + a.Compact, err = strconv.ParseBool(x) if err != nil { - a.compact = false + a.Compact = false } } - a.event = query.Get("event") - if a.event == " " || a.event == "" { - a.event = "started" + a.Event = query.Get("event") + if a.Event == " " || a.Event == "" { + a.Event = "started" } - a.requestContext.redisClient = peerStore.OpenClient() + a.RequestContext.redisClient = peerStore.OpenClient() return } @@ -79,16 +85,16 @@ func GetInt(u url.Values, key string) (ui uint64, err error) { // StartedEventHandler handles whenever a peer sends the STARTED event to the // tracker. -func (a *announceData) StartedEventHandler() (err error) { +func (a *AnnounceData) StartedEventHandler() (err error) { // Called upon announce when a client starts a download or creates a new // torrent on the tracker. Adds a user to incomplete list in redis. err = nil - if !a.infoHashExists() && a.requestContext.whitelist { + if !a.infoHashExists() && a.RequestContext.Whitelist { err = fmt.Errorf("Torrent not authorized for use") return - } else if !a.infoHashExists() && !a.requestContext.whitelist { - // If the info hash isn't in redis and we're not whitelisting, add it + } else if !a.infoHashExists() && !a.RequestContext.Whitelist { + // If the info hash isn't in redis and we're not Whitelisting, add it // to Redis. a.createInfoHashKey() } @@ -96,16 +102,16 @@ func (a *announceData) StartedEventHandler() (err error) { keymember := "" ipport := "" - if !(a.left == 0) { - keymember = fmt.Sprintf("%s:incomplete", a.info_hash) - ipport = fmt.Sprintf("%s:%d", a.ip, a.port) + if !(a.Left == 0) { + keymember = fmt.Sprintf("%s:incomplete", a.InfoHash) + ipport = fmt.Sprintf("%s:%d", a.IP, a.Port) } else { - keymember = fmt.Sprintf("%s:complete", a.info_hash) - ipport = fmt.Sprintf("%s:%d", a.ip, a.port) + keymember = fmt.Sprintf("%s:complete", a.InfoHash) + ipport = fmt.Sprintf("%s:%d", a.IP, a.Port) } - peerStore.RedisSetKeyVal(a.requestContext.redisClient, keymember, ipport) - if peerStore.RedisSetKeyIfNotExists(a.requestContext.redisClient, keymember, ipport) { + peerStore.RedisSetKeyVal(a.RequestContext.redisClient, keymember, ipport) + if peerStore.RedisSetKeyIfNotExists(a.RequestContext.redisClient, keymember, ipport) { fmt.Printf("Adding host %s to %s\n", ipport, keymember) } @@ -117,10 +123,10 @@ func (a *announceData) StartedEventHandler() (err error) { // TODO(ian): This is what happened whenever the torrent client shuts down // gracefully, so we need to call the mysql backend and store the info and // remove the ipport from completed/incomplete redis kvs -func (a *announceData) StoppedEventHandler() { +func (a *AnnounceData) StoppedEventHandler() { if a.infoHashExists() { - a.removeFromKVStorage(a.event) + a.removeFromKVStorage(a.Event) } else { return } @@ -129,7 +135,7 @@ func (a *announceData) StoppedEventHandler() { // CompletedEventHandler Called upon announce when a client finishes a download. Removes the // client from incomplete in redis and places their peer info into // complete. -func (a *announceData) CompletedEventHandler() { +func (a *AnnounceData) CompletedEventHandler() { if !a.infoHashExists() { a.createInfoHashKey() @@ -137,29 +143,29 @@ func (a *announceData) CompletedEventHandler() { a.removeFromKVStorage("incomplete") } - keymember := fmt.Sprintf("%s:complete", a.info_hash) + keymember := fmt.Sprintf("%s:complete", a.InfoHash) // TODO(ian): DRY! - ipport := fmt.Sprintf("%s:%s", a.ip, a.port) - if peerStore.RedisSetKeyIfNotExists(a.requestContext.redisClient, keymember, ipport) { - fmt.Printf("Adding host %s to %s:complete\n", ipport, a.info_hash) + ipport := fmt.Sprintf("%s:%s", a.IP, a.Port) + if peerStore.RedisSetKeyIfNotExists(a.RequestContext.redisClient, keymember, ipport) { + fmt.Printf("Adding host %s to %s:complete\n", ipport, a.InfoHash) } } -func (a *announceData) removeFromKVStorage(subkey string) { +func (a *AnnounceData) removeFromKVStorage(subkey string) { // Remove the subkey from the kv storage. - ipport := fmt.Sprintf("%s:%d", a.ip, a.port) - keymember := fmt.Sprintf("%s:%s", a.info_hash, subkey) + ipport := fmt.Sprintf("%s:%d", a.IP, a.Port) + keymember := fmt.Sprintf("%s:%s", a.InfoHash, subkey) fmt.Printf("Removing host %s from %v\n", ipport, keymember) - peerStore.RedisRemoveKeysValue(a.requestContext.redisClient, keymember, ipport) + peerStore.RedisRemoveKeysValue(a.RequestContext.redisClient, keymember, ipport) } -func (a *announceData) infoHashExists() bool { - return peerStore.RedisGetBoolKeyVal(a.info_hash) +func (a *AnnounceData) infoHashExists() bool { + return peerStore.RedisGetBoolKeyVal(a.InfoHash) } -func (a *announceData) createInfoHashKey() { - peerStore.CreateNewTorrentKey(a.info_hash) +func (a *AnnounceData) createInfoHashKey() { + peerStore.CreateNewTorrentKey(a.InfoHash) } // ParseInfoHash parses the encoded info hash. Such a simple solution for a diff --git a/server/server_test.go b/announce/announce_test.go similarity index 99% rename from server/server_test.go rename to announce/announce_test.go index ec6e56f..e66cd0f 100644 --- a/server/server_test.go +++ b/announce/announce_test.go @@ -1,4 +1,4 @@ -package server +package announce import ( "fmt" diff --git a/announce/definitions.go b/announce/definitions.go new file mode 100644 index 0000000..c00c8a1 --- /dev/null +++ b/announce/definitions.go @@ -0,0 +1,46 @@ +package announce + +import ( + "github.com/jinzhu/gorm" + "gopkg.in/redis.v3" +) + +const ( + RATIOLESS = iota + SEMIRATIOLESS + NORMALRATIO +) + +type AnnounceData struct { + InfoHash string //20 byte sha1 hash + PeerID string //max len 20 + IP string //optional + Event string // TorrentEvent + + Port uint64 // port number the peer is listening + // on + + Uploaded uint64 // base10 ascii amount uploaded so far + Downloaded uint64 // base10 ascii amount downloaded so + // far + + Left uint64 // # of bytes left to download + // (base 10 ascii) + + Numwant uint64 // Number of peers requested by client. + + Compact bool // Bep23 peer list compression + // decision: True -> compress bep23 + + RequestContext requestAppContext // The request-specific connections +} + +// requestAppContext First of all naming things is the hardest part of +// programming real talk. Second of all, this essentially houses +// request-specific data like db connections and in the future the redisClient. +// Things that should persist only within the duration of a request. +type requestAppContext struct { + dbConn *gorm.DB + redisClient *redis.Client // The redis client connection handler to use. + Whitelist bool +} diff --git a/server/announce_response.go b/server/announce_response.go index cf0a7f9..070fe8f 100644 --- a/server/announce_response.go +++ b/server/announce_response.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/GrappigPanda/notorious/bencode" "github.com/GrappigPanda/notorious/database" + . "github.com/GrappigPanda/notorious/announce" "github.com/GrappigPanda/notorious/server/peerStore" "net" "strconv" @@ -63,20 +64,20 @@ func CompactAllPeers(ipport []string) []byte { return ret.Bytes() } -func formatResponseData(ips []string, data *announceData) string { +func formatResponseData(ips []string, data *AnnounceData) string { return EncodeResponse(ips, data) } // EncodeResponse groups all of the peer-requested data into a nice bencoded // string that we respond with. -func EncodeResponse(ipport []string, data *announceData) (resp string) { +func EncodeResponse(ipport []string, data *AnnounceData) (resp string) { ret := "" - completeCount := len(peerStore.RedisGetKeyVal(data.info_hash)) - incompleteCount := len(peerStore.RedisGetKeyVal(data.info_hash)) + completeCount := len(peerStore.RedisGetKeyVal(data.InfoHash)) + incompleteCount := len(peerStore.RedisGetKeyVal(data.InfoHash)) ret += bencode.EncodeKV("complete", bencode.EncodeInt(completeCount)) ret += bencode.EncodeKV("incomplete", bencode.EncodeInt(incompleteCount)) - if data.compact || !data.compact { + if data.Compact || !data.Compact { ipstr := string(CompactAllPeers(ipport)) ret += bencode.EncodeKV("peers", ipstr) } else { diff --git a/server/announce_test.go b/server/announce_test.go deleted file mode 100644 index 8520c3d..0000000 --- a/server/announce_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package server - -import ( - "github.com/GrappigPanda/notorious/database" - "testing" -) - -var DBCONN, _ = db.OpenConnection() - -var CONTEXT = requestAppContext{ - dbConn: DBCONN, - redisClient: OpenClient(), - whitelist: true, -} - -var ANNOUNCEDATA = &announceData{ - info_hash: "54321543215432154321", - peer_id: "11111111111111111111", - ip: "127.0.0.1", - event: "STARTED", - port: 6667, - uploaded: 0, - downloaded: 0, - left: 0, - numwant: 30, - compact: true, - requestContext: CONTEXT, -} - -// TestStartedEventHandler tests that with a whitelist being active, we can not -// add a new info_hash to the tracker. -func TestStartedEventHandler(t *testing.T) { - err := ANNOUNCEDATA.StartedEventHandler() - - if err == nil { - t.Fatalf("Failed to TestStartedEventHandler: %v", err) - } -} - -// TestStartedEventHandler tests that with a whitelist being active, we can not -// add a new info_hash to the tracker. -func TestStartedEventHandlerNoWhitelist(t *testing.T) { - announce2 := ANNOUNCEDATA - announce2.requestContext.whitelist = false - err := announce2.StartedEventHandler() - - if err != nil { - t.Fatalf("Failed to TestStartedEventHandler: %v", err) - } -} diff --git a/server/definitions.go b/server/definitions.go deleted file mode 100644 index 4f7e337..0000000 --- a/server/definitions.go +++ /dev/null @@ -1,82 +0,0 @@ -package server - -import ( - "github.com/GrappigPanda/notorious/config" - "github.com/jinzhu/gorm" - "gopkg.in/redis.v3" -) - -const ( - RATIOLESS = iota - SEMIRATIOLESS - NORMALRATIO -) - -type announceData struct { - info_hash string //20 byte sha1 hash - peer_id string //max len 20 - ip string //optional - event string // TorrentEvent - - port uint64 // port number the peer is listening - // on - - uploaded uint64 // base10 ascii amount uploaded so far - downloaded uint64 // base10 ascii amount downloaded so - // far - - left uint64 // # of bytes left to download - // (base 10 ascii) - - numwant uint64 // Number of peers requested by client. - - compact bool // Bep23 peer list compression - // decision: True -> compress bep23 - - requestContext requestAppContext // The request-specific connections -} - -// requestAppContext First of all naming things is the hardest part of -// programming real talk. Second of all, this essentially houses -// request-specific data like db connections and in the future the redisClient. -// Things that should persist only within the duration of a request. -type requestAppContext struct { - dbConn *gorm.DB - redisClient *redis.Client // The redis client connection handler to use. - whitelist bool -} - -type scrapeData struct { - infoHash string -} - -// scrapeResponse is the data associated with a returned scrape. -type scrapeResponse struct { - complete uint64 - downloaded uint64 - incomplete uint64 -} - -// TorrentResponseData models what is sent back to the peer upon a succesful -// info hash lookup. -type TorrentResponseData struct { - interval int - min_interval int - tracker_id string - completed int - incomplete int - peers interface{} -} - -// ANNOUNCE_URL The announce path for the http calls to reach. -var ANNOUNCE_URL = "/announce" - -// TODO(ian): Set this expireTime to a config-loaded value. -// expireTime := 5 * 60 - -// applicationContext houses data necessary for the handler to properly -// function as the application is desired. -type applicationContext struct { - config config.ConfigStruct - trackerLevel int -} diff --git a/server/peerStore/redis_test.go b/server/peerStore/redis_test.go deleted file mode 100644 index 3906f16..0000000 --- a/server/peerStore/redis_test.go +++ /dev/null @@ -1,309 +0,0 @@ -package peerStore - -import ( - "testing" - "github.com/GrappigPanda/notorious/server" -) - -var reqContext = requestAppContext{ - redisClient: OpenClient(), - dbConn: nil, -} - -var DATA = announceData{ - info_hash: "12345123451234512345", - peer_id: "12345123451234512345", - ip: "127.0.0.1", - event: "STARTED", - port: 6767, - uploaded: 1024, - downloaded: 512, - left: 0, - numwant: 30, - compact: true, - requestContext: reqContext, -} - -func TestRedisSetIPMember(t *testing.T) { - ret := RedisSetIPMember(&DATA) - - expectedReturn := 1 - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisSetKeyVal(t *testing.T) { - RedisSetKeyVal(DATA.requestContext.redisClient, "test:1234", "1024") - - ret, _ := DATA.requestContext.redisClient.SMembers("test:1234").Result() - - expectedReturn := ">1" - - if len(ret) == 0 { - t.Fatalf("Expected %v, got %v", expectedReturn, len(ret)) - } -} - -func TestRedisGetKeyVal(t *testing.T) { - DATA.requestContext.redisClient.SAdd("RedisGetKeyValTest:1024:complete", "1024") - ret := RedisGetKeyVal(&DATA, "RedisGetKeyValTest:1024") - expectedReturn := ">1" - - if len(ret) == 0 { - t.Fatalf("Expected %v, got %v", expectedReturn, len(ret)) - } -} - -func TestRedisGetKeyValNoPreexistKey(t *testing.T) { - DATA.requestContext.redisClient.SAdd("RedisGetKeyValTest:1025", "1024") - ret := RedisGetKeyVal(&DATA, "RedisGetKeyValTest:1025") - expectedReturn := 0 - - if len(ret) != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, len(ret)) - } -} - -func TestCreateIpPortPair(t *testing.T) { - expectedReturn := "127.0.0.1:6767" - ret := createIPPortPair(&DATA) - - if expectedReturn != ret { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestConcatenateKeyMember(t *testing.T) { - expectedReturn := "127.0.0.1:1024" - ret := concatenateKeyMember("127.0.0.1", "1024") - - if expectedReturn != ret { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestCreateNewTorrentKey(t *testing.T) { - CreateNewTorrentKey(DATA.requestContext.redisClient, "testTestCreateNewTorrentKey") - - ret, err := DATA.requestContext.redisClient.Exists("testTestCreateNewTorrentKey").Result() - if err != nil { - t.Fatalf("%v", err) - } - if ret != true { - t.Fatalf("CreateNewTorrentKey:complete failed to create") - } - - ret, err = DATA.requestContext.redisClient.SIsMember("testTestCreateNewTorrentKey", "complete").Result() - if ret != true { - t.Fatalf("testTestCreateNewTorrentKey:complete is not a member") - } - - ret, err = DATA.requestContext.redisClient.SIsMember("testTestCreateNewTorrentKey", "incomplete").Result() - if ret != true { - t.Fatalf("testTestCreateNewTorrentKey:incomplete is not a member") - } - -} - -func TestRedisRemoveKeyValues(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisRemoveKeyVal", "Test1") - ret, err := DATA.requestContext.redisClient.SIsMember("TestRedisRemoveKeyVal", "Test1").Result() - if err != nil { - t.Fatalf("%v", err) - } - if ret != true { - t.Fatalf("Failed in setup of TestRedisRemoveKeyValues to add a key") - } - - RedisRemoveKeysValue(DATA.requestContext.redisClient, "TestRedisRemoveKeyVal", "Test1") - ret, err = DATA.requestContext.redisClient.SIsMember("TestRedisRemoveKeyVal", "Test1").Result() - if err != nil { - t.Fatalf("%v", err) - } - if ret == true { - t.Fatalf("RedisRemoveKeyVal failed to remove the key") - } - -} - -func TestRedisGetBoolKeyVal(t *testing.T) { - RedisSetKeyVal(DATA.requestContext.redisClient, "TestRedisGetBoolKeyVal", "1024") - - expectedReturn := true - ret := RedisGetBoolKeyVal(DATA.requestContext.redisClient, "TestRedisGetBoolKeyVal") - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisSetKeyIfNotExists(t *testing.T) { - expectedReturn := false - ret := RedisSetKeyIfNotExists(DATA.requestContext.redisClient, "TestRedisSetKeyIfNotExists", "1024") - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisSetKeyIfNotExistsPreExistingKey(t *testing.T) { - expectedReturn := true - RedisSetKeyVal(DATA.requestContext.redisClient, "TestRedisSetKeyIfNotExists", "1024") - ret := RedisSetKeyIfNotExists(DATA.requestContext.redisClient, "TestRedisSetKeyIfNotExists", "1024") - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisGetCount(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisGetCount", "Test") - DATA.requestContext.redisClient.SAdd("TestRedisGetCount:Test", "1235") - DATA.requestContext.redisClient.SAdd("TestRedisGetCount:Test", "1236") - DATA.requestContext.redisClient.SAdd("TestRedisGetCount:Test", "1237") - DATA.requestContext.redisClient.SAdd("TestRedisGetCount:Test", "1238") - - expectedReturn := 4 - ret, err := RedisGetCount(DATA.requestContext.redisClient, "TestRedisGetCount", "Test") - if err != nil { - t.Fatalf("%v", err) - } - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisGetAllPeers(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers", "complete") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers:complete", "1235") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers:complete", "1236") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers:complete", "1237") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers:complete", "1238") - - ret := RedisGetAllPeers(&DATA, "TestRedisGetAllPeers") - x := len(ret) - - if x != 4 { - t.Fatalf("Expected 4 peers, got %v", x) - } -} - -func TestRedisGetAllPeersValGT30(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1", "complete") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1201") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1202") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1203") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1204") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1205") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1206") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1207") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1208") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1209") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1210") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1211") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1212") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1213") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1214") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1215") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1216") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1217") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1218") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1209") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1200") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1201") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1202") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1203") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1204") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1205") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1216") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1217") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1218") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1219") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1220") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1221") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1222") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1221") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1222") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1223") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1224") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1225") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1226") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1227") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1228") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1229") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1230") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1231") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1232") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1233") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1234") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1235") - - ret := RedisGetAllPeers(&DATA, "TestRedisGetAllPeers1") - x := len(ret) - - if x != 30 { - t.Fatalf("Expected 30 peers, got %v", x) - } -} - -func TestRedisGetAllPeersValLT30(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2", "complete") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1201") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1202") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1203") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1204") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1205") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1206") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1207") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1208") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1209") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1210") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1211") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1212") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1213") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1214") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1215") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2", "incomplete") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1216") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1217") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1218") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1209") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1200") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1201") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1202") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1203") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1204") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1205") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1216") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1217") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1218") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1219") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1220") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1221") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1222") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1221") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1222") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1223") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1224") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1225") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1226") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1227") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1228") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1229") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1230") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1231") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1232") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1233") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1234") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1235") - - ret := RedisGetAllPeers(&DATA, "TestRedisGetAllPeers2") - x := len(ret) - - if x != 30 { - t.Fatalf("Expected 30 peers, got %v", x) - } -} diff --git a/server/server.go b/server/server.go index a0650d1..1428238 100644 --- a/server/server.go +++ b/server/server.go @@ -4,60 +4,93 @@ import ( "fmt" "github.com/GrappigPanda/notorious/config" "github.com/GrappigPanda/notorious/database" + . "github.com/GrappigPanda/notorious/announce" "github.com/GrappigPanda/notorious/server/peerStore" "net/http" ) +// applicationContext houses data necessary for the handler to properly +// function as the application is desired. +type applicationContext struct { + config config.ConfigStruct + trackerLevel int +} + +type scrapeData struct { + infoHash string +} + +// scrapeResponse is the data associated with a returned scrape. +type scrapeResponse struct { + complete uint64 + downloaded uint64 + incomplete uint64 +} + +// TorrentResponseData models what is sent back to the peer upon a succesful +// info hash lookup. +type TorrentResponseData struct { + interval int + min_interval int + tracker_id string + completed int + incomplete int + peers interface{} +} + +// ANNOUNCE_URL The announce path for the http calls to reach. +var ANNOUNCE_URL = "/announce" + +// TODO(ian): Set this expireTime to a config-loaded value. +// expireTime := 5 * 60 // FIELDS The fields that we expect from a peer upon info hash lookup var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"} -func worker(data *announceData) []string { - if peerStore.RedisGetBoolKeyVal(data.info_hash) { - x := peerStore.RedisGetKeyVal(data.info_hash) +func worker(data *AnnounceData) []string { + if peerStore.RedisGetBoolKeyVal(data.InfoHash) { + x := peerStore.RedisGetKeyVal(data.InfoHash) - peerStore.RedisSetIPMember(data.info_hash, fmt.Sprintf("%s:%s", data.ip, data.port)) + peerStore.RedisSetIPMember(data.InfoHash, fmt.Sprintf("%s:%s", data.IP, data.Port)) return x } - peerStore.CreateNewTorrentKey(data.info_hash) + peerStore.CreateNewTorrentKey(data.InfoHash) return worker(data) } -func (app *applicationContext) handleStatsTracking(data *announceData) { - db.UpdateStats(data.uploaded, data.downloaded) +func (app *applicationContext) handleStatsTracking(data *AnnounceData) { + db.UpdateStats(data.Uploaded, data.Downloaded) if app.trackerLevel > RATIOLESS { - db.UpdatePeerStats(data.uploaded, data.downloaded, data.ip) + db.UpdatePeerStats(data.Uploaded, data.Downloaded, data.IP) } - if data.event == "completed" { + if data.Event == "completed" { db.UpdateTorrentStats(1, -1) return - } else if data.left == 0 { + } else if data.Left == 0 { // TODO(ian): Don't assume the peer is already in the DB db.UpdateTorrentStats(1, -1) return - } else if data.event == "started" { + } else if data.Event == "started" { db.UpdateTorrentStats(0, 1) } } func (app *applicationContext) requestHandler(w http.ResponseWriter, req *http.Request) { - data := new(announceData) - data.requestContext = requestAppContext{ - dbConn: nil, - whitelist: app.config.Whitelist, - } - - err := data.parseAnnounceData(req) + data := new(AnnounceData) + err := data.ParseAnnounceData(req) if err != nil { panic(err) + } - fmt.Printf("Event: %s from host %s on port %v\n", data.event, data.ip, data.port) + data.RequestContext.Whitelist = app.config.Whitelist + + fmt.Printf("Event: %s from host %s on port %v\n", data.Event, data.IP, data.Port) - switch data.event { + switch data.Event { case "started": err := data.StartedEventHandler() if err != nil { @@ -74,9 +107,9 @@ func (app *applicationContext) requestHandler(w http.ResponseWriter, req *http.R panic(fmt.Errorf("We're somehow getting this strange error...")) } - if data.event == "started" || data.event == "completed" { + if data.Event == "started" || data.Event == "completed" { worker(data) - x := peerStore.RedisGetAllPeers(data.info_hash) + x := peerStore.RedisGetAllPeers(data.InfoHash) if len(x) > 0 { response := formatResponseData(x, data) @@ -84,7 +117,7 @@ func (app *applicationContext) requestHandler(w http.ResponseWriter, req *http.R } else { failMsg := fmt.Sprintf("No peers for torrent %s\n", - data.info_hash) + data.InfoHash) writeErrorResponse(w, failMsg) } } @@ -99,7 +132,7 @@ func scrapeHandler(w http.ResponseWriter, req *http.Request) { panic(err) } - infoHash := query.Get("info_hash") + infoHash := query.Get("InfoHash") if infoHash == "" { failMsg := fmt.Sprintf("Tracker does not support multiple entire DB scrapes.") writeErrorResponse(w, failMsg) From b47e4b562d4d6f73e692861555591b7d755b37ba Mon Sep 17 00:00:00 2001 From: Ian Clark Date: Thu, 24 Nov 2016 23:42:19 -0600 Subject: [PATCH 4/5] Issue #150 CHANGE: - server/peerStore/peerstore.go Added a new method all peerStores should implement. - server/peerStore/redisStore.go Added new wrapper methods to implement the `PeerStore` interface. - server/server.go Still attempting to decouple heavily spaghettied code. --- server/peerStore/peerstore.go | 3 +- server/peerStore/redisStore.go | 53 +++++++++++++++++++++++++--------- server/server.go | 24 +++++++++------ 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/server/peerStore/peerstore.go b/server/peerStore/peerstore.go index a684098..a09be4f 100644 --- a/server/peerStore/peerstore.go +++ b/server/peerStore/peerstore.go @@ -2,8 +2,9 @@ package peerStore type PeerStore interface { SetKeyIfNotExists(string, string) bool - SetKV(string, string) bool + SetKV(string, string) RemoveKV(string, string) KeyExists(string) bool GetKeyVal(string) []string + GetAllPeers(string) []string } diff --git a/server/peerStore/redisStore.go b/server/peerStore/redisStore.go index 957d420..c3232c9 100644 --- a/server/peerStore/redisStore.go +++ b/server/peerStore/redisStore.go @@ -8,6 +8,10 @@ import ( "time" ) +type RedisStore struct { + client *redis.Client +} + // EXPIRETIME signifies how long a peer will live under the specified info_hash // until the reaper removes it. var EXPIRETIME int64 = 5 * 60 @@ -23,8 +27,31 @@ func OpenClient() (client *redis.Client) { return } +func (p *RedisStore) SetKeyIfNotExists(key, value string) (retval bool) { + return redisSetKeyIfNotExists(p.client, key, value) +} + +func (p *RedisStore) SetKV(key, value string) { + redisSetKeyVal(p.client, key, value) +} + +func (p *RedisStore) RemoveKV(key, value string) { + // TODO(ian): Refactor this so we don't have to delete a value from a key + if value != "" || value == "" { + redisRemoveKeysValue(p.client, key, value) + } +} + +func (p *RedisStore) KeyExists(key string) (retval bool) { + return redisGetBoolKeyVal(key) +} + +func (p *RedisStore) GetKeyVal(key string) []string { + return redisGetKeyVal(key) +} + // RedisSetIPMember sets a key as a member of an infohash and sets a timeout. -func RedisSetIPMember(infoHash, ipPort string) (retval int) { +func redisSetIPMember(infoHash, ipPort string) (retval int) { c := OpenClient() keymember := concatenateKeyMember(infoHash, "ip") @@ -46,7 +73,7 @@ func RedisSetIPMember(infoHash, ipPort string) (retval int) { // RedisSetKeyVal Sets a key to the specified value. Used mostly with adding a // peer into an info_hash -func RedisSetKeyVal(c *redis.Client, keymember string, value string) { +func redisSetKeyVal(c *redis.Client, keymember string, value string) { // RedisSetKeyVal sets a key:member's value to value. Returns nothing as of // yet. currTime := int64(time.Now().UTC().Unix()) @@ -61,7 +88,7 @@ func RedisSetKeyVal(c *redis.Client, keymember string, value string) { } // RedisGetKeyVal Lookup a peer in the specified infohash at `key` -func RedisGetKeyVal(key string) []string { +func redisGetKeyVal(key string) []string { c := OpenClient() // RedisGetKeyVal retrieves a value from the Redis store by looking up the @@ -73,14 +100,14 @@ func RedisGetKeyVal(key string) []string { val, err := c.SMembers(keymember).Result() if err != nil { // Fail because the key doesn't exist in the KV storage. - CreateNewTorrentKey(keymember) + createNewTorrentKey(keymember) } return val } // RedisGetAllPeers fetches all peers from the info_hash at `key` -func RedisGetAllPeers(key string) []string { +func redisGetAllPeers(key string) []string { c := OpenClient() keymember := concatenateKeyMember(key, "complete") @@ -88,7 +115,7 @@ func RedisGetAllPeers(key string) []string { val, err := c.SRandMemberN(keymember, 30).Result() if err != nil { // Fail because the key doesn't exist in the KV storage. - CreateNewTorrentKey(keymember) + createNewTorrentKey(keymember) } if len(val) == 30 { @@ -108,7 +135,7 @@ func RedisGetAllPeers(key string) []string { } // RedisGetCount counts all of the peers at `info_hash` -func RedisGetCount(c *redis.Client, info_hash string, member string) (retval int, err error) { +func redisGetCount(c *redis.Client, info_hash string, member string) (retval int, err error) { // A generic function which is used to retrieve either the complete count // or the incomplete count for a specified `info_hash`. keymember := concatenateKeyMember(info_hash, member) @@ -124,7 +151,7 @@ func RedisGetCount(c *redis.Client, info_hash string, member string) (retval int } // RedisGetBoolKeyVal Checks if a `key` exists -func RedisGetBoolKeyVal(key string) bool { +func redisGetBoolKeyVal(key string) bool { c := OpenClient() ret, _ := c.Exists(key).Result() @@ -132,10 +159,10 @@ func RedisGetBoolKeyVal(key string) bool { } // RedisSetKeyIfNotExists Set a key if it doesn't exist. -func RedisSetKeyIfNotExists(c *redis.Client, keymember string, value string) (rv bool) { - rv = RedisGetBoolKeyVal(keymember) +func redisSetKeyIfNotExists(c *redis.Client, keymember string, value string) (rv bool) { + rv = redisGetBoolKeyVal(keymember) if !rv { - RedisSetKeyVal(c, keymember, value) + redisSetKeyVal(c, keymember, value) } return } @@ -143,14 +170,14 @@ func RedisSetKeyIfNotExists(c *redis.Client, keymember string, value string) (rv // RedisRemoveKeysValue Remove a `value` from `key` in the redis kv storage. `key` is typically // a keymember of info_hash:(in)complete and the value is typically the // ip:port concatenated. -func RedisRemoveKeysValue(c *redis.Client, key string, value string) { +func redisRemoveKeysValue(c *redis.Client, key string, value string) { c.SRem(key, value) } // CreateNewTorrentKey creates a new key. By default, it adds a member // ":ip". I don't think this ought to ever be generalized, as I just want // Redis to function in one specific way in notorious. -func CreateNewTorrentKey(key string) { +func createNewTorrentKey(key string) { c := OpenClient() c.SAdd(key, "complete", "incomplete") diff --git a/server/server.go b/server/server.go index 1428238..c30e680 100644 --- a/server/server.go +++ b/server/server.go @@ -14,6 +14,7 @@ import ( type applicationContext struct { config config.ConfigStruct trackerLevel int + peerStoreClient *peerStore.PeerStore } type scrapeData struct { @@ -46,19 +47,21 @@ var ANNOUNCE_URL = "/announce" // FIELDS The fields that we expect from a peer upon info hash lookup var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"} -func worker(data *AnnounceData) []string { - if peerStore.RedisGetBoolKeyVal(data.InfoHash) { - x := peerStore.RedisGetKeyVal(data.InfoHash) +func (app *applicationContext) worker(data *AnnounceData) []string { + if app.peerStoreClient.KeyExists(data.InfoHash) { + x := peerStore.GetKeyVal(data.InfoHash) - peerStore.RedisSetIPMember(data.InfoHash, fmt.Sprintf("%s:%s", data.IP, data.Port)) + app.peerStoreClient.RedisSetIPMember(data.InfoHash, fmt.Sprintf("%s:%s", data.IP, data.Port)) return x - } + } else { + app.peerStoreClient.SetKV(data.InfoHash) + } - peerStore.CreateNewTorrentKey(data.InfoHash) - return worker(data) + return app.worker(data) } + func (app *applicationContext) handleStatsTracking(data *AnnounceData) { db.UpdateStats(data.Uploaded, data.Downloaded) @@ -108,8 +111,8 @@ func (app *applicationContext) requestHandler(w http.ResponseWriter, req *http.R } if data.Event == "started" || data.Event == "completed" { - worker(data) - x := peerStore.RedisGetAllPeers(data.InfoHash) + app.worker(data) + x := app.peerStoreClient.GetAllPeers(data.InfoHash) if len(x) > 0 { response := formatResponseData(x, data) @@ -155,9 +158,12 @@ func writeResponse(w http.ResponseWriter, values string) { // RunServer spins up the server and muxes the routes. func RunServer() { + peerStore := new(peerStore.RedisStore) + app := applicationContext{ config: config.LoadConfig(), trackerLevel: RATIOLESS, + peerStoreClient: peerStore.redisStore, } mux := http.NewServeMux() From 790826d29650650ecc251ed21f7b779cdab3fd3c Mon Sep 17 00:00:00 2001 From: Ian Clark Date: Fri, 25 Nov 2016 11:03:41 -0600 Subject: [PATCH 5/5] Issue #150 CHANGE: - kvStoreInterfaces/redis.go Now houses common redis calls used **throughout** the application. The `server/peerStore/redisStore.go` file just implements a wrapper around these. --- announce/announce.go | 16 +-- kvStoreInterfaces/redis.go | 175 +++++++++++++++++++++++++++++++ reaper/reaper.go | 6 +- server/announce_response.go | 6 +- server/peerStore/peerstore.go | 1 + server/peerStore/redisStore.go | 181 ++------------------------------- server/server.go | 13 ++- 7 files changed, 206 insertions(+), 192 deletions(-) create mode 100644 kvStoreInterfaces/redis.go diff --git a/announce/announce.go b/announce/announce.go index c04a077..918f911 100644 --- a/announce/announce.go +++ b/announce/announce.go @@ -6,7 +6,7 @@ import ( "net/url" "strconv" "strings" - "github.com/GrappigPanda/notorious/server/peerStore" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" ) func (a *AnnounceData) ParseAnnounceData(req *http.Request) (err error) { @@ -63,7 +63,7 @@ func (a *AnnounceData) ParseAnnounceData(req *http.Request) (err error) { a.Event = "started" } - a.RequestContext.redisClient = peerStore.OpenClient() + a.RequestContext.redisClient = r.OpenClient() return } @@ -110,8 +110,8 @@ func (a *AnnounceData) StartedEventHandler() (err error) { ipport = fmt.Sprintf("%s:%d", a.IP, a.Port) } - peerStore.RedisSetKeyVal(a.RequestContext.redisClient, keymember, ipport) - if peerStore.RedisSetKeyIfNotExists(a.RequestContext.redisClient, keymember, ipport) { + r.RedisSetKeyVal(a.RequestContext.redisClient, keymember, ipport) + if r.RedisSetKeyIfNotExists(a.RequestContext.redisClient, keymember, ipport) { fmt.Printf("Adding host %s to %s\n", ipport, keymember) } @@ -146,7 +146,7 @@ func (a *AnnounceData) CompletedEventHandler() { keymember := fmt.Sprintf("%s:complete", a.InfoHash) // TODO(ian): DRY! ipport := fmt.Sprintf("%s:%s", a.IP, a.Port) - if peerStore.RedisSetKeyIfNotExists(a.RequestContext.redisClient, keymember, ipport) { + if r.RedisSetKeyIfNotExists(a.RequestContext.redisClient, keymember, ipport) { fmt.Printf("Adding host %s to %s:complete\n", ipport, a.InfoHash) } } @@ -157,15 +157,15 @@ func (a *AnnounceData) removeFromKVStorage(subkey string) { keymember := fmt.Sprintf("%s:%s", a.InfoHash, subkey) fmt.Printf("Removing host %s from %v\n", ipport, keymember) - peerStore.RedisRemoveKeysValue(a.RequestContext.redisClient, keymember, ipport) + r.RedisRemoveKeysValue(a.RequestContext.redisClient, keymember, ipport) } func (a *AnnounceData) infoHashExists() bool { - return peerStore.RedisGetBoolKeyVal(a.InfoHash) + return r.RedisGetBoolKeyVal(a.InfoHash) } func (a *AnnounceData) createInfoHashKey() { - peerStore.CreateNewTorrentKey(a.InfoHash) + r.CreateNewTorrentKey(a.InfoHash) } // ParseInfoHash parses the encoded info hash. Such a simple solution for a diff --git a/kvStoreInterfaces/redis.go b/kvStoreInterfaces/redis.go new file mode 100644 index 0000000..83a1fe6 --- /dev/null +++ b/kvStoreInterfaces/redis.go @@ -0,0 +1,175 @@ +package kvStoreInterface + +import ( + "gopkg.in/redis.v3" + "bytes" + "fmt" + "strings" + "time" +) + +// EXPIRETIME signifies how long a peer will live under the specified info_hash +// until the reaper removes it. +var EXPIRETIME int64 = 5 * 60 + +// OpenClient opens a connection to redis. +func OpenClient() (client *redis.Client) { + client = redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", + DB: 0, + }) + + return +} + +// RedisSetIPMember sets a key as a member of an infohash and sets a timeout. +func RedisSetIPMember(infoHash, ipPort string) (retval int) { + c := OpenClient() + + keymember := concatenateKeyMember(infoHash, "ip") + + currTime := int64(time.Now().UTC().AddDate(0, 0, 1).Unix()) + + key := fmt.Sprintf("%s:%v", ipPort, currTime) + + if err := c.SAdd(keymember, key).Err(); err != nil { + retval = 0 + panic("Failed to add key") + + } else { + retval = 1 + } + + return +} + +// RedisSetKeyVal Sets a key to the specified value. Used mostly with adding a +// peer into an info_hash +func RedisSetKeyVal(c *redis.Client, keymember string, value string) { + // RedisSetKeyVal sets a key:member's value to value. Returns nothing as of + // yet. + currTime := int64(time.Now().UTC().Unix()) + currTime += EXPIRETIME + value = fmt.Sprintf("%v:%v", value, currTime) + + if sz := strings.Split(value, ":"); len(sz) >= 1 { + // If the value being added can be converted to an int, it is a ip:port key + // and we can set an expiration on it. + c.SAdd(keymember, value) + } +} + +// RedisGetKeyVal Lookup a peer in the specified infohash at `key` +func RedisGetKeyVal(key string) []string { + c := OpenClient() + + // RedisGetKeyVal retrieves a value from the Redis store by looking up the + // provided key. If the key does not yet exist, we Create the key in the KV + // storage or if the value is empty, we add the current requester to the + // list. + keymember := concatenateKeyMember(key, "complete") + + val, err := c.SMembers(keymember).Result() + if err != nil { + // Fail because the key doesn't exist in the KV storage. + CreateNewTorrentKey(keymember) + } + + return val +} + +// RedisGetAllPeers fetches all peers from the info_hash at `key` +func RedisGetAllPeers(key string) []string { + c := OpenClient() + + keymember := concatenateKeyMember(key, "complete") + + val, err := c.SRandMemberN(keymember, 30).Result() + if err != nil { + // Fail because the key doesn't exist in the KV storage. + CreateNewTorrentKey(keymember) + } + + if len(val) == 30 { + return val + } + + keymember = concatenateKeyMember(key, "incomplete") + + val2, err := c.SRandMemberN(keymember, int64(30-len(val))).Result() + if err != nil { + panic("Failed to get incomplete peers for") + } else { + val = append(val, val2...) + } + + return val +} + +// RedisGetCount counts all of the peers at `info_hash` +func RedisGetCount(c *redis.Client, info_hash string, member string) (retval int, err error) { + // A generic function which is used to retrieve either the complete count + // or the incomplete count for a specified `info_hash`. + keymember := concatenateKeyMember(info_hash, member) + + x, err := c.SMembers(keymember).Result() + if err != nil { + // TODO(ian): Add actual error checking here. + err = fmt.Errorf("The info hash %s with member %s doesn't exist", info_hash, member) + } + + retval = len(x) + return +} + +// RedisGetBoolKeyVal Checks if a `key` exists +func RedisGetBoolKeyVal(key string) bool { + c := OpenClient() + ret, _ := c.Exists(key).Result() + + return ret +} + +// RedisSetKeyIfNotExists Set a key if it doesn't exist. +func RedisSetKeyIfNotExists(c *redis.Client, keymember string, value string) (rv bool) { + rv = RedisGetBoolKeyVal(keymember) + if !rv { + RedisSetKeyVal(c, keymember, value) + } + return +} + +// RedisRemoveKeysValue Remove a `value` from `key` in the redis kv storage. `key` is typically +// a keymember of info_hash:(in)complete and the value is typically the +// ip:port concatenated. +func RedisRemoveKeysValue(c *redis.Client, key string, value string) { + c.SRem(key, value) +} + +// CreateNewTorrentKey Creates a new key. By default, it adds a member +// ":ip". I don't think this ought to ever be generalized, as I just want +// Redis to function in one specific way in notorious. +func CreateNewTorrentKey(key string) { + c := OpenClient() + c.SAdd(key, "complete", "incomplete") + +} + +// concatenateKeyMember concatenates the key and the member delimited by the +// character ":" +func concatenateKeyMember(key string, member string) string { + var buffer bytes.Buffer + + buffer.WriteString(key) + buffer.WriteString(":") + buffer.WriteString(member) + + return buffer.String() +} + +// CreateIPPortPair Creates a string formatted ("%s:%s", value.ip, +// value.port) looking like so: "127.0.0.1:6886" and returns this value. +func createIPPortPair(ip, port string) string { + return fmt.Sprintf("%v:%v", ip, port) +} diff --git a/reaper/reaper.go b/reaper/reaper.go index e265724..eaae9bf 100644 --- a/reaper/reaper.go +++ b/reaper/reaper.go @@ -3,7 +3,7 @@ package reaper import ( "fmt" "github.com/GrappigPanda/notorious/database" - "github.com/GrappigPanda/notorious/server/peerStore" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" "gopkg.in/redis.v3" "strconv" "strings" @@ -72,7 +72,7 @@ func StartReapingScheduler(waitTime time.Duration) { go func() { for { // Handle any other cleanup or Notorious-related functions - c := peerStore.OpenClient() + c := r.OpenClient() _, err := c.Ping().Result() if err != nil { panic("No Redis instance detected. If deploying without Docker, install redis-server") @@ -86,7 +86,7 @@ func StartReapingScheduler(waitTime time.Duration) { x, err := db.GetWhitelistedTorrents() for x.Next() { x.Scan(infoHash, name, addedBy, dateAdded) - peerStore.CreateNewTorrentKey(*infoHash) + r.CreateNewTorrentKey(*infoHash) } // Start the actual peer reaper. diff --git a/server/announce_response.go b/server/announce_response.go index 070fe8f..d6e86eb 100644 --- a/server/announce_response.go +++ b/server/announce_response.go @@ -6,7 +6,7 @@ import ( "github.com/GrappigPanda/notorious/bencode" "github.com/GrappigPanda/notorious/database" . "github.com/GrappigPanda/notorious/announce" - "github.com/GrappigPanda/notorious/server/peerStore" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" "net" "strconv" "strings" @@ -72,8 +72,8 @@ func formatResponseData(ips []string, data *AnnounceData) string { // string that we respond with. func EncodeResponse(ipport []string, data *AnnounceData) (resp string) { ret := "" - completeCount := len(peerStore.RedisGetKeyVal(data.InfoHash)) - incompleteCount := len(peerStore.RedisGetKeyVal(data.InfoHash)) + completeCount := len(r.RedisGetKeyVal(data.InfoHash)) + incompleteCount := len(r.RedisGetKeyVal(data.InfoHash)) ret += bencode.EncodeKV("complete", bencode.EncodeInt(completeCount)) ret += bencode.EncodeKV("incomplete", bencode.EncodeInt(incompleteCount)) diff --git a/server/peerStore/peerstore.go b/server/peerStore/peerstore.go index a09be4f..5ada357 100644 --- a/server/peerStore/peerstore.go +++ b/server/peerStore/peerstore.go @@ -7,4 +7,5 @@ type PeerStore interface { KeyExists(string) bool GetKeyVal(string) []string GetAllPeers(string) []string + SetIPMember(string, string) int } diff --git a/server/peerStore/redisStore.go b/server/peerStore/redisStore.go index c3232c9..93b56f5 100644 --- a/server/peerStore/redisStore.go +++ b/server/peerStore/redisStore.go @@ -1,202 +1,41 @@ package peerStore import ( - "bytes" - "fmt" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" "gopkg.in/redis.v3" - "strings" - "time" ) type RedisStore struct { client *redis.Client } -// EXPIRETIME signifies how long a peer will live under the specified info_hash -// until the reaper removes it. -var EXPIRETIME int64 = 5 * 60 - -// OpenClient opens a connection to redis. -func OpenClient() (client *redis.Client) { - client = redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - Password: "", - DB: 0, - }) - - return -} - func (p *RedisStore) SetKeyIfNotExists(key, value string) (retval bool) { - return redisSetKeyIfNotExists(p.client, key, value) + return r.RedisSetKeyIfNotExists(p.client, key, value) } func (p *RedisStore) SetKV(key, value string) { - redisSetKeyVal(p.client, key, value) + r.RedisSetKeyVal(p.client, key, value) } func (p *RedisStore) RemoveKV(key, value string) { // TODO(ian): Refactor this so we don't have to delete a value from a key if value != "" || value == "" { - redisRemoveKeysValue(p.client, key, value) + r.RedisRemoveKeysValue(p.client, key, value) } } func (p *RedisStore) KeyExists(key string) (retval bool) { - return redisGetBoolKeyVal(key) + return r.RedisGetBoolKeyVal(key) } func (p *RedisStore) GetKeyVal(key string) []string { - return redisGetKeyVal(key) -} - -// RedisSetIPMember sets a key as a member of an infohash and sets a timeout. -func redisSetIPMember(infoHash, ipPort string) (retval int) { - c := OpenClient() - - keymember := concatenateKeyMember(infoHash, "ip") - - currTime := int64(time.Now().UTC().AddDate(0, 0, 1).Unix()) - - key := fmt.Sprintf("%s:%v", ipPort, currTime) - - if err := c.SAdd(keymember, key).Err(); err != nil { - retval = 0 - panic("Failed to add key") - - } else { - retval = 1 - } - - return -} - -// RedisSetKeyVal Sets a key to the specified value. Used mostly with adding a -// peer into an info_hash -func redisSetKeyVal(c *redis.Client, keymember string, value string) { - // RedisSetKeyVal sets a key:member's value to value. Returns nothing as of - // yet. - currTime := int64(time.Now().UTC().Unix()) - currTime += EXPIRETIME - value = fmt.Sprintf("%v:%v", value, currTime) - - if sz := strings.Split(value, ":"); len(sz) >= 1 { - // If the value being added can be converted to an int, it is a ip:port key - // and we can set an expiration on it. - c.SAdd(keymember, value) - } -} - -// RedisGetKeyVal Lookup a peer in the specified infohash at `key` -func redisGetKeyVal(key string) []string { - c := OpenClient() - - // RedisGetKeyVal retrieves a value from the Redis store by looking up the - // provided key. If the key does not yet exist, we create the key in the KV - // storage or if the value is empty, we add the current requester to the - // list. - keymember := concatenateKeyMember(key, "complete") - - val, err := c.SMembers(keymember).Result() - if err != nil { - // Fail because the key doesn't exist in the KV storage. - createNewTorrentKey(keymember) - } - - return val -} - -// RedisGetAllPeers fetches all peers from the info_hash at `key` -func redisGetAllPeers(key string) []string { - c := OpenClient() - - keymember := concatenateKeyMember(key, "complete") - - val, err := c.SRandMemberN(keymember, 30).Result() - if err != nil { - // Fail because the key doesn't exist in the KV storage. - createNewTorrentKey(keymember) - } - - if len(val) == 30 { - return val - } - - keymember = concatenateKeyMember(key, "incomplete") - - val2, err := c.SRandMemberN(keymember, int64(30-len(val))).Result() - if err != nil { - panic("Failed to get incomplete peers for") - } else { - val = append(val, val2...) - } - - return val -} - -// RedisGetCount counts all of the peers at `info_hash` -func redisGetCount(c *redis.Client, info_hash string, member string) (retval int, err error) { - // A generic function which is used to retrieve either the complete count - // or the incomplete count for a specified `info_hash`. - keymember := concatenateKeyMember(info_hash, member) - - x, err := c.SMembers(keymember).Result() - if err != nil { - // TODO(ian): Add actual error checking here. - err = fmt.Errorf("The info hash %s with member %s doesn't exist", info_hash, member) - } - - retval = len(x) - return -} - -// RedisGetBoolKeyVal Checks if a `key` exists -func redisGetBoolKeyVal(key string) bool { - c := OpenClient() - ret, _ := c.Exists(key).Result() - - return ret -} - -// RedisSetKeyIfNotExists Set a key if it doesn't exist. -func redisSetKeyIfNotExists(c *redis.Client, keymember string, value string) (rv bool) { - rv = redisGetBoolKeyVal(keymember) - if !rv { - redisSetKeyVal(c, keymember, value) - } - return -} - -// RedisRemoveKeysValue Remove a `value` from `key` in the redis kv storage. `key` is typically -// a keymember of info_hash:(in)complete and the value is typically the -// ip:port concatenated. -func redisRemoveKeysValue(c *redis.Client, key string, value string) { - c.SRem(key, value) + return r.RedisGetKeyVal(key) } -// CreateNewTorrentKey creates a new key. By default, it adds a member -// ":ip". I don't think this ought to ever be generalized, as I just want -// Redis to function in one specific way in notorious. -func createNewTorrentKey(key string) { - c := OpenClient() - c.SAdd(key, "complete", "incomplete") - -} - -// concatenateKeyMember concatenates the key and the member delimited by the -// character ":" -func concatenateKeyMember(key string, member string) string { - var buffer bytes.Buffer - - buffer.WriteString(key) - buffer.WriteString(":") - buffer.WriteString(member) - - return buffer.String() +func (p *RedisStore) GetAllPeers(key string) []string { + return r.RedisGetAllPeers(key) } -// createIPPortPair creates a string formatted ("%s:%s", value.ip, -// value.port) looking like so: "127.0.0.1:6886" and returns this value. -func createIPPortPair(ip, port string) string { - return fmt.Sprintf("%v:%v", ip, port) +func (p *RedisStore) SetIPMember(infoHash, ipPort string) (retval int) { + return r.RedisSetIPMember(infoHash, ipPort) } diff --git a/server/server.go b/server/server.go index c30e680..3bb84de 100644 --- a/server/server.go +++ b/server/server.go @@ -6,6 +6,7 @@ import ( "github.com/GrappigPanda/notorious/database" . "github.com/GrappigPanda/notorious/announce" "github.com/GrappigPanda/notorious/server/peerStore" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" "net/http" ) @@ -14,7 +15,7 @@ import ( type applicationContext struct { config config.ConfigStruct trackerLevel int - peerStoreClient *peerStore.PeerStore + peerStoreClient peerStore.PeerStore } type scrapeData struct { @@ -49,14 +50,14 @@ var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compac func (app *applicationContext) worker(data *AnnounceData) []string { if app.peerStoreClient.KeyExists(data.InfoHash) { - x := peerStore.GetKeyVal(data.InfoHash) + x := app.peerStoreClient.GetKeyVal(data.InfoHash) - app.peerStoreClient.RedisSetIPMember(data.InfoHash, fmt.Sprintf("%s:%s", data.IP, data.Port)) + app.peerStoreClient.SetIPMember(data.InfoHash, fmt.Sprintf("%s:%s", data.IP, data.Port)) return x } else { - app.peerStoreClient.SetKV(data.InfoHash) + r.CreateNewTorrentKey(data.InfoHash) } return app.worker(data) @@ -158,12 +159,10 @@ func writeResponse(w http.ResponseWriter, values string) { // RunServer spins up the server and muxes the routes. func RunServer() { - peerStore := new(peerStore.RedisStore) - app := applicationContext{ config: config.LoadConfig(), trackerLevel: RATIOLESS, - peerStoreClient: peerStore.redisStore, + peerStoreClient: new(peerStore.RedisStore), } mux := http.NewServeMux()