-
Notifications
You must be signed in to change notification settings - Fork 0
/
iterator.go
102 lines (86 loc) · 2.38 KB
/
iterator.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
package gocraigslist
import (
"context"
"fmt"
"strconv"
"strings"
"time"
)
// Iterator is an iterator used to retrieve multiple pages of listings
type Iterator interface {
Next(context.Context, time.Time) (*Result, error)
}
// Result is the structural representation of Result
type Result struct {
Client *Client
Done bool
Listings []Listing
Timezone string
TotalCount int
CurrentPage int
SearchURL string // the original search url without pagination
Err error
}
func newResult(c *Client, url string, totalCount int, listings []Listing, timezone string) *Result {
index := strings.Index(url, "?")
if index < 0 {
url = url + "?"
}
r := Result{
Client: c,
Done: false,
Listings: listings,
TotalCount: totalCount,
CurrentPage: 0,
SearchURL: url,
Timezone: timezone,
}
if len(listings) >= totalCount {
r.Done = true
}
if len(listings) < 120 {
r.Done = true
}
return &r
}
// Next page of listings. This will call the librarys fn GetListings.
// Note: Because postings could be happening as this is fetching results, there is
// the possibility of some duplicates coming through.
// Example in seconds:
// time 0: 1st page is fetched
// time 1: 1st page of listings is being processed, new listing is posted
// time 2: 2nd page is fetched
// the second page would contain the last listing of the previous page.
func (r *Result) Next(ctx context.Context, date time.Time) (*Result, error) {
r.CurrentPage++
nextPageStart := r.CurrentPage * 120
nextPageURL := r.SearchURL + page + strconv.Itoa(nextPageStart)
resp, err := r.Client.Request.fetch(ctx, nextPageURL)
if err != nil {
resp.Body.Close()
r.Done = true
r.Listings = []Listing{}
return r, fmt.Errorf("error fetching from url: %v", err)
}
var listings []Listing
if date == nilTime {
listings, _, err = parseSearchResults(resp.Body)
} else {
listings, _, err = parseSearchResultsAfter(resp.Body, date)
}
if err != nil {
r.Done = true
return r, err
}
r.Listings = listings
// This is required in the event a date is passed in. A search with a date
// might have a high total count but none that after posted after said date.
// in this event, it should stop as soon as there are no listings found.
if len(listings) == 0 {
r.Done = true
}
if (nextPageStart + len(r.Listings)) >= r.TotalCount {
r.Done = true
}
return r, nil
}