-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
71 lines (53 loc) · 1.66 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
package main
import (
"fmt"
"time"
)
// Response ...
type Response struct {
Title string
Author string
Claps string
}
func main() {
urls := []string{
"https://medium.freecodecamp.org/how-to-columnize-your-code-to-improve-readability-f1364e2e77ba",
"https://medium.freecodecamp.org/how-to-think-like-a-programmer-lessons-in-problem-solving-d1d8bf1de7d2",
// "https://medium.freecodecamp.org/code-comments-the-good-the-bad-and-the-ugly-be9cc65fbf83",
"https://uxdesign.cc/learning-to-code-or-sort-of-will-make-you-a-better-product-designer-e76165bdfc2d",
}
ch := make(chan Response)
ini := time.Now()
go scrapList(urls, ch)
for resp := range ch {
fmt.Println(resp)
}
fmt.Println("(Took ", time.Since(ini).Seconds(), "secs)")
}
func scrapList(urls []string, in chan Response) {
defer close(in)
var outs = []chan Response{}
for i, url := range urls {
outs = append(outs, make(chan Response))
go processURL(url, outs[i])
}
for i := range outs {
for response := range outs[i] {
in <- response
}
}
}
func processURL(url string, in chan Response) {
defer close(in)
var resp Response
htmlParsed, _ := GetHTMLParsed(url)
a := GetFirstElementByClass(htmlParsed, "a", "link link--primary u-accentColor--hoverTextNormal")
resp.Author = GetFirstTextNode(a).Data
div := GetFirstElementByClass(htmlParsed, "div", "section-content")
h1 := GetFirstElementByClass(div, "h1", "graf--title")
resp.Title = GetFirstTextNode(h1).Data
footer := GetFirstElementByClass(htmlParsed, "footer", "u-paddingTop10")
buttonLikes := GetFirstElementByClass(footer, "button", "js-multirecommendCountButton")
resp.Claps = GetFirstTextNode(buttonLikes).Data
in <- resp
}