-
Notifications
You must be signed in to change notification settings - Fork 0
/
gsearch.sh
executable file
·93 lines (82 loc) · 2.56 KB
/
gsearch.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
#!/usr/bin/bash
outputdir="output"
if [[ ! -e $outputdir ]]; then
mkdir $outputdir
fi
# https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command
rawurlencode() {
local string="${1}"
local strlen=${#string}
local encoded=""
local pos c o
for (( pos=0 ; pos<strlen ; pos++ )); do
c=${string:$pos:1}
case "$c" in
[-_.~a-zA-Z0-9] ) o="${c}" ;;
" " ) o="+" ;;
* ) printf -v o '%%%02x' "'$c"
esac
encoded+="${o}"
done
REPLY="${encoded}"
}
performquery () {
query="$1"
today=$(date +%Y%m%d)
output_today="$outputdir/$today"
if [[ ! -e $output_today ]]; then
mkdir -p $output_today
fi
output="$output_today/$query-$epoch.html"
echo "Searching $query > $output"
http_code=$(curl "https://www.google.com/search?q=$query" -H 'referer: https://www.google.com/' \
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36' \
--compressed --output $output --silent --max-time 10 -w '%{http_code}')
exit_status="$?"
if [[ $http_code == "200" ]]; then
./gparse.py $output
else
echo "curl error:$http_code:$exit_status:$query:$output"
fi
sleep 1
}
performbingquery () {
query="$1"
today=$(date +%Y%m%d)
output_today="$outputdir/$today"
if [[ ! -e $output_today ]]; then
mkdir -p $output_today
fi
output="$output_today/bing-$query-$epoch.html"
echo "Searching $query > $output"
# https://www.bing.com/search?q=looksrare&qs=n&form=QBRE&sp=-1&lq=0&pq=looksrare&sc=15-9&sk=&cvid=F4468886830C498B9CB2B17F18956F22&ghsh=0&ghacc=0&ghpl=
http_code=$(curl "https://www.bing.com/search?q=$query" -H 'referer: https://www.bing.com/' \
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36' \
--compressed --output $output --silent --max-time 10 -w '%{http_code}')
exit_status="$?"
if [[ $http_code == "200" ]]; then
./bparse.py $output
else
echo "curl error:$http_code:$exit_status:$query:$output"
fi
sleep 1
}
for (( ; ; ))
do
date
epoch=`date +%s`
IFS=$'\n'
for t in `cat queries.txt`; do
rawurlencode "$t"
performquery "$REPLY"
performbingquery "$REPLY"
done
if [[ -e "queries-custom.txt" ]]; then
for t in `cat queries-custom.txt`; do
rawurlencode "$t"
performquery "$REPLY"
performbingquery "$REPLY"
done
fi
sleep 60
done