extractors/mgtv: Adapt the new API

extractors/bcy.go

@@ -8,7 +8,7 @@ import (
 
 // Bcy download function
 func Bcy(url string) downloader.VideoData {
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	title, urls := parser.GetImages(
 		url, html, "detail_std detail_clickable", func(u string) string {
 			// https://img9.bcyimg.com/drawer/15294/post/1799t/1f5a87801a0711e898b12b640777720f.jpg/w650

extractors/bilibili/bilibili.go

@@ -41,6 +41,7 @@ func genAPI(aid, cid string, bangumi bool, quality string, seasonType string) st
 		utoken = request.Get(
 			fmt.Sprintf("%said=%s&cid=%s", bilibiliTokenAPI, aid, cid),
 			referer,
+			nil,
 		)
 		var t token
 		json.Unmarshal([]byte(utoken), &t)
@@ -120,7 +121,7 @@ func Download(url string) {
 	if strings.Contains(url, "bangumi") {
 		options.Bangumi = true
 	}
-	html := request.Get(url, referer)
+	html := request.Get(url, referer, nil)
 	if !config.Playlist {
 		options.HTML = html
 		data, err := getMultiPageData(html)
@@ -187,7 +188,7 @@ func bilibiliDownload(url string, options bilibiliOptions) downloader.VideoData
 		// reuse html string, but this can't be reused in case of playlist
 		html = options.HTML
 	} else {
-		html = request.Get(url, referer)
+		html = request.Get(url, referer, nil)
 	}
 	if options.Aid != "" && options.Cid != "" {
 		aid = options.Aid
@@ -210,7 +211,7 @@ func bilibiliDownload(url string, options bilibiliOptions) downloader.VideoData
 	var defaultQuality string
 	for _, q := range []string{"15", "32", "64", "80", "112", "74", "116"} {
 		apiURL := genAPI(aid, cid, options.Bangumi, q, seasonType)
-		jsonString := request.Get(apiURL, referer)
+		jsonString := request.Get(apiURL, referer, nil)
 		var data bilibiliData
 		json.Unmarshal([]byte(jsonString), &data)
 

extractors/douyin.go

@@ -23,7 +23,7 @@ type douyinData struct {
 
 // Douyin download function
 func Douyin(url string) downloader.VideoData {
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	vData := utils.MatchOneOf(html, `var data = \[(.*?)\];`)[1]
 	var dataDict douyinData
 	json.Unmarshal([]byte(vData), &dataDict)

extractors/douyu.go

@@ -47,11 +47,11 @@ func Douyu(url string) downloader.VideoData {
 		log.Fatal("暂不支持斗鱼直播")
 	}
 
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	title := utils.MatchOneOf(html, `<title>(.*?)</title>`)[1]
 
 	vid := utils.MatchOneOf(url, `https?://v.douyu.com/show/(\S+)`)[1]
-	dataString := request.Get("http://vmobile.douyu.com/video/getInfo?vid="+vid, url)
+	dataString := request.Get("http://vmobile.douyu.com/video/getInfo?vid="+vid, url, nil)
 	var dataDict douyuData
 	json.Unmarshal([]byte(dataString), &dataDict)
 

extractors/facebook.go

@@ -10,7 +10,7 @@ import (
 
 // Facebook download function
 func Facebook(url string) downloader.VideoData {
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	title := utils.MatchOneOf(html, `<title id="pageTitle">(.+)</title>`)[1]
 
 	format := map[string]downloader.FormatData{}

extractors/instagram.go

@@ -45,7 +45,7 @@ type instagram struct {
 
 // Instagram download function
 func Instagram(url string) downloader.VideoData {
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	// get the title
 	doc := parser.GetDoc(html)
 	title := parser.Title(doc)

extractors/iqiyi.go

@@ -53,6 +53,7 @@ func getIqiyiData(tvid, vid string) iqiyi {
 			tvid, vid, t, sc, src,
 		),
 		iqiyiReferer,
+		nil,
 	)
 	var data iqiyi
 	json.Unmarshal([]byte(info[len("var tvInfoJs="):]), &data)
@@ -61,7 +62,7 @@ func getIqiyiData(tvid, vid string) iqiyi {
 
 // Iqiyi download function
 func Iqiyi(url string) downloader.VideoData {
-	html := request.Get(url, iqiyiReferer)
+	html := request.Get(url, iqiyiReferer, nil)
 	tvid := utils.MatchOneOf(
 		url,
 		`#curid=(.+)_`,

extractors/mgtv.go

@@ -1,9 +1,13 @@
 package extractors
 
 import (
+	"encoding/base64"
 	"encoding/json"
+	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/iawia002/annie/downloader"
 	"github.com/iawia002/annie/request"
@@ -40,12 +44,21 @@ type mgtvURLInfo struct {
 	Size int64
 }
 
+type mgtvPm2Data struct {
+	Data struct {
+		Atc struct {
+			Pm2 string `json:"pm2"`
+		} `json:"atc"`
+		Info mgtvVideoInfo `json:"info"`
+	} `json:"data"`
+}
+
 func mgtvM3u8(url string) ([]mgtvURLInfo, int64) {
 	var data []mgtvURLInfo
 	var temp mgtvURLInfo
 	var size, totalSize int64
 	urls := utils.M3u8URLs(url)
-	m3u8String := request.Get(url, url)
+	m3u8String := request.Get(url, url, nil)
 	sizes := utils.MatchAll(m3u8String, `#EXT-MGTV-File-SIZE:(\d+)`)
 	// sizes: [[#EXT-MGTV-File-SIZE:1893724, 1893724]]
 	for index, u := range urls {
@@ -60,9 +73,21 @@ func mgtvM3u8(url string) ([]mgtvURLInfo, int64) {
 	return data, totalSize
 }
 
+func encodeTk2(str string) string {
+	encodeString := base64.StdEncoding.EncodeToString([]byte(str))
+	r1 := regexp.MustCompile(`/\+/g`)
+	r2 := regexp.MustCompile(`/\//g`)
+	r3 := regexp.MustCompile(`/=/g`)
+	r1.ReplaceAllString(encodeString, "_")
+	r2.ReplaceAllString(encodeString, "~")
+	r3.ReplaceAllString(encodeString, "-")
+	encodeString = utils.Reverse(encodeString)
+	return encodeString
+}
+
 // Mgtv download function
 func Mgtv(url string) downloader.VideoData {
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	vid := utils.MatchOneOf(
 		url,
 		`https?://www.mgtv.com/(?:b|l)/\d+/(\d+).html`,
@@ -71,13 +96,38 @@ func Mgtv(url string) downloader.VideoData {
 	if vid == nil {
 		vid = utils.MatchOneOf(html, `vid: (\d+),`)
 	}
+	// API extract from https://js.mgtv.com/imgotv-miniv6/global/page/play-tv.js
+	// getSource and getPlayInfo function
+	// Chrome Network JS panel
+	headers := map[string]string{
+		"Cookie": "PM_CHKID=1",
+	}
+	clit := fmt.Sprintf("clit=%d", time.Now().Unix()/1000)
+	pm2DataString := request.Get(
+		fmt.Sprintf(
+			"https://pcweb.api.mgtv.com/player/video?video_id=%s&tk2=%s",
+			vid[1],
+			encodeTk2(fmt.Sprintf(
+				"did=f11dee65-4e0d-4d25-bfce-719ad9dc991d|pno=1030|ver=5.5.1|%s", clit,
+			)),
+		),
+		url,
+		headers,
+	)
+	var pm2 mgtvPm2Data
+	json.Unmarshal([]byte(pm2DataString), &pm2)
 	dataString := request.Get(
-		"https://pcweb.api.mgtv.com/player/video?video_id="+vid[1], url,
+		fmt.Sprintf(
+			"https://pcweb.api.mgtv.com/player/getSource?video_id=%s&tk2=%s&pm2=%s",
+			vid[1], encodeTk2(clit), pm2.Data.Atc.Pm2,
+		),
+		url,
+		headers,
 	)
 	var mgtvData mgtv
 	json.Unmarshal([]byte(dataString), &mgtvData)
 	title := strings.TrimSpace(
-		mgtvData.Data.Info.Title + " " + mgtvData.Data.Info.Desc,
+		pm2.Data.Info.Title + " " + pm2.Data.Info.Desc,
 	)
 	streams := mgtvData.Data.Stream
 	var addr mgtvVideoAddr
@@ -89,7 +139,8 @@ func Mgtv(url string) downloader.VideoData {
 		// real download address
 		addr = mgtvVideoAddr{}
 		json.Unmarshal(
-			[]byte(request.Get(mgtvData.Data.StreamDomain[0]+stream.URL, url)), &addr,
+			[]byte(request.Get(mgtvData.Data.StreamDomain[0]+stream.URL, url, headers)),
+			&addr,
 		)
 		m3u8URLs, totalSize := mgtvM3u8(addr.Info)
 		urls := []downloader.URLData{}

extractors/miaopai.go

@@ -9,7 +9,7 @@ import (
 
 // Miaopai download function
 func Miaopai(url string) downloader.VideoData {
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	doc := parser.GetDoc(html)
 	title := parser.Title(doc)
 

extractors/pixivision.go

@@ -8,7 +8,7 @@ import (
 
 // Pixivision download function
 func Pixivision(url string) downloader.VideoData {
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	title, urls := parser.GetImages(url, html, "am__work__illust  ", nil)
 	format := map[string]downloader.FormatData{
 		"default": {

extractors/tumblr.go

@@ -78,7 +78,7 @@ func tumblrVideoDownload(url, html, title string) downloader.VideoData {
 	if !strings.Contains(videoURL, "tumblr.com/video") {
 		log.Fatal("annie doesn't support this URL right now")
 	}
-	videoHTML := request.Get(videoURL, url)
+	videoHTML := request.Get(videoURL, url, nil)
 	realURL := utils.MatchOneOf(videoHTML, `source src="(.+?)"`)[1]
 	urlData, size := genURLData(realURL, url)
 	format := map[string]downloader.FormatData{
@@ -99,7 +99,7 @@ func tumblrVideoDownload(url, html, title string) downloader.VideoData {
 
 // Tumblr download function
 func Tumblr(url string) downloader.VideoData {
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	// get the title
 	doc := parser.GetDoc(html)
 	title := parser.Title(doc)

extractors/twitter.go

@@ -36,7 +36,7 @@ func getVideoURI(uri string) twitter {
 	// extract tweet id from url
 	tweetID := utils.MatchOneOf(uri, `(status|statuses)/(\d+)`)[2]
 	webPlayerURL := prefix + tweetID
-	h := request.Get(webPlayerURL, uri)
+	h := request.Get(webPlayerURL, uri, nil)
 	// get dataconfig attribute
 	jsonString := html.UnescapeString(utils.MatchOneOf(h, `data-config="({.+})`)[1])
 	var twitterData twitter

extractors/vimeo.go

@@ -40,10 +40,10 @@ func Vimeo(url string) downloader.VideoData {
 	var html string
 	var vid string
 	if strings.Contains(url, "player.vimeo.com") {
-		html = request.Get(url, url)
+		html = request.Get(url, url, nil)
 	} else {
 		vid = utils.MatchOneOf(url, `vimeo\.com/(\d+)`)[1]
-		html = request.Get("https://player.vimeo.com/video/"+vid, url)
+		html = request.Get("https://player.vimeo.com/video/"+vid, url, nil)
 	}
 	jsonString := utils.MatchOneOf(html, `{var a=({.+?});`)[1]
 	var vimeoData vimeo

extractors/weibo.go

@@ -14,7 +14,7 @@ func Weibo(url string) downloader.VideoData {
 		statusID := utils.MatchOneOf(url, `weibo\.com/tv/v/([^\?/]+)`)[1]
 		url = "https://m.weibo.cn/status/" + statusID
 	}
-	html := request.Get(url, url)
+	html := request.Get(url, url, nil)
 	title := utils.MatchOneOf(html, `"content2": "(.+?)",`)[1]
 	realURL := utils.MatchOneOf(
 		html, `"stream_url_hd": "(.+?)"`, `"stream_url": "(.+?)"`,

extractors/youku.go

@@ -75,7 +75,7 @@ func youkuUps(vid string) youkuData {
 			"https://ups.youku.com/ups/get.json?vid=%s&ccode=%s&client_ip=192.168.1.1&client_ts=%d&utid=%s&ckey=%s",
 			vid, ccode, time.Now().Unix()/1000, netURL.QueryEscape(utid), netURL.QueryEscape(ckey),
 		)
-		html = request.Get(url, youkuReferer)
+		html = request.Get(url, youkuReferer, nil)
 		// data must be emptied before reassignment, otherwise it will contain the previous value(the 'error' data)
 		data = youkuData{}
 		json.Unmarshal([]byte(html), &data)

extractors/youtube/youtube.go

@@ -37,7 +37,7 @@ func getSig(sig, js string) string {
 	url := fmt.Sprintf("https://www.youtube.com%s", js)
 	tokens, ok := tokensCache[url]
 	if !ok {
-		tokens = getSigTokens(request.Get(url, referer))
+		tokens = getSigTokens(request.Get(url, referer, nil))
 		tokensCache[url] = tokens
 	}
 	return decipherTokens(tokens, sig)
@@ -70,7 +70,7 @@ func Download(uri string) {
 	if listID == "" {
 		log.Fatal("Can't get list ID from URL")
 	}
-	html := request.Get("https://www.youtube.com/playlist?list="+listID, referer)
+	html := request.Get("https://www.youtube.com/playlist?list="+listID, referer, nil)
 	// "videoId":"OQxX8zgyzuM","thumbnail"
 	videoIDs := utils.MatchAll(html, `"videoId":"([^,]+?)","thumbnail"`)
 	for _, videoID := range videoIDs {
@@ -96,7 +96,7 @@ func youtubeDownload(uri string) downloader.VideoData {
 		"https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999",
 		vid[1],
 	)
-	html := request.Get(videoURL, referer)
+	html := request.Get(videoURL, referer, nil)
 	ytplayer := utils.MatchOneOf(html, `;ytplayer\.config\s*=\s*({.+?});`)[1]
 	var youtube youtubeData
 	json.Unmarshal([]byte(ytplayer), &youtube)

request/request.go

@@ -64,6 +64,9 @@ func Request(
 		req.Header.Set(k, v)
 	}
 	req.Header.Set("Referer", url)
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
 	if config.Cookie != "" {
 		var cookie string
 		if _, fileErr := os.Stat(config.Cookie); fileErr == nil {
@@ -76,9 +79,6 @@ func Request(
 		}
 		req.Header.Set("Cookie", cookie)
 	}
-	for k, v := range headers {
-		req.Header.Set(k, v)
-	}
 	if config.Refer != "" {
 		req.Header.Set("Referer", config.Refer)
 	}
@@ -118,8 +118,10 @@ func Request(
 }
 
 // Get get request
-func Get(url, refer string) string {
-	headers := map[string]string{}
+func Get(url, refer string, headers map[string]string) string {
+	if headers == nil {
+		headers = map[string]string{}
+	}
 	if refer != "" {
 		headers["Referer"] = refer
 	}

utils/utils.go

@@ -145,7 +145,7 @@ func Md5(text string) string {
 
 // M3u8URLs get all urls from m3u8 url
 func M3u8URLs(uri string) []string {
-	html := request.Get(uri, "")
+	html := request.Get(uri, "", nil)
 	lines := strings.Split(html, "\n")
 	var urls []string
 	for _, line := range lines {
@@ -189,3 +189,12 @@ func ShouldExtract(format, bestQuality string) bool {
 	}
 	return true
 }
+
+// Reverse Reverse a string
+func Reverse(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}