main: refactor the data structures

downloader/downloader.go

@@ -19,20 +19,20 @@ import (
 
 // URLData data struct of single URL
 type URLData struct {
-	URL  string
-	Size int64
-	Ext  string
+	URL  string `json:"url"`
+	Size int64  `json:"size"`
+	Ext  string `json:"ext"`
 }
 
 // FormatData data struct of every format
 type FormatData struct {
 	// [URLData: {URL, Size, Ext}, ...]
 	// Some video files have multiple fragments
 	// and support for downloading multiple image files at once
-	URLs    []URLData
-	Quality string
+	URLs    []URLData `json:"urls"`
+	Quality string    `json:"quality"`
 	// total size of all urls
-	Size int64
+	Size int64 `json:"size"`
 	name string
 }
 
@@ -44,14 +44,17 @@ func (f formats) Less(i, j int) bool { return f[i].Size > f[j].Size }
 
 // VideoData data struct of video info
 type VideoData struct {
-	Site  string
-	Title string
-	Type  string
+	Site  string `json:"site"`
+	Title string `json:"title"`
+	Type  string `json:"type"`
 	// each format has it's own URLs and Quality
-	Formats       map[string]FormatData
+	Formats       map[string]FormatData `json:"formats"`
 	sortedFormats formats
 }
 
+// EmptyData empty VideoData list
+var EmptyData = make([]VideoData, 0)
+
 func progressBar(size int64) *pb.ProgressBar {
 	bar := pb.New64(size).SetUnits(pb.U_BYTES).SetRefreshRate(time.Millisecond * 10)
 	bar.ShowSpeed = true

downloader/downloader_test.go

@@ -65,26 +65,6 @@ func TestDownload(t *testing.T) {
 				},
 			},
 		},
-		{
-			name: "google video test",
-			data: VideoData{
-				Site:  "google",
-				Title: "google video test",
-				Type:  "video",
-				Formats: map[string]FormatData{
-					"default": FormatData{
-						URLs: []URLData{
-							{
-								URL:  "https://r3---sn-nx5e6nez.googlevideo.com/videoplayback?mn=sn-nx5e6nez%2Csn-a5mekne7&mm=31%2C29&key=yt6&ip=149.28.131.72&dur=320.280&keepalive=yes&pl=26&source=youtube&ms=au%2Crdu&ei=48mdW-eNL8-zz7sP_a6JqA0&id=o-AHWbYcCuEnkDYwjCFqaBJVCR--bBfLIq-LuGUcShAbHY&mv=m&mt=1537067391&sparams=aitags%2Cclen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&gir=yes&ipbits=0&requiressl=yes&clen=7513224&mime=video%2Fmp4&expire=1537089092&c=WEB&fvip=4&itag=133&lmt=1513573757999118&initcwndbps=345000&aitags=133%2C134%2C135%2C136%2C160%2C242%2C243%2C244%2C247%2C278&signature=A9AF6B4F23F0967B8F2A351F2D23E61D35100CB7.CB90C59B0B6E4C126EBDA3B2C3442869799FF87F&ratebypass=yes",
-								Size: 7513224,
-								Ext:  "mp4",
-							},
-						},
-						Size: 7513224,
-					},
-				},
-			},
-		},
 		{
 			name: "image test",
 			data: VideoData{

extractors/bcy.go → extractors/bcy/bcy.go

@@ -1,17 +1,17 @@
-package extractors
+package bcy
 
 import (
 	"github.com/iawia002/annie/downloader"
 	"github.com/iawia002/annie/parser"
 	"github.com/iawia002/annie/request"
 )
 
-// Bcy download function
-func Bcy(url string) (downloader.VideoData, error) {
+// Download main download function
+func Download(url string) ([]downloader.VideoData, error) {
 	var err error
 	html, err := request.Get(url, url, nil)
 	if err != nil {
-		return downloader.VideoData{}, err
+		return downloader.EmptyData, err
 	}
 	title, urls, err := parser.GetImages(
 		url, html, "detail_std detail_clickable", func(u string) string {
@@ -20,23 +20,20 @@ func Bcy(url string) (downloader.VideoData, error) {
 		},
 	)
 	if err != nil {
-		return downloader.VideoData{}, err
+		return downloader.EmptyData, err
 	}
 	format := map[string]downloader.FormatData{
 		"default": {
 			URLs: urls,
 			Size: 0,
 		},
 	}
-	extractedData := downloader.VideoData{
-		Site:    "半次元 bcy.net",
-		Title:   title,
-		Type:    "image",
-		Formats: format,
-	}
-	err = extractedData.Download(url)
-	if err != nil {
-		return downloader.VideoData{}, err
-	}
-	return extractedData, nil
+	return []downloader.VideoData{
+		{
+			Site:    "半次元 bcy.net",
+			Title:   title,
+			Type:    "image",
+			Formats: format,
+		},
+	}, nil
 }

extractors/bcy_test.go → extractors/bcy/bcy_test.go

@@ -1,4 +1,4 @@
-package extractors
+package bcy
 
 import (
 	"testing"
@@ -7,7 +7,7 @@ import (
 	"github.com/iawia002/annie/test"
 )
 
-func TestBcy(t *testing.T) {
+func TestDownload(t *testing.T) {
 	config.InfoOnly = true
 	tests := []struct {
 		name string
@@ -30,7 +30,7 @@ func TestBcy(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			data, _ := Bcy(tt.args.URL)
+			data, _ := Download(tt.args.URL)
 			test.Check(t, tt.args, data)
 		})
 	}

extractors/bilibili/bilibili.go

@@ -118,19 +118,19 @@ func getMultiPageData(html string) (multiPage, error) {
 }
 
 // Download bilibili main download function
-func Download(url string) error {
+func Download(url string) ([]downloader.VideoData, error) {
 	var options bilibiliOptions
 	var err error
 	if strings.Contains(url, "bangumi") {
 		options.Bangumi = true
 	}
 	html, err := request.Get(url, referer, nil)
 	if err != nil {
-		return err
+		return downloader.EmptyData, err
 	}
 	if !config.Playlist {
 		options.HTML = html
-		data, err := getMultiPageData(html)
+		pageData, err := getMultiPageData(html)
 		if err == nil && !options.Bangumi {
 			// handle URL that has a playlist, mainly for unified titles
 			// <h1> tag does not include subtitles
@@ -145,8 +145,8 @@ func Download(url string) error {
 				p, _ = strconv.Atoi(pageString[1])
 			}
 			options.P = p
-			page := data.VideoData.Pages[p-1]
-			options.Aid = data.Aid
+			page := pageData.VideoData.Pages[p-1]
+			options.Aid = pageData.Aid
 			options.Cid = strconv.Itoa(page.Cid)
 			// "part":"" or "part":"Untitled"
 			if page.Part == "Untitled" {
@@ -155,53 +155,61 @@ func Download(url string) error {
 				options.Subtitle = page.Part
 			}
 		}
-		_, err = bilibiliDownload(url, options)
+		data, err := bilibiliDownload(url, options)
 		if err != nil {
-			return err
+			return downloader.EmptyData, err
 		}
-		return nil
+		return []downloader.VideoData{data}, nil
 	}
+	// for Bangumi playlist
 	if options.Bangumi {
 		dataString := utils.MatchOneOf(html, `window.__INITIAL_STATE__=(.+?);\(function`)[1]
 		var data bangumiData
 		json.Unmarshal([]byte(dataString), &data)
 		needDownloadItems := utils.NeedDownloadList(len(data.EpList))
+		extractedData := make([]downloader.VideoData, len(needDownloadItems))
 		for index, u := range data.EpList {
 			if !utils.ItemInSlice(index+1, needDownloadItems) {
 				continue
 			}
-			bilibiliDownload(
+			videoData, err := bilibiliDownload(
 				fmt.Sprintf("https://www.bilibili.com/bangumi/play/ep%d", u.EpID), options,
 			)
+			if err == nil {
+				// if err is not nil, the data is empty struct
+				extractedData[index] = videoData
+			}
 		}
-	} else {
-		data, err := getMultiPageData(html)
+		return extractedData, nil
+	}
+	// for normal video playlist
+	data, err := getMultiPageData(html)
+	if err != nil {
+		// this page has no playlist
+		options.HTML = html
+		videoData, err := bilibiliDownload(url, options)
 		if err != nil {
-			// this page has no playlist
-			options.HTML = html
-			_, err = bilibiliDownload(url, options)
-			if err != nil {
-				return err
-			}
-			return nil
+			return downloader.EmptyData, err
 		}
-		// https://www.bilibili.com/video/av20827366/?p=1
-		needDownloadItems := utils.NeedDownloadList(len(data.VideoData.Pages))
-		for index, u := range data.VideoData.Pages {
-			if !utils.ItemInSlice(index+1, needDownloadItems) {
-				continue
-			}
-			options.Aid = data.Aid
-			options.Cid = strconv.Itoa(u.Cid)
-			options.Subtitle = u.Part
-			options.P = u.Page
-			_, err := bilibiliDownload(url, options)
-			if err != nil {
-				return err
-			}
+		return []downloader.VideoData{videoData}, nil
+	}
+	// https://www.bilibili.com/video/av20827366/?p=1
+	needDownloadItems := utils.NeedDownloadList(len(data.VideoData.Pages))
+	extractedData := make([]downloader.VideoData, len(needDownloadItems))
+	for index, u := range data.VideoData.Pages {
+		if !utils.ItemInSlice(index+1, needDownloadItems) {
+			continue
+		}
+		options.Aid = data.Aid
+		options.Cid = strconv.Itoa(u.Cid)
+		options.Subtitle = u.Part
+		options.P = u.Page
+		videoData, err := bilibiliDownload(url, options)
+		if err == nil {
+			extractedData[index] = videoData
 		}
 	}
-	return nil
+	return extractedData, nil
 }
 
 func bilibiliDownload(url string, options bilibiliOptions) (downloader.VideoData, error) {
@@ -289,22 +297,16 @@ func bilibiliDownload(url string, options bilibiliOptions) (downloader.VideoData
 		title = tempTitle
 	}
 	title = utils.FileName(title)
-	extractedData := downloader.VideoData{
+
+	downloader.Caption(
+		fmt.Sprintf("https://comment.bilibili.com/%s.xml", cid),
+		url, title, "xml",
+	)
+
+	return downloader.VideoData{
 		Site:    "哔哩哔哩 bilibili.com",
 		Title:   title,
 		Type:    "video",
 		Formats: format,
-	}
-	err = extractedData.Download(url)
-	if err != nil {
-		return downloader.VideoData{}, err
-	}
-	err = downloader.Caption(
-		fmt.Sprintf("https://comment.bilibili.com/%s.xml", cid),
-		url, title, "xml",
-	)
-	if err != nil {
-		return downloader.VideoData{}, err
-	}
-	return extractedData, nil
+	}, nil
 }

extractors/bilibili/bilibili_test.go

@@ -19,7 +19,6 @@ func TestBilibili(t *testing.T) {
 			name: "normal test",
 			args: test.Args{
 				URL:     "https://www.bilibili.com/video/av20203945/",
-				Bangumi: false,
 				Title:   "【2018拜年祭单品】相遇day by day",
 				Quality: "高清 1080P",
 			},
@@ -38,7 +37,6 @@ func TestBilibili(t *testing.T) {
 			name: "bangumi test",
 			args: test.Args{
 				URL:     "https://www.bilibili.com/bangumi/play/ep167000",
-				Bangumi: true,
 				Title:   "狐妖小红娘：第70话 苏苏智商上线",
 				Quality: "高清 1080P",
 			},
@@ -47,7 +45,6 @@ func TestBilibili(t *testing.T) {
 			name: "bangumi playlist test",
 			args: test.Args{
 				URL:     "https://www.bilibili.com/bangumi/play/ss5050",
-				Bangumi: true,
 				Title:   "一人之下：第1话 异人刀兵起，道炁携阴阳",
 				Quality: "高清 1080P",
 			},
@@ -57,7 +54,6 @@ func TestBilibili(t *testing.T) {
 			name: "playlist test",
 			args: test.Args{
 				URL:     "https://www.bilibili.com/video/av16907446/",
-				Bangumi: false,
 				Title:   "\"不要相信歌词，他们为了押韵什么都干得出来\"",
 				Quality: "高清 720P",
 			},
@@ -66,29 +62,20 @@ func TestBilibili(t *testing.T) {
 		{
 			name: "bangumi test",
 			args: test.Args{
-				URL:     "https://www.bilibili.com/bangumi/play/ss12044",
-				Bangumi: true,
-				Title:   "你的名字。",
+				URL:   "https://www.bilibili.com/bangumi/play/ss12044",
+				Title: "你的名字。",
 			},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			var data downloader.VideoData
-			options := bilibiliOptions{
-				Bangumi: tt.args.Bangumi,
-			}
+			var data []downloader.VideoData
 			if tt.playlist {
-				// playlist mode
 				config.Playlist = true
 				Download(tt.args.URL)
-				// single mode
-				config.Playlist = false
-				Download(tt.args.URL)
-				data, _ = bilibiliDownload(tt.args.URL, options)
 			} else {
 				config.Playlist = false
-				data, _ = bilibiliDownload(tt.args.URL, options)
+				data, _ = Download(tt.args.URL)
 			}
 			test.Check(t, tt.args, data)
 		})