From 3afdd718eed5dbed3dd96eceb5b0408d4c87d3aa Mon Sep 17 00:00:00 2001 From: Xonshiz Date: Thu, 13 Apr 2017 09:24:07 +0530 Subject: [PATCH] New Feature + Fixes Fix for #6 and #3 Read the [Changelog](https://github.com/Xonshiz/anime-dl/blob/master/Changelog.md) for more information and check the [README](https://github.com/Xonshiz/anime-dl#windows-binary) to know how to use this. --- Changelog.md | 4 +- ReadMe.md | 13 ++- anime_dl/AnimeDL.py | 4 +- anime_dl/__main__.py | 10 ++- anime_dl/sites/crunchyroll.py | 153 +++++++++++++++++++++++++++------- anime_dl/version.py | 2 +- docs/Changelog.md | 4 +- docs/index.md | 13 ++- 8 files changed, 154 insertions(+), 49 deletions(-) diff --git a/Changelog.md b/Changelog.md index 9095ab1..ed9c9db 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,4 +5,6 @@ - Fix for #2 [2017.03.06] - ReadMe updated for Python Script execution [2017.03.06] - Support for Whole Show Downloading for Crunchyroll [2017.03.06] -- Selection of language for the Crunchyroll Show [2017.03.06] \ No newline at end of file +- Selection of language for the Crunchyroll Show [2017.03.06] +- Downloading only subtitles (skip video downloads) [2017.04.13] +- Fix for [6](https://github.com/Xonshiz/anime-dl/issues/6) and Fix for [3](https://github.com/Xonshiz/anime-dl/issues/3) [2017.04.13] \ No newline at end of file diff --git a/ReadMe.md b/ReadMe.md index cc40a60..a0a4c25 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -31,13 +31,13 @@ Anime-dl is a Command-line program to download anime from CrunchyRoll and Funima You can check the list of supported websites [**`HERE`**](https://github.com/Xonshiz/anime-dl/blob/master/Supported_Sites.md). ## Dependencies Installation -This script can run on multiple Operating Systems. So, if you're using the `python` script instead of the `windows binary` of this script, then you'll need to get things ready first. Since this script doesn't rely on a lot of external dependencies, you just need to grab a few things, same for all operating systems. +This script can run on multiple Operating Systems. But, the script depends on some external binaries or libs. We need `FFmpeg` and `Node.js` in our paths. 1.) Make sure you have Python installed and is present in your system's path. -2.) Grab [FFmpeg from this link.](https://ffmpeg.org/download.html) +2.) Grab [FFmpeg from this link](https://ffmpeg.org/download.html) and [Node.js from this link](https://nodejs.org/en/download/). -3.) Install FFmpeg and place it in the directory of this script, or put FFmpeg in your system's path. +3.) Install FFmpeg and Node.js and place it in the directory of this script, or put them in your system's path. 4.) Browse to the directory of this script and open command prompt/shell in that directory and run this command : @@ -49,7 +49,7 @@ python pip install -r requirements.txt After installing and setting up all the dependencies in your Operating System, you're good to go and use this script. The instructions for all the OS would remain same. Download [`THIS REPOSITORY`](https://github.com/Xonshiz/anime-dl/archive/master.zip) and put it somewhere in your system. Move over the `anime_dl` folder. -**Windows users**, it's better to not place it places where it requires administrator privileges. Good example of such place would be `C:\Windows`. This goes for both, the Python script and the windows binary file (.exe). +**Windows users**, it's better to not place it places where it requires administrator privileges. Good example of places to avoid would be `C:\Windows` etc.. This goes for both, the Python script and the windows binary file (.exe). **Linux/Debian** users make sure that this script is executable.Just run this command, if you run into problem(s) : @@ -59,8 +59,6 @@ The instructions for all the OS would remain same. Download [`THIS REPOSITORY`]( and then, execute with this : -`./anime-dl.py` - `./__main__.py` ## Python Support @@ -84,6 +82,7 @@ Currently, the script supports these arguments : -u,--username Indicates username for a website. [REQUIRED] -p,--password Indicates password for a website. [REQUIRED] -r,--resolution Indicates the desired resolution. (default = 720p) +--skip Skip video downloads (Will only download subtitles) -l,--language Selects the language for the show. (default = Japanese) [Langs = english, dub, sub, Japanese, eng] ``` @@ -141,7 +140,7 @@ If your're planning to open an issue for the script or ask for a new feature or ### Reporting Issues PLEASE RUN THIS SCRIPT IN A COMMAND LINE (as mentioned in the Usage section) AND DON'T SAY THAT `THE SCRIPT CLOSED TOO QUICK, I COULDN'T SEE`. -If you're here to make suggestions, please follow the basic syntax to post a request : +If you're here to report an issue, please follow the basic syntax to post a request : **Subject** : Error That You Get. diff --git a/anime_dl/AnimeDL.py b/anime_dl/AnimeDL.py index 3c8ebab..b4d7562 100644 --- a/anime_dl/AnimeDL.py +++ b/anime_dl/AnimeDL.py @@ -13,7 +13,7 @@ class AnimeDL(object): - def __init__(self, url, username, password, resolution, language): + def __init__(self, url, username, password, resolution, language, skipper): website = str(self.honcho(url=url[0])) @@ -24,7 +24,7 @@ def __init__(self, url, username, password, resolution, language): else: sites.crunchyroll.CrunchyRoll( - url=url[0], password=password, username=username, resolution=resolution, language=language) + url=url[0], password=password, username=username, resolution=resolution, language=language, skipper=skipper) def honcho(self, url): # print("Got url : %s" % url) diff --git a/anime_dl/__main__.py b/anime_dl/__main__.py index 960be89..8601a09 100644 --- a/anime_dl/__main__.py +++ b/anime_dl/__main__.py @@ -25,12 +25,20 @@ class main(object): required_args.add_argument('-i', '--input', nargs=1, help='Inputs the URL to anime.') parser.add_argument('-r', '--resolution', nargs=1, help='Inputs the URL to anime.', default='720p') parser.add_argument('-l', '--language', nargs=1, help='Selects the language for the show.', default='Japanese') + parser.add_argument('--skip', action='store_true', help='skips the video download and downloads only subs.') args = parser.parse_args() if args.version: print("Current Version : %s" % __version__) exit() + if args.skip: + print("Will be skipping video downloads") + skipper = "yes" + elif not args.skip: + # print("No skipper") + skipper = "no" + if args.username == None or args.password == None or args.input == None: print("Please enter the required arguments. Run __main__.py --help") exit() @@ -45,4 +53,4 @@ class main(object): if type(args.language) == list: args.language = args.language[0] - AnimeDL(url= args.input, username=args.username, password=args.password, resolution=args.resolution, language=args.language) + AnimeDL(url= args.input, username=args.username, password=args.password, resolution=args.resolution, language=args.language, skipper=skipper) diff --git a/anime_dl/sites/crunchyroll.py b/anime_dl/sites/crunchyroll.py index e3042b3..333cf9b 100644 --- a/anime_dl/sites/crunchyroll.py +++ b/anime_dl/sites/crunchyroll.py @@ -3,9 +3,9 @@ from cfscrape import create_scraper from requests import session -from re import search, findall, match +from re import search, findall, match, sub from os import getcwd -from subprocess import call +from subprocess import call, check_output # External libs have been taken from youtube-dl for decoding the subtitles. from external.utils import bytes_to_intlist, intlist_to_bytes from external.aes import aes_cbc_decrypt @@ -21,7 +21,7 @@ class CrunchyRoll(object): - def __init__(self, url, password, username, resolution, language): + def __init__(self, url, password, username, resolution, language, skipper): Crunchy_Show_regex = r'https?://(?:(?Pwww|m)\.)?(?Pcrunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P[\w\-]+))/?(?:\?|$)' Crunchy_Video_regex = r'https?:\/\/(?:(?Pwww|m)\.)?(?Pcrunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P[0-9]+))(?:[/?&]|$)' @@ -31,12 +31,15 @@ def __init__(self, url, password, username, resolution, language): if Crunchy_Video: cookies, Token = self.webpagedownloader(url=url, username=username[0], password=password[0]) - self.singleEpisode( - url=url, cookies=cookies, token=Token, resolution=resolution) + if skipper == "yes": + self.onlySubs(url=url, cookies=cookies) + else: + self.singleEpisode( + url=url, cookies=cookies, token=Token, resolution=resolution) elif Crunchy_Show: cookies, Token = self.webpagedownloader(url=url, username=username[0], password=password[0]) - self.wholeShow(url=url, cookie=cookies, token=Token, language=language, resolution=resolution) + self.wholeShow(url=url, cookie=cookies, token=Token, language=language, resolution=resolution, skipper=skipper) def loginCheck(self, htmlsource): # Open the page and check the title. CrunchyRoll redirects the user and the title has the text "Redirecting...". @@ -134,10 +137,19 @@ def singleEpisode(self, url, cookies, token, resolution): 1)).strip() # print("m3u8_link : %s\nanime_name : %s\nepisode_number : %s\nwidth : %s\nheight : %s\n" % (m3u8_link_raw, anime_name, episode_number, width, height)) # self.subFetcher(xml=str(xml_page), anime_name=anime_name, episode_number=episode_number) - file_name = str(anime_name) + " - " + str( + file_name = sub(r'[^A-Za-z0-9\ \-\' \\]+', '', str(anime_name)) + " - " + str( episode_number) + " [%sx%s].mp4" % (width, height) # print("File Name : %s\n" % file_name) + try: + MAX_PATH = int(check_output(['getconf', 'PATH_MAX', '/'])) + #print(MAX_PATH) + except (Exception): + MAX_PATH = 4096 + + if len(file_name) > MAX_PATH: + file_name = file_name[:MAX_PATH] + if not path.exists("Output"): makedirs("Output") @@ -196,9 +208,17 @@ def singleEpisode(self, url, cookies, token, resolution): 1)).strip() # print("m3u8_link : %s\nanime_name : %s\nepisode_number : %s\nwidth : %s\nheight : %s\n" % (m3u8_link_raw, anime_name, episode_number, width, height)) # self.subFetcher(xml=str(xml_page), anime_name=anime_name, episode_number=episode_number) - file_name = str(anime_name) + " - " + str( + file_name = sub(r'[^A-Za-z0-9\ \-\' \\]+', '', str(anime_name)) + " - " + str( episode_number) + " [%sx%s].mp4" % (width, height) # print("File Name : %s\n" % file_name) + try: + MAX_PATH = int(check_output(['getconf', 'PATH_MAX', '/'])) + #print(MAX_PATH) + except (Exception): + MAX_PATH = 4096 + + if len(file_name) > MAX_PATH: + file_name = file_name[:MAX_PATH] if not path.exists("Output"): makedirs("Output") @@ -258,10 +278,19 @@ def singleEpisode(self, url, cookies, token, resolution): 1)).strip() # print("m3u8_link : %s\nanime_name : %s\nepisode_number : %s\nwidth : %s\nheight : %s\n" % (m3u8_link_raw, anime_name, episode_number, width, height)) # self.subFetcher(xml=str(xml_page), anime_name=anime_name, episode_number=episode_number) - file_name = str(anime_name) + " - " + str( + file_name = sub(r'[^A-Za-z0-9\ \-\' \\]+', '', str(anime_name)) + " - " + str( episode_number) + " [%sx%s].mp4" % (width, height) # print("File Name : %s\n" % file_name) + try: + MAX_PATH = int(check_output(['getconf', 'PATH_MAX', '/'])) + #print(MAX_PATH) + except (Exception): + MAX_PATH = 4096 + + if len(file_name) > MAX_PATH: + file_name = file_name[:MAX_PATH] + if not path.exists("Output"): makedirs("Output") @@ -320,10 +349,19 @@ def singleEpisode(self, url, cookies, token, resolution): 1)).strip() # print("m3u8_link : %s\nanime_name : %s\nepisode_number : %s\nwidth : %s\nheight : %s\n" % (m3u8_link_raw, anime_name, episode_number, width, height)) # self.subFetcher(xml=str(xml_page), anime_name=anime_name, episode_number=episode_number) - file_name = str(anime_name) + " - " + str( + file_name = sub(r'[^A-Za-z0-9\ \-\' \\]+', '', str(anime_name)) + " - " + str( episode_number) + " [%sx%s].mp4" % (width, height) # print("File Name : %s\n" % file_name) + try: + MAX_PATH = int(check_output(['getconf', 'PATH_MAX', '/'])) + #print(MAX_PATH) + except (Exception): + MAX_PATH = 4096 + + if len(file_name) > MAX_PATH: + file_name = file_name[:MAX_PATH] + if not path.exists("Output"): makedirs("Output") @@ -362,7 +400,7 @@ def singleEpisode(self, url, cookies, token, resolution): return (video_id, m3u8_link_raw, anime_name, episode_number, width, height, file_name, cookies, token) - def wholeShow(self, url, cookie, token, language, resolution): + def wholeShow(self, url, cookie, token, language, resolution, skipper): # print("Check my patreon for this : http://patreon.com/Xonshiz") headers = { @@ -387,28 +425,38 @@ def wholeShow(self, url, cookie, token, language, resolution): print("Could not find the show links. Report on https://github.com/Xonshiz/anime-dl/issues/new") exit() - if str(language).lower() in ["english", "eng", "dub"]: - # If the "dub_list" is empty, that means there are no English Dubs for the show, or CR changed something. - if len(dub_list) == 0: - print("No English Dub Available For This Series.") - print("If you can see the Dubs, please open an Issue on https://github.com/Xonshiz/anime-dl/issues/new") - exit() - else: - print("Total Episodes to download : %s" % len(dub_list)) - for episode_url in dub_list[::-1]: - # cookies, Token = self.webpagedownloader(url=url) - # print("Dub list : %s" % dub_list) - self.singleEpisode(url=episode_url, cookies=cookie, token=token, resolution=resolution) - print("-----------------------------------------------------------") - print("\n") - else: - print("Total Episodes to download : %s" % len(sub_list)) + if skipper == "yes": + # print("DLing everything") + print("Total Subs to download : %s" % len(sub_list)) for episode_url in sub_list[::-1]: # cookies, Token = self.webpagedownloader(url=url) # print("Sub list : %s" % sub_list) - self.singleEpisode(url=episode_url, cookies=cookie, token=token, resolution=resolution) + self.onlySubs(url=episode_url, cookies=cookie) print("-----------------------------------------------------------") print("\n") + else: + if str(language).lower() in ["english", "eng", "dub"]: + # If the "dub_list" is empty, that means there are no English Dubs for the show, or CR changed something. + if len(dub_list) == 0: + print("No English Dub Available For This Series.") + print("If you can see the Dubs, please open an Issue on https://github.com/Xonshiz/anime-dl/issues/new") + exit() + else: + print("Total Episodes to download : %s" % len(dub_list)) + for episode_url in dub_list[::-1]: + # cookies, Token = self.webpagedownloader(url=url) + # print("Dub list : %s" % dub_list) + self.singleEpisode(url=episode_url, cookies=cookie, token=token, resolution=resolution) + print("-----------------------------------------------------------") + print("\n") + else: + print("Total Episodes to download : %s" % len(sub_list)) + for episode_url in sub_list[::-1]: + # cookies, Token = self.webpagedownloader(url=url) + # print("Sub list : %s" % sub_list) + self.singleEpisode(url=episode_url, cookies=cookie, token=token, resolution=resolution) + print("-----------------------------------------------------------") + print("\n") def subFetcher(self, xml, anime_name, episode_number): headers = { @@ -445,13 +493,60 @@ def subFetcher(self, xml, anime_name, episode_number): lang_code = str( search(r'lang_code\=\"(.*?)\"', str(subtitle)).group( 1)).strip() - sub_file_name = str(anime_name) + " - " + str( + sub_file_name = sub(r'[^A-Za-z0-9\ \-\' \\]+', '', str(anime_name)) + " - " + str( episode_number) + ".%s.ass" % lang_code + + try: + MAX_PATH = int(check_output(['getconf', 'PATH_MAX', '/'])) + #print(MAX_PATH) + except (Exception): + MAX_PATH = 4096 + + if len(sub_file_name) > MAX_PATH: + sub_file_name = sub_file_name[:MAX_PATH] # print(sub_file_name) print("Writing subtitles to files...") with open(sub_file_name, "w", encoding="utf-8") as sub_file: sub_file.write(str(sub_data)) + def onlySubs(self, url, cookies): + # print("Running only subs") + current_directory = getcwd() + video_id = str(url.split('-')[-1]).replace("/", "") + # print("URL : %s\nCookies : %s\nToken : %s\nResolution : %s\nMedia ID : %s" % (url, cookies, token, resolution, video_id)) + headers = { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.7 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.7', + 'Upgrade-Insecure-Requests': '1', + 'Accept-Encoding': 'gzip, deflate' + } + + sess = session() + sess = create_scraper(sess) + infoURL = "http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=108&video_quality=80¤t_page=%s" % ( + video_id, url) + xml_page = sess.get(url=infoURL, headers=headers, cookies=cookies).text + + m3u8_link_raw = str(search(r'\(.*?)\<\/file\>', xml_page).group(1)).strip().replace("&", "&") + anime_name = str(search(r'\(.*?)\<\/series_title\>', xml_page).group(1)).strip().replace("’", + "'").replace( + ":", " - ").replace("'", "'") + episode_number = str(search(r'\(.*?)\<\/episode_number\>', xml_page).group(1)).strip() + width = str(search(r'\(.*?)\<\/width\>', xml_page).group(1)).strip() + height = str(search(r'\(.*?)\<\/height\>', xml_page).group(1)).strip() + # print("m3u8_link : %s\nanime_name : %s\nepisode_number : %s\nwidth : %s\nheight : %s\n" % (m3u8_link_raw, anime_name, episode_number, width, height)) + if not path.exists("Output"): + makedirs("Output") + + self.subFetcher(xml=str(xml_page), anime_name=anime_name, episode_number=episode_number) + + for mkv_file in glob("*.ass"): + try: + move(mkv_file, current_directory + "/Output/") + except Exception as e: + print("Couldn't move the file. Got following error : \n") + print(e) + pass + def _decrypt_subtitles(self, data, iv, id): data = bytes_to_intlist(base64.b64decode(data.encode('utf-8'))) iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8'))) diff --git a/anime_dl/version.py b/anime_dl/version.py index 8857497..7cf827c 100644 --- a/anime_dl/version.py +++ b/anime_dl/version.py @@ -1,2 +1,2 @@ # Format : YY/MM/DD -__version__ = "2017.03.06.2" +__version__ = "2017.04.13" diff --git a/docs/Changelog.md b/docs/Changelog.md index 9095ab1..ed9c9db 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -5,4 +5,6 @@ - Fix for #2 [2017.03.06] - ReadMe updated for Python Script execution [2017.03.06] - Support for Whole Show Downloading for Crunchyroll [2017.03.06] -- Selection of language for the Crunchyroll Show [2017.03.06] \ No newline at end of file +- Selection of language for the Crunchyroll Show [2017.03.06] +- Downloading only subtitles (skip video downloads) [2017.04.13] +- Fix for [6](https://github.com/Xonshiz/anime-dl/issues/6) and Fix for [3](https://github.com/Xonshiz/anime-dl/issues/3) [2017.04.13] \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index cc40a60..a0a4c25 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,13 +31,13 @@ Anime-dl is a Command-line program to download anime from CrunchyRoll and Funima You can check the list of supported websites [**`HERE`**](https://github.com/Xonshiz/anime-dl/blob/master/Supported_Sites.md). ## Dependencies Installation -This script can run on multiple Operating Systems. So, if you're using the `python` script instead of the `windows binary` of this script, then you'll need to get things ready first. Since this script doesn't rely on a lot of external dependencies, you just need to grab a few things, same for all operating systems. +This script can run on multiple Operating Systems. But, the script depends on some external binaries or libs. We need `FFmpeg` and `Node.js` in our paths. 1.) Make sure you have Python installed and is present in your system's path. -2.) Grab [FFmpeg from this link.](https://ffmpeg.org/download.html) +2.) Grab [FFmpeg from this link](https://ffmpeg.org/download.html) and [Node.js from this link](https://nodejs.org/en/download/). -3.) Install FFmpeg and place it in the directory of this script, or put FFmpeg in your system's path. +3.) Install FFmpeg and Node.js and place it in the directory of this script, or put them in your system's path. 4.) Browse to the directory of this script and open command prompt/shell in that directory and run this command : @@ -49,7 +49,7 @@ python pip install -r requirements.txt After installing and setting up all the dependencies in your Operating System, you're good to go and use this script. The instructions for all the OS would remain same. Download [`THIS REPOSITORY`](https://github.com/Xonshiz/anime-dl/archive/master.zip) and put it somewhere in your system. Move over the `anime_dl` folder. -**Windows users**, it's better to not place it places where it requires administrator privileges. Good example of such place would be `C:\Windows`. This goes for both, the Python script and the windows binary file (.exe). +**Windows users**, it's better to not place it places where it requires administrator privileges. Good example of places to avoid would be `C:\Windows` etc.. This goes for both, the Python script and the windows binary file (.exe). **Linux/Debian** users make sure that this script is executable.Just run this command, if you run into problem(s) : @@ -59,8 +59,6 @@ The instructions for all the OS would remain same. Download [`THIS REPOSITORY`]( and then, execute with this : -`./anime-dl.py` - `./__main__.py` ## Python Support @@ -84,6 +82,7 @@ Currently, the script supports these arguments : -u,--username Indicates username for a website. [REQUIRED] -p,--password Indicates password for a website. [REQUIRED] -r,--resolution Indicates the desired resolution. (default = 720p) +--skip Skip video downloads (Will only download subtitles) -l,--language Selects the language for the show. (default = Japanese) [Langs = english, dub, sub, Japanese, eng] ``` @@ -141,7 +140,7 @@ If your're planning to open an issue for the script or ask for a new feature or ### Reporting Issues PLEASE RUN THIS SCRIPT IN A COMMAND LINE (as mentioned in the Usage section) AND DON'T SAY THAT `THE SCRIPT CLOSED TOO QUICK, I COULDN'T SEE`. -If you're here to make suggestions, please follow the basic syntax to post a request : +If you're here to report an issue, please follow the basic syntax to post a request : **Subject** : Error That You Get.