From 1a403a3b0da9b74630dd96813fde5dd7d259c443 Mon Sep 17 00:00:00 2001 From: Evert Geldenhuys Date: Sun, 14 May 2017 18:27:16 +0200 Subject: [PATCH] Closes #7, closes #5, closes #3 --- CHANGELOG.md | 9 ++++++++ README.md | 11 ++++++++-- clickup-mirror.py | 52 +++++++++++++++++++++++++++++++++++++---------- 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cce90f..a9262bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## 1.1.0 - 2017-05-14 +### Fixed +- Database file being saved at the wrong place + +### Added +- Can now specify `all` as a file type to dowload all file types (Except `iso`) +- Pass the file containing the `s_session_id` with the `-s` flag +- Update the database file with `-u` + ## 1.0.4 - 2017-05-12 ### Fixed - Incorrect data type returned when session token could not be loaded diff --git a/README.md b/README.md index 254a0b0..790e70f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ Log into ClickUP using Chrome or Firefox before running the script. ``` usage: clickup-mirror.py [-h] [-b BROWSER] [-o OUTPUR_DIR] [-t FILE_TYPES] - [-d DATA_FILE] [-n] [--version] + [-d DATA_FILE] [-u] [-s SESSION_FILE] [-n] + [--version] Create a file mirror from ClickUP @@ -35,21 +36,27 @@ optional arguments: mirror/] -t FILE_TYPES, --file-types FILE_TYPES Comma-separated list of file types to download - [default: pdf] + [default: all] -d DATA_FILE, --data-file DATA_FILE File to be used for the database cache. Will be created if it does not exist [default: mirror/database.json] + -u, --update-database + Overwrite the database file + -s SESSION_FILE, --session-file SESSION_FILE + File containing the s_session_id -n, --dry-run Only generate the database file. Do not download anything [default: False] --version show program's version number and exit + BROWSER_TYPE: chromium chrome firefox COMMON FILE TYPES: + all pdf doc docx diff --git a/clickup-mirror.py b/clickup-mirror.py index 8242f2b..a4ec48b 100644 --- a/clickup-mirror.py +++ b/clickup-mirror.py @@ -24,24 +24,34 @@ TYPE_FILE = 'resource/x-bb-file' TYPE_DOCUMENT = 'resource/x-bb-document' +USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' + YEAR = 2017 -VERSION = 'v1.0.4' +VERSION = 'v1.1.0' def main(): parser = argparse.ArgumentParser(description='Create a file mirror from ClickUP') parser.add_argument('-b', '--browser', help='The browser to be used for authentication [default: chromium]', required=False, type=str, default='chromium') parser.add_argument('-o', '--outpur-dir', help='The directory to create the mirror in [default: mirror/]', required=False, type=str, default='mirror/') - parser.add_argument('-t', '--file-types', help='Comma-separated list of file types to download [default: pdf]', required=False, type=str, default='pdf') + parser.add_argument('-t', '--file-types', help='Comma-separated list of file types to download [default: all]', required=False, type=str, default='all') parser.add_argument('-d', '--data-file', help='File to be used for the database cache. \ Will be created if it does not exist [default: mirror/database.json]', required=False, type=str, default='mirror/database.json') + parser.add_argument('-u', '--update-database', help='Overwrite the database file', required=False, default=False, action='store_true') + parser.add_argument('-s', '--session-file', help='File containing the s_session_id', required=False, type=str) #parser.add_argument('-f', '--force-overwrite', help='Overwrite existing files in the mirror directroy [default: False]', required=False, action='store_true', default=False) parser.add_argument('-n', '--dry-run', help='Only generate the database file. Do not download anything [default: False]', required=False, action='store_true', default=False) parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION) - args = parser.parse_args() + args, unk = parser.parse_known_args() fileTypes = args.file_types.split(',') + allTypes = ['pdf', 'doc', 'docx', 'xlsx', 'sql', 'txt', 'reg', 'pptx', 'ods'] + + if 'all' in fileTypes: + fileTypes += allTypes + fileTypes.remove('all') + rootFolder = expanduser(args.outpur_dir) if not os.path.exists(rootFolder): @@ -55,22 +65,33 @@ def main(): print('File types: ' + str(fileTypes)) print('Mirror directory: ' + rootFolder) print('Database file: ' + dataFile) + print('Browser: ' + args.browser) if args.dry_run: print('DRY RUN...') - print('Fetching Session Token from ' + args.browser) - s_session_id = getCookie(args.browser) + if args.session_file: + print('Loading s_session_id from file ' + args.session_file) + f = open(args.session_file, 'r') + s_session_id = f.readline().strip() + f.close() - if not isLoggedIn(s_session_id): - print("Please log into ClickUP using " + args.browser) - exit(1) + if not isLoggedIn(s_session_id): + print('Invalid s_session_id provided in file') + exit(1) + else: + print('Fetching Session Token from ' + args.browser) + s_session_id = getCookie(args.browser) + + if not isLoggedIn(s_session_id): + print("Please log into ClickUP using " + args.browser) + exit(1) - if os.path.isfile(dataFile): + if os.path.isfile(dataFile) and not args.update_database: data = loadDataFile(dataFile) else: data = getStructure(s_session_id) - saveDataFile(args.data_file, data) + saveDataFile(dataFile, data) printData(data) @@ -123,7 +144,7 @@ def getCookie(browser): return s_session_id; def getResponse(s_session_id, method, url): - headers = {'Cookie': 's_session_id=' + s_session_id, 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36'} + headers = {'Cookie': 's_session_id=' + s_session_id, 'User-Agent': USER_AGENT} r = requests.request(method, url, headers=headers, allow_redirects=False) return r @@ -134,6 +155,15 @@ def isLoggedIn(s_session_id): if (r.status_code == 200): return True else: + print("[Error] Could not authenticate") + print('s_session_id = ' + s_session_id) + print("Request Headers: ") + print(r.request.headers) + + + print("\n\nResponse Headers: ") + print(r.headers) + return False def getUID(s_session_id):