diff --git a/InstaTweet/core/instatweet.py b/InstaTweet/core/instatweet.py index c1fcfb6..af43be0 100644 --- a/InstaTweet/core/instatweet.py +++ b/InstaTweet/core/instatweet.py @@ -4,7 +4,7 @@ import time from tqdm import tqdm from collections.abc import Iterable -from InstaTweet.utils import UserAgent, get_root, get_filepath +from InstaTweet.utils import get_agent, get_root, get_filepath from . import InstaClient, TweetClient DEFAULT_USER_MAPPING = {'hashtags': [], 'scraped': [], 'tweets': []} @@ -13,17 +13,18 @@ class InstaTweet: def __init__(self, **kwargs): - self.profile_name = kwargs.pop('profile', 'default') - self.session_id = kwargs.pop('session_id', '') - self.user_agent = kwargs.pop('user_agent', UserAgent().default) - self.twitter_keys = kwargs.pop('twitter_keys', None) - self.user_map = kwargs.pop('user_map', {}) + self.profile_name = kwargs.get('profile', 'default') - if self.is_default: - print('Using default profile.') - else: + if not self.is_default: self.load_profile(self.profile_name) + else: + self.session_id = kwargs.get('session_id', '') + self.twitter_keys = kwargs.get('twitter_keys', None) + self.user_agent = kwargs.get('user_agent', get_agent()) + self.user_map = kwargs.get('user_map', {}) + print('Using default profile.') + @classmethod def load(cls, profile_name: str): return cls(profile=profile_name) @@ -122,7 +123,7 @@ def validate(self): if missing_keys := [key for key in TweetClient.DEFAULT_KEYS if key not in self.twitter_keys]: raise KeyError(f''' - Invalid Twitter API Keys Provided + Invalid Twitter API Keys Provided Missing Keys: {missing_keys}''') if not all(self.twitter_keys.values()): @@ -190,19 +191,14 @@ def twitter_keys(self, keys: dict): def load_profile(self, profile_name: str): if profile_path := self.profile_exists(profile_name): profile = self.load_data(profile_path) - self._session_id = profile['session_id'] self.user_agent = profile['user_agent'] + self._session_id = profile['session_id'] self._twitter_keys = profile['twitter_keys'] self.user_map = profile['user_map'] - print(f'Loaded profile "{profile_name}"') - + self.profile_name = profile['profile'] + print(f'Loaded profile "{self.profile_name}"') else: - new_profile = input(f'No profile found with the name {profile_name},' - f'would you like to create this profile? Y/N' + '\n' + '>> ') - if new_profile.lower() == 'y': - self.save_profile(profile_name) - else: - raise FileNotFoundError('No profile loaded') + raise FileNotFoundError('No profile with that name was found') def save_profile(self, profile_name: str = None, alert: bool = True): """Update currently loaded profile, or save a new one. Name only required for new profiles.""" @@ -239,6 +235,7 @@ def is_default(self): @property def config(self): return { + 'profile': self.profile_name, 'session_id': self.session_id, 'user_agent': self.user_agent, 'twitter_keys': self.twitter_keys, diff --git a/InstaTweet/profiles/My Template.txt b/InstaTweet/profiles/My Template.txt index 481dfa6..8fbc4a5 100644 --- a/InstaTweet/profiles/My Template.txt +++ b/InstaTweet/profiles/My Template.txt @@ -1,6 +1,7 @@ { - "session_id": "sessionid", - "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", + "profile": "My Template", + "session_id": "session_id", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36", "twitter_keys": { "Consumer Key": "key", "Consumer Secret": "secret", diff --git a/InstaTweet/utils.py b/InstaTweet/utils.py new file mode 100644 index 0000000..a463e8d --- /dev/null +++ b/InstaTweet/utils.py @@ -0,0 +1,24 @@ +import os +import requests +from pathlib import Path +from bs4 import BeautifulSoup as bs + +BACKUP_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36' + + +def get_agent(): + r = requests.get('https://www.whatismybrowser.com/guides/the-latest-user-agent/chrome') + if r.ok: + soup = bs(r.text, 'html.parser') + if soup_agents := soup.find_all('span', {'class': 'code'}): + return soup_agents[0].text + # If function fails to scrape, will use hardcoded user agent + return BACKUP_AGENT + + +def get_root(): + return Path(__file__).parent + + +def get_filepath(filename, filetype='.txt'): + return os.path.join(get_root(), filename) + filetype diff --git a/InstaTweet/utils/__init__.py b/InstaTweet/utils/__init__.py deleted file mode 100644 index 320db8b..0000000 --- a/InstaTweet/utils/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .utils import * - diff --git a/InstaTweet/utils/default_agents.txt b/InstaTweet/utils/default_agents.txt deleted file mode 100644 index ef6301b..0000000 --- a/InstaTweet/utils/default_agents.txt +++ /dev/null @@ -1,302 +0,0 @@ -[ - { - "percent": "18.2%", - "system": "Chrome 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" - }, - { - "percent": "11.5%", - "system": "Firefox 95.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0" - }, - { - "percent": "6.4%", - "system": "Chrome 96.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" - }, - { - "percent": "4.4%", - "system": "Chrome 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36" - }, - { - "percent": "3.8%", - "system": "Firefox 95.0 Linux", - "useragent": "Mozilla/5.0 (X11; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0" - }, - { - "percent": "3.6%", - "system": "Firefox 91.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0" - }, - { - "percent": "3.2%", - "system": "Firefox 95.0 Linux", - "useragent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0" - }, - { - "percent": "2.7%", - "system": "Firefox 95.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:95.0) Gecko/20100101 Firefox/95.0" - }, - { - "percent": "2.4%", - "system": "Chrome 96.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36" - }, - { - "percent": "2.1%", - "system": "Edge 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62" - }, - { - "percent": "2.1%", - "system": "Safari 15.1 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" - }, - { - "percent": "2.0%", - "system": "Safari 15.2 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15" - }, - { - "percent": "1.7%", - "system": "Chrome 96.0 Linux", - "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" - }, - { - "percent": "1.6%", - "system": "Chrome 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" - }, - { - "percent": "1.5%", - "system": "Chrome 97.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - { - "percent": "1.0%", - "system": "Firefox 94.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0" - }, - { - "percent": "1.0%", - "system": "Chrome 96.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36" - }, - { - "percent": "0.7%", - "system": "Chrome 96.0 Linux", - "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" - }, - { - "percent": "0.7%", - "system": "Chrome 96.0 Linux", - "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36" - }, - { - "percent": "0.6%", - "system": "Safari 14.1 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15" - }, - { - "percent": "0.6%", - "system": "Firefox 95.0 Linux", - "useragent": "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0" - }, - { - "percent": "0.5%", - "system": "Chrome 96.0 Win7", - "useragent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" - }, - { - "percent": "0.5%", - "system": "Firefox 91.0 Linux", - "useragent": "Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 Firefox/91.0" - }, - { - "percent": "0.5%", - "system": "Firefox 78.0 Linux", - "useragent": "Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" - }, - { - "percent": "0.5%", - "system": "Edge 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36 Edg/96.0.1054.53" - }, - { - "percent": "0.4%", - "system": "Firefox 91.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0" - }, - { - "percent": "0.4%", - "system": "Firefox 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0" - }, - { - "percent": "0.4%", - "system": "Firefox 95.0 Win7", - "useragent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0" - }, - { - "percent": "0.4%", - "system": "Edge 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.57" - }, - { - "percent": "0.4%", - "system": "Safari 15.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15" - }, - { - "percent": "0.4%", - "system": "Firefox 94.0 Linux", - "useragent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0" - }, - { - "percent": "0.4%", - "system": "Chrome 95.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36" - }, - { - "percent": "0.4%", - "system": "Opera 82 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36 OPR/82.0.4227.33" - }, - { - "percent": "0.4%", - "system": "Firefox 94.0 Linux", - "useragent": "Mozilla/5.0 (X11; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0" - }, - { - "percent": "0.3%", - "system": "Firefox 94.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:94.0) Gecko/20100101 Firefox/94.0" - }, - { - "percent": "0.3%", - "system": "Chrome 95.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36" - }, - { - "percent": "0.3%", - "system": "Edge 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36 Edg/96.0.1054.43" - }, - { - "percent": "0.3%", - "system": "Opera 82 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 OPR/82.0.4227.43" - }, - { - "percent": "0.3%", - "system": "Chrome 96.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" - }, - { - "percent": "0.3%", - "system": "Chrome 97.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - { - "percent": "0.3%", - "system": "Chrome 96.0 Win8.1", - "useragent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" - }, - { - "percent": "0.3%", - "system": "Firefox 96.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:96.0) Gecko/20100101 Firefox/96.0" - }, - { - "percent": "0.3%", - "system": "Chrome 96.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" - }, - { - "percent": "0.3%", - "system": "Safari 15.1 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" - }, - { - "percent": "0.2%", - "system": "Opera 82 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 OPR/82.0.4227.50" - }, - { - "percent": "0.2%", - "system": "Chrome 96.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" - }, - { - "percent": "0.2%", - "system": "Opera 82 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36 OPR/82.0.4227.23" - }, - { - "percent": "0.2%", - "system": "Chrome 96.0 Win7", - "useragent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36" - }, - { - "percent": "0.2%", - "system": "Firefox 78.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0" - }, - { - "percent": "0.2%", - "system": "Firefox 95.0 Win8.1", - "useragent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0" - }, - { - "percent": "0.2%", - "system": "DefaultProperties unknown", - "useragent": "null" - }, - { - "percent": "0.2%", - "system": "Chrome 99.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.7113.93 Safari/537.36" - }, - { - "percent": "0.2%", - "system": "Chrome 95.0 Linux", - "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36" - }, - { - "percent": "0.2%", - "system": "Safari 13.1 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Safari/605.1.15" - }, - { - "percent": "0.2%", - "system": "Edge 97.0 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36 Edg/97.0.1072.55" - }, - { - "percent": "0.2%", - "system": "Chrome 96.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36" - }, - { - "percent": "0.2%", - "system": "Safari 14.0 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15" - }, - { - "percent": "0.2%", - "system": "Safari 14.1 macOS", - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15" - }, - { - "percent": "0.2%", - "system": "Opera 81 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 OPR/81.0.4196.61" - }, - { - "percent": "0.2%", - "system": "Opera 81 Win10", - "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 OPR/81.0.4196.61" - } -] diff --git a/InstaTweet/utils/utils.py b/InstaTweet/utils/utils.py deleted file mode 100644 index 691538e..0000000 --- a/InstaTweet/utils/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import json -import random -import pkgutil -from pathlib import Path - - -class UserAgent(object): - default_agents = json.loads(pkgutil.get_data(__name__, "default_agents.txt")) - - def __init__(self, agents=None): - self.agents = agents - - if self.agents is None: - self.agents = UserAgent.default_agents - - @classmethod - def common(cls, amount=20): - return cls(agents=UserAgent.default_agents[:amount]) - - @property - def default(self): - return self.default_agents[0]['useragent'] - - def random(self): - agent = random.choice(self.agents) - if isinstance(agent, dict) and agent.get('useragent'): - return agent['useragent'] - # Continue error check if needed - return agent - - -def get_root(): - return Path(__file__).parent.parent - - -def get_filepath(filename, filetype='.txt'): - return os.path.join(get_root(), filename) + filetype