Fix .env

.env

@@ -1,2 +1,2 @@
-SCWEET_USERNAME=
-SCWEET_PASSWORD=
+SCWEET_USERNAME=*****@gmail.com
+SCWEET_PASSWORD=***************

Example.py

@@ -5,14 +5,14 @@
 # scrape top tweets with the words 'covid','covid19' in proximity and without replies.
 # the process is slower as the interval is smaller (choose an interval that can divide the period of time betwee, start and max date)
 
-data = scrap(words=['covid','covid19'], start_date="2020-04-01", max_date="2020-04-15", from_account = None,interval=1, 
+data = scrap(words=['covid','covid19'], start_date="2020-04-01", max_date="2020-04-15", from_account = None, interval=1, 
 	headless=True, display_type="Top", save_images=False, 
 	resume=False, filter_replies=True, proximity=True)
 
 # scrape top tweets of with the hashtag #covid19, in proximity and without replies.
 # the process is slower as the interval is smaller (choose an interval that can divide the period of time betwee, start and max date)
 
-data = scrap(hashtag="covid19", start_date="2020-04-01", max_date="2020-04-15", from_account = None,interval=1, 
+data = scrap(hashtag="covid19", start_date="2020-04-01", max_date="2020-04-15", from_account = None, interval=1, 
 	headless=True, display_type="Top", save_images=False, 
 	resume=False, filter_replies=True, proximity=True)
 
@@ -33,6 +33,9 @@
 # Enter your username and password in .env file. I recommande you dont use your main account.
 # Increase wait argument to avoid banning your account and maximise the crawling process if the internet is slow. I used 1 and it's safe.
 
-following = get_users_following(users=users, verbose=0, headless = True, wait=1)
+# set your .env file with SCWEET_USERNAME and SCWEET_PASSWORD variables and provide its path
+env_path = "****/****/****/.env"
 
-followers = get_users_followers(users=users, verbose=0, headless = True, wait=1)
+following = get_users_following(users=users, env=env_path, verbose=0, headless = True, wait=2)
+
+followers = get_users_followers(users=users, env=env_path, verbose=0, headless = True, wait=2)

Scweet/const.py

@@ -4,8 +4,8 @@
 
 current_dir = Path(__file__).parent.absolute()
 
-env_file = os.getenv("SCWEET_ENV_FILE", current_dir.parent.joinpath(".env"))
-dotenv.load_dotenv(env_file, verbose=True)
+#env_file = os.getenv("SCWEET_ENV_FILE", current_dir.parent.joinpath(".env"))
+#dotenv.load_dotenv(env_file, verbose=True)
 
 
 def load_env_variable(key, default_value=None, none_allowed=False):
@@ -15,5 +15,12 @@ def load_env_variable(key, default_value=None, none_allowed=False):
     return v
 
 
-USERNAME = load_env_variable("SCWEET_USERNAME", none_allowed=True)
-PASSWORD = load_env_variable("SCWEET_PASSWORD", none_allowed=True)
+def get_username(env):
+
+	dotenv.load_dotenv(env, verbose=True)
+	return load_env_variable("SCWEET_USERNAME", none_allowed=True)
+
+def get_password(env):
+
+	dotenv.load_dotenv(env, verbose=True)
+	return load_env_variable("SCWEET_PASSWORD", none_allowed=True)

Scweet/user.py

@@ -94,16 +94,16 @@ def log_user_page(user, driver, headless=True):
     sleep(random.uniform(1, 2))
 
 
-def get_users_followers(users, verbose=1, headless=True, wait=2):
+def get_users_followers(users, env, verbose=1, headless=True, wait=2):
 
-    followers = utils.get_users_follow(users, headless, "followers", verbose, wait=wait)
+    followers = utils.get_users_follow(users, headless, env, "followers", verbose, wait=wait)
 
     return followers
 
 
-def get_users_following(users, verbose=1, headless=True, wait=2):
+def get_users_following(users, env, verbose=1, headless=True, wait=2):
 
-    following = utils.get_users_follow(users, headless, "following", verbose, wait=wait)
+    following = utils.get_users_follow(users, headless, env, "following", verbose, wait=wait)
 
     return following
 

Scweet/utils.py

@@ -19,6 +19,8 @@
 from . import const
 import urllib
 
+from .const import get_username, get_password
+
 # current_dir = pathlib.Path(__file__).parent.absolute()
 
 def get_data(card, save_images = False, save_dir = None):
@@ -190,9 +192,9 @@ def get_last_date_from_csv(path):
     return datetime.datetime.strftime(max(pd.to_datetime(df["Timestamp"])), '%Y-%m-%dT%H:%M:%S.000Z')
 
 
-def log_in(driver, timeout=10):
-    username = const.USERNAME
-    password = const.PASSWORD
+def log_in(driver, env, timeout=10):
+    username = get_username(env) #const.USERNAME
+    password = get_password(env) #const.PASSWORD
 
     driver.get('https://www.twitter.com/login')
     username_xpath = '//input[@name="session[username_or_email]"]'
@@ -255,14 +257,14 @@ def keep_scroling(driver, data, writer, tweet_ids, scrolling, tweet_parsed, limi
     return driver, data, writer, tweet_ids, scrolling, tweet_parsed, scroll, last_position
 
 
-def get_users_follow(users, headless, follow=None, verbose=1, wait=2):
+def get_users_follow(users, headless, env, follow=None, verbose=1, wait=2):
 	""" get the following or followers of a list of users """
 
 	# initiate the driver
 	driver = init_driver(headless=headless)
 	sleep(wait)
 	# log in (the .env file should contain the username and password)
-	log_in(driver)
+	log_in(driver, env)
 	sleep(wait)
 	# followers and following dict of each user
 	follows_users = {}

outputs/covid19_2020-04-01_2020-04-15.csv

@@ -1,14 +0,0 @@
-UserScreenName,UserName,Timestamp,Text,Embedded_text,Emojis,Comments,Likes,Retweets,Image link,Tweet URL
-imane benzarouel,@ImaneBenzarouel,2020-04-01T19:44:02.000Z,"BCEAO, intéressantes mesures prises pour encourager les paiements mobiles et contrecarrer certaines suites du #covid19. Bravo à toute l’équipe derrière. #inclusionfinanciere #financialinclusion #mobilepayment","Quote Tweet
-Jacqueline Niffone
-@Niffone_J_D
- · Apr 1, 2020
-Communiqué de la @BCEAO_Officiel - Promotion des paiements électroniques # mesures crise sanitaire #Covid-19 @sogued @NewsAFI @ImaneBenzarouel @Naboufall @cinalawson @MazamessoA @fnfi_togo @afpfr @Financialafrik @AfDB_Group @FSDAfrica @KamissaCamara @AurelieASZ @artcitic",,,1,1,"['https://pbs.twimg.com/profile_images/1346692179079688194/RuWbfrm-_mini.jpg', 'https://pbs.twimg.com/media/EUia471XQAM8kYQ?format=jpg&name=small', 'https://pbs.twimg.com/media/EUia5vaWAAAX5z3?format=jpg&name=small']",https://twitter.com/ImaneBenzarouel/status/1245436781362851842
-Ghita Tazi,@Ghitazi,2020-04-01T19:39:51.000Z,"COVID-19 
-Le nombre des nouveaux cas est en baisse.
-** Du 28 au 29 mars : 104 cas
-** Du 29 au 30 mars : 71 cas
-** Du 30 au 31 mars : 68 cas
-** Du 31 mars au 1er avril : 40 cas
-
-#COVID19 #بقا_فدارك",,,1,,5,[],https://twitter.com/Ghitazi/status/1245435730966519809