From e67604883c8c5f480ef76955b56660abe221719e Mon Sep 17 00:00:00 2001 From: Yuli Date: Tue, 11 Feb 2020 14:18:41 +0800 Subject: [PATCH 1/7] Pipenv installation and Twitter API data fetch installed Pipenv for dependencies management installed libraries for HTTP request with pivenv installation details can be found in README.md Data fetching is done using requests library in worker.py before start fetching data, get all the keys and tokens required for this transaction all the detailscan be found in README.md --- workers/Pipfile | 13 +++++++++ workers/Pipfile.lock | 57 +++++++++++++++++++++++++++++++++++++ workers/README.md | 35 +++++++++++++++++++++++ workers/get_bearer_token.py | 36 +++++++++++++++++++++++ workers/worker.py | 28 ++++++++++++++++++ 5 files changed, 169 insertions(+) create mode 100644 workers/Pipfile create mode 100644 workers/Pipfile.lock create mode 100644 workers/README.md create mode 100644 workers/get_bearer_token.py create mode 100644 workers/worker.py diff --git a/workers/Pipfile b/workers/Pipfile new file mode 100644 index 0000000..bfafdcc --- /dev/null +++ b/workers/Pipfile @@ -0,0 +1,13 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" +base64 = "*" + +[requires] +python_version = "3.8" diff --git a/workers/Pipfile.lock b/workers/Pipfile.lock new file mode 100644 index 0000000..cf99f04 --- /dev/null +++ b/workers/Pipfile.lock @@ -0,0 +1,57 @@ +{ + "_meta": { + "hash": { + "sha256": "acbc8c4e7f2f98f1059b2a93d581ef43f4aa0c9741e64e6253adff8e35fbd99e" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + ], + "version": "==2019.11.28" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "index": "pypi", + "version": "==2.22.0" + }, + "urllib3": { + "hashes": [ + "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", + "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + ], + "version": "==1.25.8" + } + }, + "develop": {} +} diff --git a/workers/README.md b/workers/README.md new file mode 100644 index 0000000..8ea55d4 --- /dev/null +++ b/workers/README.md @@ -0,0 +1,35 @@ +# Worker + +## Installation +Ensure that pip and python is installed + +````` +install pipenv for dependencies management +pip install --user pipenv +make sure the path is added to the system +````` +install Requests library for HTTP request +pipenv install requests +list of dependencies will be listed on Pipfile + + +## Authentication for Twitter +Create an account folloed by an app +https://developer.twitter.com/en/account/get-started +````` +Get your application tokens at "Keys and Access Tokens" +consumer key, consumer secret key, access token, access token secret +````` +Use get_bearer_token.py to get Bearer Token +Bearer Token is required to use the Request lib for Twitter API +````` + +````` +## Fetch Data from API +Insert the Bearer Token into worker.py +````` + +Insert params for the Twitter API +Info about the endpoint of Twitter API +https://developer.twitter.com/en/docs/tweets/search/api-reference/get-search-tweets# +When the response status is 200, JSON formatted data will be returned diff --git a/workers/get_bearer_token.py b/workers/get_bearer_token.py new file mode 100644 index 0000000..ea18f5a --- /dev/null +++ b/workers/get_bearer_token.py @@ -0,0 +1,36 @@ +import base64 +import requests +import urllib.parse + +OAUTH2_TOKEN = 'https://api.twitter.com/oauth2/token' + + +def get_bearer_token(consumer_key, consumer_secret): + # enconde consumer key + consumer_key = urllib.parse.quote(consumer_key) + # encode consumer secret + consumer_secret = urllib.parse.quote(consumer_secret) + # create bearer token + bearer_token = consumer_key + ':' + consumer_secret + # base64 encode the token + base64_encoded_bearer_token = base64.b64encode(bearer_token.encode('utf-8')) + # set headers + headers = { + "Authorization": "Basic " + base64_encoded_bearer_token.decode('utf-8') + "", + "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8", + "Content-Length": "29"} + + response = requests.post(OAUTH2_TOKEN, headers=headers, data={'grant_type': 'client_credentials'}) + to_json = response.json() + print("token_type = %s\naccess_token = %s" % (to_json['token_type'], to_json['access_token'])) + + +def main(): + consumer_key = 'Enter your consumer key' + consumer_secret = 'Enter your consumer secret' + print("***** ***** ***** *****") + get_bearer_token(consumer_key, consumer_secret) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/workers/worker.py b/workers/worker.py new file mode 100644 index 0000000..7645e09 --- /dev/null +++ b/workers/worker.py @@ -0,0 +1,28 @@ +import requests + +base_url = 'https://api.twitter.com/' +search_url = '{}1.1/search/tweets.json'.format(base_url) +access_token = 'AAAAAAAAAAAAAAAAAAAAAPGxCQEAAAAADbQQiAp5ZEJ%2BoV1Ze7fmQwY27Xk%3DdqnLLg4GcUrhVz5zYpkplPxQ7DZNnBlICYOhngVeVrtcHQabQh' + + +search_headers = { + 'Authorization': 'Bearer {}'.format(access_token) +} + +search_params = { + 'q': 'Makeup', + 'result_type': 'recent', + 'count': 2 +} + + +search_resp = requests.get(search_url, headers=search_headers, params=search_params) + +if search_resp.status_code == 200 : + tweet_data = search_resp.json() + for tweet in tweet_data["statuses"] : + # print(tweet['entities']) + for t in tweet['entities']['user_mentions'] : + print(t) +else : + print('Result for' , search_url , 'is unsuccesful') \ No newline at end of file From 13f8792ded96a6e91192189db825f7e6395158d9 Mon Sep 17 00:00:00 2001 From: Yuli Date: Thu, 13 Feb 2020 12:32:03 +0800 Subject: [PATCH 2/7] Created a config file for environment variables - installed a library called python-detonv to manage the environment variables files - edited README.md file --- workers/.env | 4 +++ workers/Pipfile | 3 ++- workers/Pipfile.lock | 29 +++++++++++++++++++- workers/README.md | 49 +++++++++++++++++++++------------- workers/connect_to_postgres.py | 14 ++++++++++ workers/worker.py | 13 ++++----- 6 files changed, 85 insertions(+), 27 deletions(-) create mode 100644 workers/.env create mode 100644 workers/connect_to_postgres.py diff --git a/workers/.env b/workers/.env new file mode 100644 index 0000000..f82e9c0 --- /dev/null +++ b/workers/.env @@ -0,0 +1,4 @@ +#TWITTER API TOKENS AND KEYS +BEARER_TOKEN = AAAAAAAAAAAAAAAAAAAAAPGxCQEAAAAADbQQiAp5ZEJ%2BoV1Ze7fmQwY27Xk%3DdqnLLg4GcUrhVz5zYpkplPxQ7DZNnBlICYOhngVeVrtcHQabQh +ACC_TOKEN = 1226371381165576193-q5jurLd4dAqCNF2dpZPSmqbdxFr5RJ +ACC_TOKEN_SECRET = 4ThKuRMbP538csiDvfLCM5EF9eeLsErKQXcJEd9eFt4wy diff --git a/workers/Pipfile b/workers/Pipfile index bfafdcc..cd1ad49 100644 --- a/workers/Pipfile +++ b/workers/Pipfile @@ -7,7 +7,8 @@ verify_ssl = true [packages] requests = "*" -base64 = "*" +psycopg2 = "*" +python-dotenv = "*" [requires] python_version = "3.8" diff --git a/workers/Pipfile.lock b/workers/Pipfile.lock index cf99f04..9299965 100644 --- a/workers/Pipfile.lock +++ b/workers/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "acbc8c4e7f2f98f1059b2a93d581ef43f4aa0c9741e64e6253adff8e35fbd99e" + "sha256": "384f0f5013c6076fe1ad3192c21e5e8d55dc81eb6cc9dcebb433d72a8477dc1f" }, "pipfile-spec": 6, "requires": { @@ -37,6 +37,33 @@ ], "version": "==2.8" }, + "psycopg2": { + "hashes": [ + "sha256:4212ca404c4445dc5746c0d68db27d2cbfb87b523fe233dc84ecd24062e35677", + "sha256:47fc642bf6f427805daf52d6e52619fe0637648fe27017062d898f3bf891419d", + "sha256:72772181d9bad1fa349792a1e7384dde56742c14af2b9986013eb94a240f005b", + "sha256:8396be6e5ff844282d4d49b81631772f80dabae5658d432202faf101f5283b7c", + "sha256:893c11064b347b24ecdd277a094413e1954f8a4e8cdaf7ffbe7ca3db87c103f0", + "sha256:92a07dfd4d7c325dd177548c4134052d4842222833576c8391aab6f74038fc3f", + "sha256:965c4c93e33e6984d8031f74e51227bd755376a9df6993774fd5b6fb3288b1f4", + "sha256:9ab75e0b2820880ae24b7136c4d230383e07db014456a476d096591172569c38", + "sha256:b0845e3bdd4aa18dc2f9b6fb78fbd3d9d371ad167fd6d1b7ad01c0a6cdad4fc6", + "sha256:dca2d7203f0dfce8ea4b3efd668f8ea65cd2b35112638e488a4c12594015f67b", + "sha256:ed686e5926929887e2c7ae0a700e32c6129abb798b4ad2b846e933de21508151", + "sha256:ef6df7e14698e79c59c7ee7cf94cd62e5b869db369ed4b1b8f7b729ea825712a", + "sha256:f898e5cc0a662a9e12bde6f931263a1bbd350cfb18e1d5336a12927851825bb6" + ], + "index": "pypi", + "version": "==2.8.4" + }, + "python-dotenv": { + "hashes": [ + "sha256:8429f459fc041237d98c9ff32e1938e7e5535b5ff24388876315a098027c3a57", + "sha256:ca9f3debf2262170d6f46571ce4d6ca1add60bb93b69c3a29dcb3d1a00a65c93" + ], + "index": "pypi", + "version": "==0.11.0" + }, "requests": { "hashes": [ "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", diff --git a/workers/README.md b/workers/README.md index 8ea55d4..8c77e2e 100644 --- a/workers/README.md +++ b/workers/README.md @@ -3,33 +3,44 @@ ## Installation Ensure that pip and python is installed -````` -install pipenv for dependencies management +install pipenv for dependencies management. List of dependencies will be listed on Pipfile. Make sure the pipenv path is added to the system. +``` pip install --user pipenv -make sure the path is added to the system -````` +``` + install Requests library for HTTP request +``` pipenv install requests -list of dependencies will be listed on Pipfile +``` + +install psycopg2 for library to access PostgreSQL Database +``` +pipenv install psycopg2 +``` +install python-dotenv to add environment variables into the app +``` +pip install python-dotenv +``` ## Authentication for Twitter -Create an account folloed by an app -https://developer.twitter.com/en/account/get-started -````` +Create an account followed by application creation on Twitter Developer Account. More information can be found in https://developer.twitter.com/en/account/get-started Get your application tokens at "Keys and Access Tokens" -consumer key, consumer secret key, access token, access token secret -````` -Use get_bearer_token.py to get Bearer Token +- consumer key +- consumer secret key +- access token +- access token secret + +Beare Token can be generated from get_bearer_token.py Bearer Token is required to use the Request lib for Twitter API -````` -````` ## Fetch Data from API -Insert the Bearer Token into worker.py -````` +- Insert the Bearer Token into worker.py +- Insert params for the Twitter API +Info about the endpoint of Twitter API https://developer.twitter.com/en/docs/tweets/search/api-reference/get-search-tweets# + +## Environment variables set up +- Create a .env file and input the variables +- Call dotenv library and os to get the desired variables + -Insert params for the Twitter API -Info about the endpoint of Twitter API -https://developer.twitter.com/en/docs/tweets/search/api-reference/get-search-tweets# -When the response status is 200, JSON formatted data will be returned diff --git a/workers/connect_to_postgres.py b/workers/connect_to_postgres.py new file mode 100644 index 0000000..79e9aef --- /dev/null +++ b/workers/connect_to_postgres.py @@ -0,0 +1,14 @@ +import psycopg2 + +def connection() : + try : + connection_details = "dbname='makeupdb' user='icebear' host='localhost' port='5432'" + connection = psycopg2.connect(connection_details) + connection.autocommit = True + cursor = connection.cursor() + except : + print('connection unsuccessful') + + +if __name__ == '__main__' : + connection() \ No newline at end of file diff --git a/workers/worker.py b/workers/worker.py index 7645e09..b8d7005 100644 --- a/workers/worker.py +++ b/workers/worker.py @@ -1,9 +1,11 @@ import requests +import os +from dotenv import load_dotenv +load_dotenv() base_url = 'https://api.twitter.com/' search_url = '{}1.1/search/tweets.json'.format(base_url) -access_token = 'AAAAAAAAAAAAAAAAAAAAAPGxCQEAAAAADbQQiAp5ZEJ%2BoV1Ze7fmQwY27Xk%3DdqnLLg4GcUrhVz5zYpkplPxQ7DZNnBlICYOhngVeVrtcHQabQh' - +access_token = os.getenv('BEARER_TOKEN') search_headers = { 'Authorization': 'Bearer {}'.format(access_token) @@ -15,14 +17,13 @@ 'count': 2 } - search_resp = requests.get(search_url, headers=search_headers, params=search_params) if search_resp.status_code == 200 : tweet_data = search_resp.json() for tweet in tweet_data["statuses"] : - # print(tweet['entities']) - for t in tweet['entities']['user_mentions'] : - print(t) + print(tweet['entities']) + # for t in tweet['entities']['user_mentions'] : + # print(t) else : print('Result for' , search_url , 'is unsuccesful') \ No newline at end of file From 0c6a1349be9ea7a29731084a8439b8a54249f9c7 Mon Sep 17 00:00:00 2001 From: Yuli Date: Sat, 15 Feb 2020 14:07:45 +0800 Subject: [PATCH 3/7] Merged worker.py into connect_to_postgres.py: 1. deleted worker and moved logic into connect_to_postgres 2. updated env with postgres connection details 3. updated README to describe .env 4. added a section to describe postgres connection and queries into README --- workers/.env | 7 +++ workers/README.md | 9 +++- workers/connect_to_postgres.py | 83 ++++++++++++++++++++++++++++++---- workers/worker.py | 29 ------------ 4 files changed, 89 insertions(+), 39 deletions(-) delete mode 100644 workers/worker.py diff --git a/workers/.env b/workers/.env index f82e9c0..f30c345 100644 --- a/workers/.env +++ b/workers/.env @@ -2,3 +2,10 @@ BEARER_TOKEN = AAAAAAAAAAAAAAAAAAAAAPGxCQEAAAAADbQQiAp5ZEJ%2BoV1Ze7fmQwY27Xk%3DdqnLLg4GcUrhVz5zYpkplPxQ7DZNnBlICYOhngVeVrtcHQabQh ACC_TOKEN = 1226371381165576193-q5jurLd4dAqCNF2dpZPSmqbdxFr5RJ ACC_TOKEN_SECRET = 4ThKuRMbP538csiDvfLCM5EF9eeLsErKQXcJEd9eFt4wy + +#POSTGRESQL INFO +USERNAME = icebear +PASSWORD = 1234 +DB_NAME = makeupdb +DB_PORT = 5432 +DB_HOST = localhost \ No newline at end of file diff --git a/workers/README.md b/workers/README.md index 8c77e2e..24ec03e 100644 --- a/workers/README.md +++ b/workers/README.md @@ -31,7 +31,7 @@ Get your application tokens at "Keys and Access Tokens" - access token - access token secret -Beare Token can be generated from get_bearer_token.py +Bearer Token can be generated from get_bearer_token.py Bearer Token is required to use the Request lib for Twitter API ## Fetch Data from API @@ -43,4 +43,11 @@ Info about the endpoint of Twitter API https://developer.twitter.com/en/docs/twe - Create a .env file and input the variables - Call dotenv library and os to get the desired variables +## Connect to PostgreSQL server and queries +- start postgres server with psql -U +- insert all the params to psycopg2.connect +- use INSERT query to insert data each time it is fetched +- commit and close connection and its cursor + + diff --git a/workers/connect_to_postgres.py b/workers/connect_to_postgres.py index 79e9aef..69e136f 100644 --- a/workers/connect_to_postgres.py +++ b/workers/connect_to_postgres.py @@ -1,14 +1,79 @@ import psycopg2 +import requests +import os +import worker +from dotenv import load_dotenv +load_dotenv() -def connection() : - try : - connection_details = "dbname='makeupdb' user='icebear' host='localhost' port='5432'" - connection = psycopg2.connect(connection_details) - connection.autocommit = True - cursor = connection.cursor() - except : - print('connection unsuccessful') +DB_NAME = os.getenv('DB_NAME') +DB_HOST = os.getenv('DB_HOST') +DB_PORT = os.getenv('DB_PORT') +USERNAME = os.getenv('USERNAME') +PASSWORD = os.getenv('PASSWORD') +class DatabaseCon : + def __init__(self) : + try : + #connect to postgres server + self.connection = psycopg2.connect(database = DB_NAME , user = USERNAME , password = PASSWORD , host = DB_HOST , port = DB_PORT ) + self.connection.autocommit = True + self.cursor = self.connection.cursor() + print('CONNECTED!') + except : + print('connection unsuccessful') + + def twitter_conn(self) : + base_url = 'https://api.twitter.com/' + search_url = '{}1.1/search/tweets.json'.format(base_url) + access_token = os.getenv('BEARER_TOKEN') + + search_headers = { + 'Authorization': 'Bearer {}'.format(access_token) + } + + search_params = { + 'q': 'Makeup', + 'result_type': 'recent', + 'count': 3 + } + + #Send request to Twitter API + search_resp = requests.get(search_url, headers=search_headers, params=search_params) + + return search_resp + + def data_get_and_insert(self, search_resp) : + + #Postgres query and params + sql_query = """INSERT INTO tweets(name, text, time) VALUES ( %s, %s, %s) RETURNING id ;""" + id = None + + if search_resp.status_code == 200 : + + #convert text into a data object (dictionary) for Python + tweet_data = search_resp.json() + + #Get data from Twitter API + for tweet in tweet_data["statuses"] : + text = tweet['text'] + time = tweet['created_at'] + name = tweet['user']['screen_name'] + print('FIRST DATA' , text +'\n'+ time +'\n'+ name ) + + #Insert data into postgres + self.cursor.execute(sql_query , (name, text, time)) + id = self.cursor.fetchone()[0] + print('SECOND DATA' , text +'\n'+ time +'\n'+ name ) + + #Postgres commit and close cursor and connection + self.connection.commit() + self.cursor.close() + self.connection.close() + else : + print('Result for' , search_url , 'is unsuccesful') if __name__ == '__main__' : - connection() \ No newline at end of file + db = DatabaseCon() + db.twitter_conn() + search_resp = db.twitter_conn() + db.data_get_and_insert(search_resp) \ No newline at end of file diff --git a/workers/worker.py b/workers/worker.py deleted file mode 100644 index b8d7005..0000000 --- a/workers/worker.py +++ /dev/null @@ -1,29 +0,0 @@ -import requests -import os -from dotenv import load_dotenv -load_dotenv() - -base_url = 'https://api.twitter.com/' -search_url = '{}1.1/search/tweets.json'.format(base_url) -access_token = os.getenv('BEARER_TOKEN') - -search_headers = { - 'Authorization': 'Bearer {}'.format(access_token) -} - -search_params = { - 'q': 'Makeup', - 'result_type': 'recent', - 'count': 2 -} - -search_resp = requests.get(search_url, headers=search_headers, params=search_params) - -if search_resp.status_code == 200 : - tweet_data = search_resp.json() - for tweet in tweet_data["statuses"] : - print(tweet['entities']) - # for t in tweet['entities']['user_mentions'] : - # print(t) -else : - print('Result for' , search_url , 'is unsuccesful') \ No newline at end of file From c1457745ac65d3937f288b2a1a1084c0a143b7ec Mon Sep 17 00:00:00 2001 From: Yuli Date: Sat, 15 Feb 2020 15:46:13 +0800 Subject: [PATCH 4/7] removed .env file from workers/ --- workers/.env | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 workers/.env diff --git a/workers/.env b/workers/.env deleted file mode 100644 index f30c345..0000000 --- a/workers/.env +++ /dev/null @@ -1,11 +0,0 @@ -#TWITTER API TOKENS AND KEYS -BEARER_TOKEN = AAAAAAAAAAAAAAAAAAAAAPGxCQEAAAAADbQQiAp5ZEJ%2BoV1Ze7fmQwY27Xk%3DdqnLLg4GcUrhVz5zYpkplPxQ7DZNnBlICYOhngVeVrtcHQabQh -ACC_TOKEN = 1226371381165576193-q5jurLd4dAqCNF2dpZPSmqbdxFr5RJ -ACC_TOKEN_SECRET = 4ThKuRMbP538csiDvfLCM5EF9eeLsErKQXcJEd9eFt4wy - -#POSTGRESQL INFO -USERNAME = icebear -PASSWORD = 1234 -DB_NAME = makeupdb -DB_PORT = 5432 -DB_HOST = localhost \ No newline at end of file From 6b6f6d9569fd7d807b355b5008b4a42fff2b6087 Mon Sep 17 00:00:00 2001 From: Yuli Date: Sat, 15 Feb 2020 16:16:44 +0800 Subject: [PATCH 5/7] renamed connect_to_postgres to DatabaseConnection --- workers/{connect_to_postgres.py => DatabaseConnection.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename workers/{connect_to_postgres.py => DatabaseConnection.py} (98%) diff --git a/workers/connect_to_postgres.py b/workers/DatabaseConnection.py similarity index 98% rename from workers/connect_to_postgres.py rename to workers/DatabaseConnection.py index 69e136f..bfaae85 100644 --- a/workers/connect_to_postgres.py +++ b/workers/DatabaseConnection.py @@ -11,7 +11,7 @@ USERNAME = os.getenv('USERNAME') PASSWORD = os.getenv('PASSWORD') -class DatabaseCon : +class DatabaseConnection: def __init__(self) : try : #connect to postgres server From 9f7d0f30dcdc6d68c11736d192b5e9f236eb6d5f Mon Sep 17 00:00:00 2001 From: Yuli Date: Sun, 16 Feb 2020 00:02:47 +0800 Subject: [PATCH 6/7] Used ORM for database connection 1. installed SQLAlchemy as ORM with pipenv 2. made a connection to postgres database 3. inserted the data into database 4. updated README file to describe the ORM connection --- workers/ORMConnection.py | 66 ++++++++++++++++++++++++++++++++++++++++ workers/Pipfile | 1 + workers/Pipfile.lock | 9 +++++- workers/README.md | 19 +++++++++--- 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 workers/ORMConnection.py diff --git a/workers/ORMConnection.py b/workers/ORMConnection.py new file mode 100644 index 0000000..eac6a4e --- /dev/null +++ b/workers/ORMConnection.py @@ -0,0 +1,66 @@ +import sqlalchemy as db +import psycopg2 +import os +import requests +from dotenv import load_dotenv +load_dotenv() + +URL_ADDON = os.getenv('DATABASE_URL_ADDON') + +def orm_config() : + #ORM connection + engine = db.create_engine('postgresql+psycopg2://{}'.format(URL_ADDON)) + connection = engine.connect() + metadata = db.MetaData() + tweets = db.Table('tweets', metadata , autoload=True , autoload_with=engine) + + + return metadata, tweets, connection + +def twitter_conn() : + base_url = 'https://api.twitter.com/' + search_url = '{}1.1/search/tweets.json'.format(base_url) + access_token = os.getenv('BEARER_TOKEN') + + search_headers = { + 'Authorization': 'Bearer {}'.format(access_token) + } + + search_params = { + 'q': 'Makeup', + 'result_type': 'recent', + 'count': 3 + } + + #Send request to Twitter API + search_resp = requests.get(search_url, headers=search_headers, params=search_params) + + return search_resp + +def data_get_and_insert(search_resp, metadata, tweets, connection) : + + id = None + if search_resp.status_code == 200 : + + #convert text into a data object (dictionary) for Python + tweet_data = search_resp.json() + + #Get data from Twitter API + for tweet in tweet_data["statuses"] : + text = tweet['text'] + time = tweet['created_at'] + name = tweet['user']['screen_name'] + print('FIRST DATA' , text +'\n'+ time +'\n'+ name ) + + #ORM query + query = tweets.insert().values(name = name , text = text , time = time).returning(tweets.c.id) + result = connection.execute(query) + + else : + print('Result for' , search_url , 'is unsuccesful') + +if __name__ == '__main__': + orm_config() + metadata, tweets, connection = orm_config() + search_resp = twitter_conn() + data_get_and_insert(search_resp, metadata, tweets, connection) \ No newline at end of file diff --git a/workers/Pipfile b/workers/Pipfile index cd1ad49..e3abcd6 100644 --- a/workers/Pipfile +++ b/workers/Pipfile @@ -9,6 +9,7 @@ verify_ssl = true requests = "*" psycopg2 = "*" python-dotenv = "*" +sqlalchemy = "*" [requires] python_version = "3.8" diff --git a/workers/Pipfile.lock b/workers/Pipfile.lock index 9299965..f6a5ff8 100644 --- a/workers/Pipfile.lock +++ b/workers/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "384f0f5013c6076fe1ad3192c21e5e8d55dc81eb6cc9dcebb433d72a8477dc1f" + "sha256": "dcaca460360c68fce9c4082104fbc6d7df134c8debf8923030b97460c7d04437" }, "pipfile-spec": 6, "requires": { @@ -72,6 +72,13 @@ "index": "pypi", "version": "==2.22.0" }, + "sqlalchemy": { + "hashes": [ + "sha256:64a7b71846db6423807e96820993fa12a03b89127d278290ca25c0b11ed7b4fb" + ], + "index": "pypi", + "version": "==1.3.13" + }, "urllib3": { "hashes": [ "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", diff --git a/workers/README.md b/workers/README.md index 24ec03e..fa647d9 100644 --- a/workers/README.md +++ b/workers/README.md @@ -1,26 +1,32 @@ # Worker ## Installation + Ensure that pip and python is installed install pipenv for dependencies management. List of dependencies will be listed on Pipfile. Make sure the pipenv path is added to the system. -``` +```sh pip install --user pipenv ``` install Requests library for HTTP request -``` +```sh pipenv install requests ``` install psycopg2 for library to access PostgreSQL Database -``` +```sh pipenv install psycopg2 ``` install python-dotenv to add environment variables into the app +```sh +pipenv install python-dotenv ``` -pip install python-dotenv + +install SQLAlchemy for ORM database +```sh +pipenv install SQLAlchemy ``` ## Authentication for Twitter @@ -49,5 +55,10 @@ Info about the endpoint of Twitter API https://developer.twitter.com/en/docs/twe - use INSERT query to insert data each time it is fetched - commit and close connection and its cursor +## ORM with SQLAlchemy setup and usage +SQLAlchemy is an Object-Relation Mapper that interact with database with Python programming language +- set up the engine and connection with the URL (with params to Postgres) +- keep all database information into Metadata object +- query with Python languange. More info can be found here https://docs.sqlalchemy.org/en/13/core/tutorial.html#connecting From 53041ebc9d04443b6d13031ef40de0e3643edd36 Mon Sep 17 00:00:00 2001 From: Yuli Date: Sat, 11 Apr 2020 17:01:53 +0800 Subject: [PATCH 7/7] Environment variables information in README file - steps to creat and use env file - dscription of every variables in env file --- workers/DatabaseConnection.py | 2 +- workers/ORMConnection.py | 7 ++++--- workers/README.md | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/workers/DatabaseConnection.py b/workers/DatabaseConnection.py index bfaae85..2f13bb6 100644 --- a/workers/DatabaseConnection.py +++ b/workers/DatabaseConnection.py @@ -50,7 +50,7 @@ def data_get_and_insert(self, search_resp) : if search_resp.status_code == 200 : - #convert text into a data object (dictionary) for Python + #convert text into a data object (dictionary) for Python tweet_data = search_resp.json() #Get data from Twitter API diff --git a/workers/ORMConnection.py b/workers/ORMConnection.py index eac6a4e..a4c9f37 100644 --- a/workers/ORMConnection.py +++ b/workers/ORMConnection.py @@ -29,7 +29,7 @@ def twitter_conn() : search_params = { 'q': 'Makeup', 'result_type': 'recent', - 'count': 3 + 'count': 10 } #Send request to Twitter API @@ -40,6 +40,7 @@ def twitter_conn() : def data_get_and_insert(search_resp, metadata, tweets, connection) : id = None + print(search_resp.status_code) if search_resp.status_code == 200 : #convert text into a data object (dictionary) for Python @@ -50,14 +51,14 @@ def data_get_and_insert(search_resp, metadata, tweets, connection) : text = tweet['text'] time = tweet['created_at'] name = tweet['user']['screen_name'] - print('FIRST DATA' , text +'\n'+ time +'\n'+ name ) + print('FIRST DATA' , text +'\n'+ time +'\n'+ name + '\n') #ORM query query = tweets.insert().values(name = name , text = text , time = time).returning(tweets.c.id) result = connection.execute(query) else : - print('Result for' , search_url , 'is unsuccesful') + print('Request unsuccesful') if __name__ == '__main__': orm_config() diff --git a/workers/README.md b/workers/README.md index fa647d9..866a9eb 100644 --- a/workers/README.md +++ b/workers/README.md @@ -48,6 +48,27 @@ Info about the endpoint of Twitter API https://developer.twitter.com/en/docs/twe ## Environment variables set up - Create a .env file and input the variables - Call dotenv library and os to get the desired variables +variables description : +1. BEARER_TOKEN + Token that consist of all kinds of character. + OAuth 2.0 generated from Account Token and Account Token Secret from Twitter developer page, Once these two tokens are available, use get_bearer_token.py to generate Bearer Token for the account +2. ACC_TOKEN + Token that consist of all kinds of character. + This token can be generated after user create an account and a new aplication in the developer page. +3. ACC_TOKEN_SECRET + Token that consist of all kinds of character. + This token can be generated after user create an account and a new aplication in the developer page. +4. USERNAME + consist of string/int that the user created in PostgreSQL +5. PASSWORD + consist of string/int that the user created in PostgreSQL +6. DB_NAME + consist of string/int that the user created in PostgreSQL +7. DB_PORT and DB_HOST + default value in PostgreSQL are DB_PORT = 5432 , DB_HOST = localhost +8. DATABASE_URL_ADDON + an URL for ORM endpoint of PostgreSQL that consist of + USERNAME:PASSWORD@HOST:PORT/DATABASE_NAME ## Connect to PostgreSQL server and queries - start postgres server with psql -U