diff --git a/workers/DatabaseConnection.py b/workers/DatabaseConnection.py new file mode 100644 index 0000000..2f13bb6 --- /dev/null +++ b/workers/DatabaseConnection.py @@ -0,0 +1,79 @@ +import psycopg2 +import requests +import os +import worker +from dotenv import load_dotenv +load_dotenv() + +DB_NAME = os.getenv('DB_NAME') +DB_HOST = os.getenv('DB_HOST') +DB_PORT = os.getenv('DB_PORT') +USERNAME = os.getenv('USERNAME') +PASSWORD = os.getenv('PASSWORD') + +class DatabaseConnection: + def __init__(self) : + try : + #connect to postgres server + self.connection = psycopg2.connect(database = DB_NAME , user = USERNAME , password = PASSWORD , host = DB_HOST , port = DB_PORT ) + self.connection.autocommit = True + self.cursor = self.connection.cursor() + print('CONNECTED!') + except : + print('connection unsuccessful') + + def twitter_conn(self) : + base_url = 'https://api.twitter.com/' + search_url = '{}1.1/search/tweets.json'.format(base_url) + access_token = os.getenv('BEARER_TOKEN') + + search_headers = { + 'Authorization': 'Bearer {}'.format(access_token) + } + + search_params = { + 'q': 'Makeup', + 'result_type': 'recent', + 'count': 3 + } + + #Send request to Twitter API + search_resp = requests.get(search_url, headers=search_headers, params=search_params) + + return search_resp + + def data_get_and_insert(self, search_resp) : + + #Postgres query and params + sql_query = """INSERT INTO tweets(name, text, time) VALUES ( %s, %s, %s) RETURNING id ;""" + id = None + + if search_resp.status_code == 200 : + + #convert text into a data object (dictionary) for Python + tweet_data = search_resp.json() + + #Get data from Twitter API + for tweet in tweet_data["statuses"] : + text = tweet['text'] + time = tweet['created_at'] + name = tweet['user']['screen_name'] + print('FIRST DATA' , text +'\n'+ time +'\n'+ name ) + + #Insert data into postgres + self.cursor.execute(sql_query , (name, text, time)) + id = self.cursor.fetchone()[0] + print('SECOND DATA' , text +'\n'+ time +'\n'+ name ) + + #Postgres commit and close cursor and connection + self.connection.commit() + self.cursor.close() + self.connection.close() + else : + print('Result for' , search_url , 'is unsuccesful') + +if __name__ == '__main__' : + db = DatabaseCon() + db.twitter_conn() + search_resp = db.twitter_conn() + db.data_get_and_insert(search_resp) \ No newline at end of file diff --git a/workers/ORMConnection.py b/workers/ORMConnection.py new file mode 100644 index 0000000..a4c9f37 --- /dev/null +++ b/workers/ORMConnection.py @@ -0,0 +1,67 @@ +import sqlalchemy as db +import psycopg2 +import os +import requests +from dotenv import load_dotenv +load_dotenv() + +URL_ADDON = os.getenv('DATABASE_URL_ADDON') + +def orm_config() : + #ORM connection + engine = db.create_engine('postgresql+psycopg2://{}'.format(URL_ADDON)) + connection = engine.connect() + metadata = db.MetaData() + tweets = db.Table('tweets', metadata , autoload=True , autoload_with=engine) + + + return metadata, tweets, connection + +def twitter_conn() : + base_url = 'https://api.twitter.com/' + search_url = '{}1.1/search/tweets.json'.format(base_url) + access_token = os.getenv('BEARER_TOKEN') + + search_headers = { + 'Authorization': 'Bearer {}'.format(access_token) + } + + search_params = { + 'q': 'Makeup', + 'result_type': 'recent', + 'count': 10 + } + + #Send request to Twitter API + search_resp = requests.get(search_url, headers=search_headers, params=search_params) + + return search_resp + +def data_get_and_insert(search_resp, metadata, tweets, connection) : + + id = None + print(search_resp.status_code) + if search_resp.status_code == 200 : + + #convert text into a data object (dictionary) for Python + tweet_data = search_resp.json() + + #Get data from Twitter API + for tweet in tweet_data["statuses"] : + text = tweet['text'] + time = tweet['created_at'] + name = tweet['user']['screen_name'] + print('FIRST DATA' , text +'\n'+ time +'\n'+ name + '\n') + + #ORM query + query = tweets.insert().values(name = name , text = text , time = time).returning(tweets.c.id) + result = connection.execute(query) + + else : + print('Request unsuccesful') + +if __name__ == '__main__': + orm_config() + metadata, tweets, connection = orm_config() + search_resp = twitter_conn() + data_get_and_insert(search_resp, metadata, tweets, connection) \ No newline at end of file diff --git a/workers/Pipfile b/workers/Pipfile new file mode 100644 index 0000000..e3abcd6 --- /dev/null +++ b/workers/Pipfile @@ -0,0 +1,15 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" +psycopg2 = "*" +python-dotenv = "*" +sqlalchemy = "*" + +[requires] +python_version = "3.8" diff --git a/workers/Pipfile.lock b/workers/Pipfile.lock new file mode 100644 index 0000000..f6a5ff8 --- /dev/null +++ b/workers/Pipfile.lock @@ -0,0 +1,91 @@ +{ + "_meta": { + "hash": { + "sha256": "dcaca460360c68fce9c4082104fbc6d7df134c8debf8923030b97460c7d04437" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + ], + "version": "==2019.11.28" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "psycopg2": { + "hashes": [ + "sha256:4212ca404c4445dc5746c0d68db27d2cbfb87b523fe233dc84ecd24062e35677", + "sha256:47fc642bf6f427805daf52d6e52619fe0637648fe27017062d898f3bf891419d", + "sha256:72772181d9bad1fa349792a1e7384dde56742c14af2b9986013eb94a240f005b", + "sha256:8396be6e5ff844282d4d49b81631772f80dabae5658d432202faf101f5283b7c", + "sha256:893c11064b347b24ecdd277a094413e1954f8a4e8cdaf7ffbe7ca3db87c103f0", + "sha256:92a07dfd4d7c325dd177548c4134052d4842222833576c8391aab6f74038fc3f", + "sha256:965c4c93e33e6984d8031f74e51227bd755376a9df6993774fd5b6fb3288b1f4", + "sha256:9ab75e0b2820880ae24b7136c4d230383e07db014456a476d096591172569c38", + "sha256:b0845e3bdd4aa18dc2f9b6fb78fbd3d9d371ad167fd6d1b7ad01c0a6cdad4fc6", + "sha256:dca2d7203f0dfce8ea4b3efd668f8ea65cd2b35112638e488a4c12594015f67b", + "sha256:ed686e5926929887e2c7ae0a700e32c6129abb798b4ad2b846e933de21508151", + "sha256:ef6df7e14698e79c59c7ee7cf94cd62e5b869db369ed4b1b8f7b729ea825712a", + "sha256:f898e5cc0a662a9e12bde6f931263a1bbd350cfb18e1d5336a12927851825bb6" + ], + "index": "pypi", + "version": "==2.8.4" + }, + "python-dotenv": { + "hashes": [ + "sha256:8429f459fc041237d98c9ff32e1938e7e5535b5ff24388876315a098027c3a57", + "sha256:ca9f3debf2262170d6f46571ce4d6ca1add60bb93b69c3a29dcb3d1a00a65c93" + ], + "index": "pypi", + "version": "==0.11.0" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "index": "pypi", + "version": "==2.22.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:64a7b71846db6423807e96820993fa12a03b89127d278290ca25c0b11ed7b4fb" + ], + "index": "pypi", + "version": "==1.3.13" + }, + "urllib3": { + "hashes": [ + "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", + "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + ], + "version": "==1.25.8" + } + }, + "develop": {} +} diff --git a/workers/README.md b/workers/README.md new file mode 100644 index 0000000..866a9eb --- /dev/null +++ b/workers/README.md @@ -0,0 +1,85 @@ +# Worker + +## Installation + +Ensure that pip and python is installed + +install pipenv for dependencies management. List of dependencies will be listed on Pipfile. Make sure the pipenv path is added to the system. +```sh +pip install --user pipenv +``` + +install Requests library for HTTP request +```sh +pipenv install requests +``` + +install psycopg2 for library to access PostgreSQL Database +```sh +pipenv install psycopg2 +``` + +install python-dotenv to add environment variables into the app +```sh +pipenv install python-dotenv +``` + +install SQLAlchemy for ORM database +```sh +pipenv install SQLAlchemy +``` + +## Authentication for Twitter +Create an account followed by application creation on Twitter Developer Account. More information can be found in https://developer.twitter.com/en/account/get-started +Get your application tokens at "Keys and Access Tokens" +- consumer key +- consumer secret key +- access token +- access token secret + +Bearer Token can be generated from get_bearer_token.py +Bearer Token is required to use the Request lib for Twitter API + +## Fetch Data from API +- Insert the Bearer Token into worker.py +- Insert params for the Twitter API +Info about the endpoint of Twitter API https://developer.twitter.com/en/docs/tweets/search/api-reference/get-search-tweets# + +## Environment variables set up +- Create a .env file and input the variables +- Call dotenv library and os to get the desired variables +variables description : +1. BEARER_TOKEN + Token that consist of all kinds of character. + OAuth 2.0 generated from Account Token and Account Token Secret from Twitter developer page, Once these two tokens are available, use get_bearer_token.py to generate Bearer Token for the account +2. ACC_TOKEN + Token that consist of all kinds of character. + This token can be generated after user create an account and a new aplication in the developer page. +3. ACC_TOKEN_SECRET + Token that consist of all kinds of character. + This token can be generated after user create an account and a new aplication in the developer page. +4. USERNAME + consist of string/int that the user created in PostgreSQL +5. PASSWORD + consist of string/int that the user created in PostgreSQL +6. DB_NAME + consist of string/int that the user created in PostgreSQL +7. DB_PORT and DB_HOST + default value in PostgreSQL are DB_PORT = 5432 , DB_HOST = localhost +8. DATABASE_URL_ADDON + an URL for ORM endpoint of PostgreSQL that consist of + USERNAME:PASSWORD@HOST:PORT/DATABASE_NAME + +## Connect to PostgreSQL server and queries +- start postgres server with psql -U +- insert all the params to psycopg2.connect +- use INSERT query to insert data each time it is fetched +- commit and close connection and its cursor + +## ORM with SQLAlchemy setup and usage +SQLAlchemy is an Object-Relation Mapper that interact with database with Python programming language +- set up the engine and connection with the URL (with params to Postgres) +- keep all database information into Metadata object +- query with Python languange. More info can be found here https://docs.sqlalchemy.org/en/13/core/tutorial.html#connecting + + diff --git a/workers/get_bearer_token.py b/workers/get_bearer_token.py new file mode 100644 index 0000000..ea18f5a --- /dev/null +++ b/workers/get_bearer_token.py @@ -0,0 +1,36 @@ +import base64 +import requests +import urllib.parse + +OAUTH2_TOKEN = 'https://api.twitter.com/oauth2/token' + + +def get_bearer_token(consumer_key, consumer_secret): + # enconde consumer key + consumer_key = urllib.parse.quote(consumer_key) + # encode consumer secret + consumer_secret = urllib.parse.quote(consumer_secret) + # create bearer token + bearer_token = consumer_key + ':' + consumer_secret + # base64 encode the token + base64_encoded_bearer_token = base64.b64encode(bearer_token.encode('utf-8')) + # set headers + headers = { + "Authorization": "Basic " + base64_encoded_bearer_token.decode('utf-8') + "", + "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8", + "Content-Length": "29"} + + response = requests.post(OAUTH2_TOKEN, headers=headers, data={'grant_type': 'client_credentials'}) + to_json = response.json() + print("token_type = %s\naccess_token = %s" % (to_json['token_type'], to_json['access_token'])) + + +def main(): + consumer_key = 'Enter your consumer key' + consumer_secret = 'Enter your consumer secret' + print("***** ***** ***** *****") + get_bearer_token(consumer_key, consumer_secret) + + +if __name__ == "__main__": + main() \ No newline at end of file