From 9d5091877278d476ebadb1d7388e3ce482fd01e9 Mon Sep 17 00:00:00 2001 From: Anton Nikolaev Date: Sat, 18 Jul 2020 23:24:53 +0700 Subject: [PATCH] Core fixes. Provide global variables. Update runner. Upgrade module-package system. (#14) * Clean init all the modules with the __init__.py file * Remove unused gitkeeps * Add base inits for script packages * Update main file * Set __file__ variable at the module level * Add inits for the modules * Modify all the modules * Delete one more .gitkeep * Add WIP test for user_greeting * Update test module for user_greeting * Fix module runner a bit * Add relative imports to the modules * Add run tests script * Update README.md * Fix module-package runner * Update requirements * Format everything with Black * Handle module and execution errors with runner --- README.md | 69 +++++++++++ main.py | 47 ++++---- requirements.txt | 13 +++ run_tests.sh | 3 + src/{core/exceptions/.gitkeep => __init__.py} | 0 src/core/{handlers/.gitkeep => __init__.py} | 0 .../.gitkeep => core/base/__init__.py} | 0 .../osint/.gitkeep => core/case/__init__.py} | 0 .../.gitkeep => core/exceptions/__init__.py} | 0 src/core/handlers/__init__.py | 0 src/core/runner/__init__.py | 0 src/core/runner/runner.py | 37 ++++-- src/core/utils/__init__.py | 0 src/core/utils/module.py | 16 +++ src/scripts/__init__.py | 0 src/scripts/convert/__init__.py | 0 src/scripts/osint/__init__.py | 0 src/scripts/osint/check_nickname/__init__.py | 5 + src/scripts/osint/check_nickname/__main__.py | 100 +--------------- src/scripts/osint/check_nickname/module.py | 100 ++++++++++++++++ src/scripts/osint/email_verifier/__init__.py | 5 + src/scripts/osint/email_verifier/__main__.py | 26 ++--- src/scripts/osint/email_verifier/module.py | 22 ++++ src/scripts/osint/region_check/__init__.py | 5 + src/scripts/osint/region_check/__main__.py | 52 +-------- src/scripts/osint/region_check/module.py | 50 ++++++++ src/scripts/other/__init__.py | 0 src/scripts/other/simple_example/__init__.py | 5 + src/scripts/other/simple_example/__main__.py | 31 +---- src/scripts/other/simple_example/module.py | 28 +++++ src/scripts/other/user_greeting/__init__.py | 5 + src/scripts/other/user_greeting/__main__.py | 43 +------ src/scripts/other/user_greeting/module.py | 35 ++++++ .../other/user_greeting/test_module.py | 50 ++++++++ src/scripts/recon/__init__.py | 0 src/scripts/recon/allowed_methods/__init__.py | 5 + src/scripts/recon/allowed_methods/__main__.py | 110 ++---------------- src/scripts/recon/allowed_methods/module.py | 106 +++++++++++++++++ src/scripts/recon/get_host_status/__init__.py | 5 + src/scripts/recon/get_host_status/__main__.py | 29 ++--- src/scripts/recon/get_host_status/module.py | 25 ++++ .../recon/shodan_favicon_hash/__init__.py | 5 + .../recon/shodan_favicon_hash/__main__.py | 48 ++------ .../recon/shodan_favicon_hash/module.py | 43 +++++++ 44 files changed, 712 insertions(+), 411 deletions(-) create mode 100755 run_tests.sh rename src/{core/exceptions/.gitkeep => __init__.py} (100%) rename src/core/{handlers/.gitkeep => __init__.py} (100%) rename src/{scripts/convert/.gitkeep => core/base/__init__.py} (100%) rename src/{scripts/osint/.gitkeep => core/case/__init__.py} (100%) rename src/{scripts/recon/.gitkeep => core/exceptions/__init__.py} (100%) create mode 100644 src/core/handlers/__init__.py create mode 100644 src/core/runner/__init__.py create mode 100644 src/core/utils/__init__.py create mode 100644 src/core/utils/module.py create mode 100644 src/scripts/__init__.py create mode 100644 src/scripts/convert/__init__.py create mode 100644 src/scripts/osint/__init__.py create mode 100644 src/scripts/osint/check_nickname/__init__.py create mode 100644 src/scripts/osint/check_nickname/module.py create mode 100644 src/scripts/osint/email_verifier/__init__.py create mode 100644 src/scripts/osint/email_verifier/module.py create mode 100644 src/scripts/osint/region_check/__init__.py create mode 100644 src/scripts/osint/region_check/module.py create mode 100644 src/scripts/other/__init__.py create mode 100644 src/scripts/other/simple_example/__init__.py create mode 100644 src/scripts/other/simple_example/module.py create mode 100644 src/scripts/other/user_greeting/__init__.py create mode 100644 src/scripts/other/user_greeting/module.py create mode 100644 src/scripts/other/user_greeting/test_module.py create mode 100644 src/scripts/recon/__init__.py create mode 100644 src/scripts/recon/allowed_methods/__init__.py create mode 100644 src/scripts/recon/allowed_methods/module.py create mode 100644 src/scripts/recon/get_host_status/__init__.py create mode 100644 src/scripts/recon/get_host_status/module.py create mode 100644 src/scripts/recon/shodan_favicon_hash/__init__.py create mode 100644 src/scripts/recon/shodan_favicon_hash/module.py diff --git a/README.md b/README.md index a2f8199..46058f4 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,79 @@ virtualenv -p python3 venv (or python3 -m venv venv) pip3 install -r requirements.txt +``` + +## Running as a framework +_First of all: provide some arguments in the `main.py` file to collect information based on your data (WIP now, will be improved later)_ + +To run the framework: +```bash python3 main.py ``` +To run the tests: +```bash +chmod +x run_tests.sh +./run_tests.sh +``` +_or you can run them like this, of course:_ +```bash +python3 -m unittest discover -v +``` +## Running as a separated module +Basic: +```python3 +python3 -m src.scripts.. any_arguments_here +``` +Example command: +```bash +python3 -m src.scripts.other.user_greeting JohnDoe +``` +Example output: +``` +{'message': "Successfully finished! (args: (), kwargs: {'username': " + "'johndoe'})", + 'result': 'Hello, JohnDoe!', + 'status': 'success'} + +``` ## Create your own script Use the following structure: +1. Create your own module directory in the following way: +``` +/src/scripts/// +``` +2. Provide the following structure of your script directory: +``` +your_script_name +├── __init__.py - use this module to set the default parent directory (you can copy this file from any other script) +├── __main__.py - use this module to provide some basic interface to use your script as a module (the same as if __name__ == "__main__") +├── module.py - use this module to describe the basic logic of your module (you can import it in the __main__.py to provide interface) +└── test_module.py - use this module for unittest tests +``` +3. Create the `__init__.py` file. An example of the `__init__.py` boilerplate structure can be seen below: +```python3 +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) + +``` +4. Create the `__main__.py` file. An example of the `__main__.py` boilerplate structure can be seen below: +```python3 +#!/usr/bin/env python3 + +from pprint import pprint +from sys import argv + +from src.core.utils.module import run_module +from .module import Runner + +result = run_module(Runner, args=argv, arg_name="username", arg_default="johndoe") +pprint(result) +``` +5. Create the module itself. An example of the basic `module.py` file can be seen below: ```python3 #!/usr/bin/env python3 @@ -60,3 +128,4 @@ class Runner(OsintRunner): argument = kwargs.get("my_argument", "Arguments were not provided!") return ScriptResponse.success(message=f"Script finished with argument {argument}") ``` +6. For `test_module.py` you can use any required tests (as you wish). A test case for your module is required to keep the project clean. diff --git a/main.py b/main.py index 4cee228..cac9217 100644 --- a/main.py +++ b/main.py @@ -13,26 +13,33 @@ from src.core.case.osint import OsintCase from src.core.case.recon import ReconCase from src.core.case.base import BaseCase +from src.core.utils.log import Logger + + +logger = Logger.get_logger(name="osint-framework") + + +def run_case(case_class: type, *args, **kwargs): + """ + Define and smoke run the BaseCase + :param case_class: original class of the case + :param args: some args + :param kwargs: some kwargs + :return: result of the execution + """ + logger.info(f"start {case_class.__name__} case processing") + case = case_class() + case.process(*args, **kwargs) + return case.get_results() if __name__ == "__main__": - # Will return 2 results from "other" category scripts - other_case = BaseCase() - other_case.process( - username="johndoe", email="johndoe@gmail.com", fullname="John Doe", - ) - other_case_results = other_case.get_results() - - # Will return 1 result from "recon" category scripts - recon_case = ReconCase() - recon_case.process(url="https://facebook.com") - recon_case_results = recon_case.get_results() - - # Will return nothing (no scripts for now, sorry!) - osint_case = OsintCase() - osint_case.process(username="any_value_here") - osint_case_results = osint_case.get_results() - - # Print out all the results - for result in other_case_results, recon_case_results, osint_case_results: - pprint(result) + # fmt: off + base_case = run_case(case_class=BaseCase, username="johndoe", email="johndoe@gmail.com", fullname="John Doe") + osint_case = run_case(case_class=OsintCase, username="johndoe", email="johndoe@gmail.com", fullname="John Doe") + recon_case = run_case(case_class=ReconCase, url="https://facebook.com") + + pprint(base_case) + pprint(osint_case) + pprint(recon_case) + # fmt: on diff --git a/requirements.txt b/requirements.txt index 4d72486..62a27e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,18 @@ +aiodns==2.0.0 +aiohttp==3.6.2 +aiosmtpd==1.2 +async-timeout==3.0.1 +atpublic==1.0 +attrs==19.3.0 certifi==2020.6.20 +cffi==1.14.0 chardet==3.0.4 idna==2.10 +mmh3==2.5.1 +multidict==4.7.6 +pycares==3.1.1 +pycparser==2.20 requests==2.24.0 urllib3==1.25.9 +verify-email==2.4.1 +yarl==1.4.2 diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..72ab9c5 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +python3 -m unittest discover -v diff --git a/src/core/exceptions/.gitkeep b/src/__init__.py similarity index 100% rename from src/core/exceptions/.gitkeep rename to src/__init__.py diff --git a/src/core/handlers/.gitkeep b/src/core/__init__.py similarity index 100% rename from src/core/handlers/.gitkeep rename to src/core/__init__.py diff --git a/src/scripts/convert/.gitkeep b/src/core/base/__init__.py similarity index 100% rename from src/scripts/convert/.gitkeep rename to src/core/base/__init__.py diff --git a/src/scripts/osint/.gitkeep b/src/core/case/__init__.py similarity index 100% rename from src/scripts/osint/.gitkeep rename to src/core/case/__init__.py diff --git a/src/scripts/recon/.gitkeep b/src/core/exceptions/__init__.py similarity index 100% rename from src/scripts/recon/.gitkeep rename to src/core/exceptions/__init__.py diff --git a/src/core/handlers/__init__.py b/src/core/handlers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/runner/__init__.py b/src/core/runner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/runner/runner.py b/src/core/runner/runner.py index f21539a..4e6ddd8 100644 --- a/src/core/runner/runner.py +++ b/src/core/runner/runner.py @@ -32,14 +32,14 @@ def __init__(self): self.scripts = {} self.results = {} + @staticmethod def exec_script( - self, path: str or Path, script_class: str = "Runner", function: str = "run", args: list or None = None, kwargs: dict or None = None, - ) -> ScriptResponse: + ) -> ScriptResponse or dict: """ Load and exec python script :param path: name of the script to load @@ -55,13 +55,34 @@ def exec_script( kwargs = {} loader = SourceFileLoader(fullname=script_class, path=str(path)) module = ModuleType(name=loader.name) - loader.exec_module(module) - class_instance = getattr(module, script_class)(logger=path.parent.stem) + result = {"script": Path(path).parent.stem} + + # Check if don't forget to install all the dependencies, and that module can + # be successfully loaded. + + # fmt: off + try: + loader.exec_module(module) + except Exception as unexp_err: + result.update(ScriptResponse.error(message=f"Unexpected module error: {str(unexp_err)}")) + return result + # fmt: on + + # Module successfully loaded. We can set some module-scope variables that + # we missed. + module.__file__ = path + + # Execute the runner and check if something goes wrong. + + # fmt: off try: - result = getattr(class_instance, function)(*args, **kwargs) + class_instance = getattr(module, script_class)(logger=path.parent.stem) + result.update(getattr(class_instance, function)(*args, **kwargs)) except Exception as unexp_err: - result = ScriptResponse.error(message=str(unexp_err)) - result.update({"script": Path(path).parent.stem}) + result.update(ScriptResponse.error(message=f"Unexpected execution error: {str(unexp_err)}")) + # fmt: on + + # In any possible case, return result from the module or ScriptResponse.error + script name return result def get_scripts(self) -> dict: @@ -76,7 +97,7 @@ def get_scripts(self) -> dict: ScriptRunnerPaths.CONVERT, ]: self.scripts.update({directory.stem: list()}) - for file in directory.glob("*/__main__.py"): + for file in directory.glob("*/module.py"): self.scripts[directory.stem].append(file) return self.scripts diff --git a/src/core/utils/__init__.py b/src/core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/utils/module.py b/src/core/utils/module.py new file mode 100644 index 0000000..3ca78b7 --- /dev/null +++ b/src/core/utils/module.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + + +def run_module( + runner: type, args: list, arg_name: str, arg_default: str or None = None +) -> dict: + """ + Use module as a 'python3 -m ...' module + :param runner: runner object + :param args: list of args from CLI + :param arg_name: name of arg to use + :param arg_default: default arg value + :return: results + """ + runner = runner() + return runner.run(**{arg_name: args[1] if len(args) >= 2 else arg_default}) diff --git a/src/scripts/__init__.py b/src/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/scripts/convert/__init__.py b/src/scripts/convert/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/scripts/osint/__init__.py b/src/scripts/osint/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/scripts/osint/check_nickname/__init__.py b/src/scripts/osint/check_nickname/__init__.py new file mode 100644 index 0000000..acdd053 --- /dev/null +++ b/src/scripts/osint/check_nickname/__init__.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) diff --git a/src/scripts/osint/check_nickname/__main__.py b/src/scripts/osint/check_nickname/__main__.py index 3a1af46..0262441 100644 --- a/src/scripts/osint/check_nickname/__main__.py +++ b/src/scripts/osint/check_nickname/__main__.py @@ -1,98 +1,10 @@ #!/usr/bin/env python3 -import asyncio -from pathlib import Path +from pprint import pprint +from sys import argv -import aiohttp -import requests +from src.core.utils.module import run_module +from .module import Runner -from src.core.base.osint import OsintRunner -from src.core.utils.response import ScriptResponse - - -class Defaults: - NETWORKS_LIST = "social_networks.txt" - - -class Networks: - def __init__(self): - with open(Path("./src/scripts/osint/check_nickname/data/social_networks.txt")) as file: - self.net = file.read().splitlines() - - -def check_nickname_sync(nickname: str) -> list: - """ - checks nicknames from social networks(sync) - :param nickname: just nickname :) - :return: list with links to user from social - networks which have this nickname - """ - ans = [] - social = Networks().net - for site in social: - try: - url = "https://{site}{nickname}".format(site=site, nickname=nickname) - response = requests.get(url) - if response.status_code == 200: - ans.append(url) - except: - pass - return ans - - -async def check_nickname_async(nickname: str, social) -> list: - """ - checks nicknames from social networks(async) - :param nickname: just nickname :) - :param social: social - :return: list with links to user from social - networks which have this nickname - """ - ans = [] - async with aiohttp.ClientSession() as session: - while not social.empty(): - url = await social.get() - try: - async with session.get(url) as response: - if response.status == 200: - ans.append(url) - except: - pass - return ans - - -class Runner(OsintRunner): - def __init__(self, logger: str = __name__): - super().__init__(logger=logger) - - @staticmethod - async def __run(*args, **kwargs): - try: - username = kwargs.get("username") - social = asyncio.Queue() - for site in Networks().net: - await social.put( - "https://{site}{username}".format(site=site, username=username) - ) - temp_result = await asyncio.gather( - *[ - asyncio.create_task(check_nickname_async(username, social)) - for _ in range(10) - ] - ) - result = {username: []} - for sub_massive in temp_result: - for site in sub_massive: - result[username].append(site) - return ScriptResponse.success( - result=result, - message="Found {count} user accounts".format( - count=len(result[username]) - ), - ) - except Exception as err: - return ScriptResponse.error(message=str(err)) - - def run(self, *args, **kwargs): - username = kwargs.get("username") - return asyncio.run(self.__run(username=username)) +result = run_module(Runner, args=argv, arg_name="username", arg_default="johndoe") +pprint(result) diff --git a/src/scripts/osint/check_nickname/module.py b/src/scripts/osint/check_nickname/module.py new file mode 100644 index 0000000..bd15deb --- /dev/null +++ b/src/scripts/osint/check_nickname/module.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +import asyncio +from pathlib import Path + +import aiohttp +import requests + +from src.core.base.osint import OsintRunner +from src.core.utils.response import ScriptResponse + + +class Defaults: + NETWORKS_LIST = "social_networks.txt" + + +class Networks: + def __init__(self): + with open( + Path(__file__).parent.joinpath(f"data/{Defaults.NETWORKS_LIST}") + ) as file: + self.net = file.read().splitlines() + + +def check_nickname_sync(nickname: str) -> list: + """ + checks nicknames from social networks(sync) + :param nickname: just nickname :) + :return: list with links to user from social + networks which have this nickname + """ + ans = [] + social = Networks().net + for site in social: + try: + url = "https://{site}{nickname}".format(site=site, nickname=nickname) + response = requests.get(url) + if response.status_code == 200: + ans.append(url) + except: + pass + return ans + + +async def check_nickname_async(nickname: str, social) -> list: + """ + checks nicknames from social networks(async) + :param nickname: just nickname :) + :param social: social + :return: list with links to user from social + networks which have this nickname + """ + ans = [] + async with aiohttp.ClientSession() as session: + while not social.empty(): + url = await social.get() + try: + async with session.get(url) as response: + if response.status == 200: + ans.append(url) + except: + pass + return ans + + +class Runner(OsintRunner): + def __init__(self, logger: str = __name__): + super().__init__(logger=logger) + + @staticmethod + async def __run(*args, **kwargs): + try: + username = kwargs.get("username") + social = asyncio.Queue() + for site in Networks().net: + await social.put( + "https://{site}{username}".format(site=site, username=username) + ) + temp_result = await asyncio.gather( + *[ + asyncio.create_task(check_nickname_async(username, social)) + for _ in range(10) + ] + ) + result = {username: []} + for sub_massive in temp_result: + for site in sub_massive: + result[username].append(site) + return ScriptResponse.success( + result=result, + message="Found {count} user accounts".format( + count=len(result[username]) + ), + ) + except Exception as err: + return ScriptResponse.error(message=str(err)) + + def run(self, *args, **kwargs): + username = kwargs.get("username") + return asyncio.run(self.__run(username=username)) diff --git a/src/scripts/osint/email_verifier/__init__.py b/src/scripts/osint/email_verifier/__init__.py new file mode 100644 index 0000000..acdd053 --- /dev/null +++ b/src/scripts/osint/email_verifier/__init__.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) diff --git a/src/scripts/osint/email_verifier/__main__.py b/src/scripts/osint/email_verifier/__main__.py index 4108dd0..b58e333 100644 --- a/src/scripts/osint/email_verifier/__main__.py +++ b/src/scripts/osint/email_verifier/__main__.py @@ -1,22 +1,12 @@ #!/usr/bin/env python3 -from verify_email import verify_email +from pprint import pprint +from sys import argv -from src.core.base.osint import OsintRunner, PossibleKeys -from src.core.utils.response import ScriptResponse -from src.core.utils.validators import validate_kwargs +from src.core.utils.module import run_module +from .module import Runner - -class Runner(OsintRunner): - def __init__(self, logger: str = __name__): - super(Runner, self).__init__(logger) - - @validate_kwargs(PossibleKeys.KEYS) - def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error: - email = kwargs.get("email") - result = verify_email(email) - if not result: - return ScriptResponse.success( - result=result, message=f"Sorry, email {email} does not exist" - ) - return ScriptResponse.success(result=result, message=f"Email {email} exists") +result = run_module( + Runner, args=argv, arg_name="email", arg_default="johndoe@gmail.com" +) +pprint(result) diff --git a/src/scripts/osint/email_verifier/module.py b/src/scripts/osint/email_verifier/module.py new file mode 100644 index 0000000..4108dd0 --- /dev/null +++ b/src/scripts/osint/email_verifier/module.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +from verify_email import verify_email + +from src.core.base.osint import OsintRunner, PossibleKeys +from src.core.utils.response import ScriptResponse +from src.core.utils.validators import validate_kwargs + + +class Runner(OsintRunner): + def __init__(self, logger: str = __name__): + super(Runner, self).__init__(logger) + + @validate_kwargs(PossibleKeys.KEYS) + def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error: + email = kwargs.get("email") + result = verify_email(email) + if not result: + return ScriptResponse.success( + result=result, message=f"Sorry, email {email} does not exist" + ) + return ScriptResponse.success(result=result, message=f"Email {email} exists") diff --git a/src/scripts/osint/region_check/__init__.py b/src/scripts/osint/region_check/__init__.py new file mode 100644 index 0000000..acdd053 --- /dev/null +++ b/src/scripts/osint/region_check/__init__.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) diff --git a/src/scripts/osint/region_check/__main__.py b/src/scripts/osint/region_check/__main__.py index 7e047b8..469294f 100644 --- a/src/scripts/osint/region_check/__main__.py +++ b/src/scripts/osint/region_check/__main__.py @@ -1,50 +1,10 @@ #!/usr/bin/env python3 -import requests +from pprint import pprint +from sys import argv -from src.core.base.osint import OsintRunner, PossibleKeys -from src.core.utils.response import ScriptResponse -from src.core.utils.validators import validate_kwargs +from src.core.utils.module import run_module +from .module import Runner - -class Runner(OsintRunner): - def __init__(self, logger: str = __name__): - super(Runner, self).__init__(logger) - - @staticmethod - def __phone_to_region(phone: str = "Unknown") -> str: - """ - Region recognition using the phone number - :param phone: phone number to check - :return: information about the region and the operator - """ - response = requests.get( - "https://eduscan.net/help/phone_ajax.php", params={"num": phone} - ) - return response.text - - @validate_kwargs(PossibleKeys.KEYS) - def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error: - """ - Main runner function for the script - :param args: args from core runner - :param kwargs: kwargs from core runner - :return: ScriptResponse message - """ - try: - result = self.__phone_to_region(phone=kwargs.get("phone")) - except Exception: - return ScriptResponse.error(result=None, message="Something went wrong!") - result = result.split("~") - if result[1] != "0": - return ScriptResponse.success(result=None, message="Sorry, no such number!") - return ScriptResponse.success( - result=(result[-1], result[-2]), - message=f"Found region: {result[-1]} | found operator: {result[-2]} | for phone number {result[0]}", - ) - - -if __name__ == "__main__": - script_module = Runner() - script_result = script_module.run() - print(script_result) +result = run_module(Runner, args=argv, arg_name="phone", arg_default="89138811111") +pprint(result) diff --git a/src/scripts/osint/region_check/module.py b/src/scripts/osint/region_check/module.py new file mode 100644 index 0000000..7e047b8 --- /dev/null +++ b/src/scripts/osint/region_check/module.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import requests + +from src.core.base.osint import OsintRunner, PossibleKeys +from src.core.utils.response import ScriptResponse +from src.core.utils.validators import validate_kwargs + + +class Runner(OsintRunner): + def __init__(self, logger: str = __name__): + super(Runner, self).__init__(logger) + + @staticmethod + def __phone_to_region(phone: str = "Unknown") -> str: + """ + Region recognition using the phone number + :param phone: phone number to check + :return: information about the region and the operator + """ + response = requests.get( + "https://eduscan.net/help/phone_ajax.php", params={"num": phone} + ) + return response.text + + @validate_kwargs(PossibleKeys.KEYS) + def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error: + """ + Main runner function for the script + :param args: args from core runner + :param kwargs: kwargs from core runner + :return: ScriptResponse message + """ + try: + result = self.__phone_to_region(phone=kwargs.get("phone")) + except Exception: + return ScriptResponse.error(result=None, message="Something went wrong!") + result = result.split("~") + if result[1] != "0": + return ScriptResponse.success(result=None, message="Sorry, no such number!") + return ScriptResponse.success( + result=(result[-1], result[-2]), + message=f"Found region: {result[-1]} | found operator: {result[-2]} | for phone number {result[0]}", + ) + + +if __name__ == "__main__": + script_module = Runner() + script_result = script_module.run() + print(script_result) diff --git a/src/scripts/other/__init__.py b/src/scripts/other/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/scripts/other/simple_example/__init__.py b/src/scripts/other/simple_example/__init__.py new file mode 100644 index 0000000..acdd053 --- /dev/null +++ b/src/scripts/other/simple_example/__init__.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) diff --git a/src/scripts/other/simple_example/__main__.py b/src/scripts/other/simple_example/__main__.py index 6bbdd9c..0262441 100644 --- a/src/scripts/other/simple_example/__main__.py +++ b/src/scripts/other/simple_example/__main__.py @@ -1,29 +1,10 @@ #!/usr/bin/env python3 -from src.core.base.osint import BaseRunner, PossibleKeys -from src.core.utils.response import ScriptResponse -from src.core.utils.validators import validate_kwargs +from pprint import pprint +from sys import argv +from src.core.utils.module import run_module +from .module import Runner -class Runner(BaseRunner): - """ - Basic example - """ - def __init__(self, logger: str = __name__): - """ - Re-init base class instance - :param logger: logger to use - """ - super(Runner, self).__init__(logger) - - @validate_kwargs(PossibleKeys.KEYS) - def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error: - """ - Return basic success response - :param args: args - :param kwargs: kwargs - :return: ScriptResponse message - """ - return ScriptResponse.success( - message="Script finished" - ) +result = run_module(Runner, args=argv, arg_name="username", arg_default="johndoe") +pprint(result) diff --git a/src/scripts/other/simple_example/module.py b/src/scripts/other/simple_example/module.py new file mode 100644 index 0000000..2dba38a --- /dev/null +++ b/src/scripts/other/simple_example/module.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +from src.core.base.osint import BaseRunner, PossibleKeys +from src.core.utils.response import ScriptResponse +from src.core.utils.validators import validate_kwargs + + +class Runner(BaseRunner): + """ + Basic example + """ + + def __init__(self, logger: str = __name__): + """ + Re-init base class instance + :param logger: logger to use + """ + super(Runner, self).__init__(logger) + + @validate_kwargs(PossibleKeys.KEYS) + def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error: + """ + Return basic success response + :param args: args + :param kwargs: kwargs + :return: ScriptResponse message + """ + return ScriptResponse.success(message="Script finished") diff --git a/src/scripts/other/user_greeting/__init__.py b/src/scripts/other/user_greeting/__init__.py new file mode 100644 index 0000000..acdd053 --- /dev/null +++ b/src/scripts/other/user_greeting/__init__.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) diff --git a/src/scripts/other/user_greeting/__main__.py b/src/scripts/other/user_greeting/__main__.py index 27578cd..0262441 100644 --- a/src/scripts/other/user_greeting/__main__.py +++ b/src/scripts/other/user_greeting/__main__.py @@ -1,41 +1,10 @@ #!/usr/bin/env python3 -from src.core.base.osint import OsintRunner, PossibleKeys -from src.core.utils.response import ScriptResponse -from src.core.utils.validators import validate_kwargs +from pprint import pprint +from sys import argv +from src.core.utils.module import run_module +from .module import Runner -class Runner(OsintRunner): - def __init__(self, logger: str = __name__): - super(Runner, self).__init__(logger) - - @staticmethod - def user_greeting(user: str = "Unknown") -> str: - """ - Simple example function for user greeting - :param user: name of the user to check - :return: greeting message - """ - return f"Hello, {user}!" - - @validate_kwargs(PossibleKeys.KEYS) - def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error: - """ - Main runner function for the script - :param args: args from core runner - :param kwargs: kwargs from core runner - :return: ScriptResponse message - """ - result = self.user_greeting(user=kwargs.get("username")) - if not result: - return ScriptResponse.success(message="Sorry, no greetings for this user!") - return ScriptResponse.success( - result=result, - message=f"Successfully finished! (args: {args}, kwargs: {kwargs})", - ) - - -if __name__ == "__main__": - script_module = Runner() - script_result = script_module.run() - print(script_result) +result = run_module(Runner, args=argv, arg_name="username", arg_default="johndoe") +pprint(result) diff --git a/src/scripts/other/user_greeting/module.py b/src/scripts/other/user_greeting/module.py new file mode 100644 index 0000000..1c18f6f --- /dev/null +++ b/src/scripts/other/user_greeting/module.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +from src.core.base.osint import OsintRunner, PossibleKeys +from src.core.utils.response import ScriptResponse +from src.core.utils.validators import validate_kwargs + + +class Runner(OsintRunner): + def __init__(self, logger: str = __name__): + super(Runner, self).__init__(logger) + + @staticmethod + def user_greeting(user: str = "Unknown") -> str: + """ + Simple example function for user greeting + :param user: name of the user to check + :return: greeting message + """ + return f"Hello, {user}!" + + @validate_kwargs(PossibleKeys.KEYS) + def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error: + """ + Main runner function for the script + :param args: args from core runner + :param kwargs: kwargs from core runner + :return: ScriptResponse message + """ + result = self.user_greeting(user=kwargs.get("username")) + if not result: + return ScriptResponse.success(message="Sorry, no greetings for this user!") + return ScriptResponse.success( + result=result, + message=f"Successfully finished! (args: {args}, kwargs: {kwargs})", + ) diff --git a/src/scripts/other/user_greeting/test_module.py b/src/scripts/other/user_greeting/test_module.py new file mode 100644 index 0000000..03f2702 --- /dev/null +++ b/src/scripts/other/user_greeting/test_module.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +from unittest import TestCase +from .module import Runner + + +class UserGreetingTest(TestCase): + def setUp(self): + """ + Setup something before each test function + :return: None + """ + self.runner = Runner() + + def test_create_runner(self): + """ + Test creation of the class instance + :return: None + """ + self.assertIsNotNone(self.runner) + self.assertIsInstance(self.runner, Runner) + + def test_pass_argument(self): + """ + Test passing an arguments + :return: None + """ + result = self.runner.run(username="john") + self.assertEqual(result.get("result"), "Hello, john!") + self.assertEqual(result.get("status"), "success") + + def test_no_arguments(self): + """ + Test passing no arguments + :return: None + """ + result = self.runner.run() + self.assertEqual(result.get("result"), "Hello, None!") + self.assertEqual(result.get("status"), "success") + + def test_wrong_argument(self): + """ + Test case when we can't pass the validator (exception) + :return: None + """ + + def exception_func(): + self.runner.run(wrong_argument="john") + + self.assertRaises(KeyError, exception_func) diff --git a/src/scripts/recon/__init__.py b/src/scripts/recon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/scripts/recon/allowed_methods/__init__.py b/src/scripts/recon/allowed_methods/__init__.py new file mode 100644 index 0000000..acdd053 --- /dev/null +++ b/src/scripts/recon/allowed_methods/__init__.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) diff --git a/src/scripts/recon/allowed_methods/__main__.py b/src/scripts/recon/allowed_methods/__main__.py index 7a1df59..dbd35b2 100644 --- a/src/scripts/recon/allowed_methods/__main__.py +++ b/src/scripts/recon/allowed_methods/__main__.py @@ -1,106 +1,12 @@ #!/usr/bin/env python3 -from requests import request -from requests.exceptions import RequestException +from pprint import pprint +from sys import argv -from src.core.base.recon import ReconRunner, PossibleKeys -from src.core.utils.response import ScriptResponse -from src.core.utils.validators import validate_kwargs +from src.core.utils.module import run_module +from .module import Runner -from string import ascii_uppercase -from random import seed, choice, randint - - -class Defaults: - # List of the Request Methods Recognized by Apache - METHODS: list = [ - "GET", - "PUT", - "POST", - "DELETE", - "CONNECT", - "OPTIONS", - "TRACE", - "PATCH", - "PROPFIND", - "PROPPATCH", - "MKCOL", - "COPY", - "MOVE", - "LOCK", - "UNLOCK", - "VERSION_CONTROL", - "CHECKOUT", - "UNCHECKOUT", - "CHECKIN", - "UPDATE", - "LABEL", - "REPORT", - "MKWORKSPACE", - "MKACTIVITY", - "BASELINE_CONTROL", - "MERGE", - "INVALID", - ] - - -def get_random_method(): - seed() - letters = ascii_uppercase - length = randint(6, 10) - method = "".join(choice(letters) for i in range(length)) - return method - - -class Runner(ReconRunner): - def __init__(self, logger: str = __name__): - super(Runner, self).__init__(logger) - - @validate_kwargs(PossibleKeys.KEYS) - def run(self, *args, **kwargs): - """ - Returns list of allowed methods. E.g. ['GET', 'POST']. - Also checks if method is forbidden or somehow filtered. - Needs url to run. - :param args: args from core runner - :param kwargs: kwargs from core runner - :return: ScriptResponse message - """ - allowed_methods = [] - filtered_methods = [] - forbidden_methods = [] - url = kwargs.get("url") - - # Copy original methods list to avoid class variable modifications - methods = list(Defaults.METHODS) - - # Append random method to check if server is not faking. - methods.append(get_random_method()) - - for method in methods: - try: - status = request(method, url).status_code - method_result = {"method": method, "status": status} - - # 2xx - success - # 405 - method not allowed - if 200 <= status < 300: - allowed_methods.append(method_result) - elif status == 405: - forbidden_methods.append(method_result) - else: - filtered_methods.append(method_result) - except RequestException: - pass - - return ScriptResponse.success( - result={ - "allowed": allowed_methods, - "forbidden": forbidden_methods, - "filtered": filtered_methods, - }, - message=f"URL: {url} - " - f"allowed: {len(allowed_methods)}, " - f"forbidden: {len(forbidden_methods)}, " - f"filtered: {len(filtered_methods)}", - ) +result = run_module( + Runner, args=argv, arg_name="url", arg_default="https://www.intel.com/" +) +pprint(result) diff --git a/src/scripts/recon/allowed_methods/module.py b/src/scripts/recon/allowed_methods/module.py new file mode 100644 index 0000000..dd52c76 --- /dev/null +++ b/src/scripts/recon/allowed_methods/module.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +from requests import request +from requests.exceptions import RequestException + +from src.core.base.recon import ReconRunner, PossibleKeys +from src.core.utils.response import ScriptResponse +from src.core.utils.validators import validate_kwargs + +from string import ascii_uppercase +from random import seed, choice, randint + + +class Defaults: + # List of the Request Methods Recognized by Apache + METHODS: list = [ + "GET", + "PUT", + "POST", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PROPFIND", + "PROPPATCH", + "MKCOL", + "COPY", + "MOVE", + "LOCK", + "UNLOCK", + "VERSION_CONTROL", + "CHECKOUT", + "UNCHECKOUT", + "CHECKIN", + "UPDATE", + "LABEL", + "REPORT", + "MKWORKSPACE", + "MKACTIVITY", + "BASELINE_CONTROL", + "MERGE", + "INVALID", + ] + + +def get_random_method(): + seed() + letters = ascii_uppercase + length = randint(6, 10) + method = "".join(choice(letters) for i in range(length)) + return method + + +class Runner(ReconRunner): + def __init__(self, logger: str = __name__): + super(Runner, self).__init__(logger) + + @validate_kwargs(PossibleKeys.KEYS) + def run(self, *args, **kwargs): + """ + Returns list of allowed methods. E.g. ['GET', 'POST']. + Also checks if method is forbidden or somehow filtered. + Needs url to run. + :param args: args from core runner + :param kwargs: kwargs from core runner + :return: ScriptResponse message + """ + allowed_methods = [] + filtered_methods = [] + forbidden_methods = [] + url = kwargs.get("url") + + # Copy original methods list to avoid class variable modifications + methods = list(Defaults.METHODS) + + # Append random method to check if server is not faking. + methods.append(get_random_method()) + + for method in methods: + try: + status = request(method, url).status_code + method_result = {"method": method, "status": status} + + # 2xx - success + # 405 - method not allowed + if 200 <= status < 300: + allowed_methods.append(method_result) + elif status == 405: + forbidden_methods.append(method_result) + else: + filtered_methods.append(method_result) + except RequestException: + pass + + return ScriptResponse.success( + result={ + "allowed": allowed_methods, + "forbidden": forbidden_methods, + "filtered": filtered_methods, + }, + message=f"URL: {url} - " + f"allowed: {len(allowed_methods)}, " + f"forbidden: {len(forbidden_methods)}, " + f"filtered: {len(filtered_methods)}", + ) diff --git a/src/scripts/recon/get_host_status/__init__.py b/src/scripts/recon/get_host_status/__init__.py new file mode 100644 index 0000000..acdd053 --- /dev/null +++ b/src/scripts/recon/get_host_status/__init__.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) diff --git a/src/scripts/recon/get_host_status/__main__.py b/src/scripts/recon/get_host_status/__main__.py index 0026674..dbd35b2 100644 --- a/src/scripts/recon/get_host_status/__main__.py +++ b/src/scripts/recon/get_host_status/__main__.py @@ -1,25 +1,12 @@ #!/usr/bin/env python3 -from src.core.base.recon import ReconRunner, PossibleKeys -from src.core.utils.response import ScriptResponse -from src.core.utils.validators import validate_kwargs -from requests import get +from pprint import pprint +from sys import argv +from src.core.utils.module import run_module +from .module import Runner -class Runner(ReconRunner): - def __init__(self, logger: str = __name__): - super(Runner, self).__init__(logger) - - @validate_kwargs(PossibleKeys.KEYS) - def run(self, *args, **kwargs): - """ - Returns HTTP response status code - :param args: args from core runner - :param kwargs: kwargs from core runner - :return: ScriptResponse message - """ - url = kwargs.get("url") - status = get(url=url).status_code - return ScriptResponse.success( - result=status, message=f"Got HTTP response status from {url}" - ) +result = run_module( + Runner, args=argv, arg_name="url", arg_default="https://www.intel.com/" +) +pprint(result) diff --git a/src/scripts/recon/get_host_status/module.py b/src/scripts/recon/get_host_status/module.py new file mode 100644 index 0000000..0026674 --- /dev/null +++ b/src/scripts/recon/get_host_status/module.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +from src.core.base.recon import ReconRunner, PossibleKeys +from src.core.utils.response import ScriptResponse +from src.core.utils.validators import validate_kwargs +from requests import get + + +class Runner(ReconRunner): + def __init__(self, logger: str = __name__): + super(Runner, self).__init__(logger) + + @validate_kwargs(PossibleKeys.KEYS) + def run(self, *args, **kwargs): + """ + Returns HTTP response status code + :param args: args from core runner + :param kwargs: kwargs from core runner + :return: ScriptResponse message + """ + url = kwargs.get("url") + status = get(url=url).status_code + return ScriptResponse.success( + result=status, message=f"Got HTTP response status from {url}" + ) diff --git a/src/scripts/recon/shodan_favicon_hash/__init__.py b/src/scripts/recon/shodan_favicon_hash/__init__.py new file mode 100644 index 0000000..acdd053 --- /dev/null +++ b/src/scripts/recon/shodan_favicon_hash/__init__.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +__root_dir = Path(__file__).parents[4] +sys.path.append(str(__root_dir)) diff --git a/src/scripts/recon/shodan_favicon_hash/__main__.py b/src/scripts/recon/shodan_favicon_hash/__main__.py index c53b946..dbd35b2 100644 --- a/src/scripts/recon/shodan_favicon_hash/__main__.py +++ b/src/scripts/recon/shodan_favicon_hash/__main__.py @@ -1,44 +1,12 @@ #!/usr/bin/env python3 -from base64 import encodebytes +from pprint import pprint +from sys import argv -from mmh3 import hash -from requests import get -from requests.exceptions import RequestException +from src.core.utils.module import run_module +from .module import Runner -from src.core.base.recon import ReconRunner, PossibleKeys -from src.core.utils.response import ScriptResponse -from src.core.utils.validators import validate_kwargs - - -class Runner(ReconRunner): - def __init__(self, logger: str = __name__): - super(Runner, self).__init__(logger) - - @validate_kwargs(PossibleKeys.KEYS) - def run(self, *args, **kwargs): - """ - Returns hash of favicon.ico of given url. - :param args: args from core runner - :param kwargs: kwargs from core runner - :return: ScriptResponse message - """ - url = kwargs.get("url") - - try: - response = get(f"{url}/favicon.ico") - except RequestException as req_err: - return ScriptResponse.error( - result=None, - message=f"Can't connect to {url}!" - f"Error message: {req_err}", - ) - - favicon = encodebytes(response.content) - favicon_hash = hash(favicon) - - return ScriptResponse.success( - result=favicon_hash, - message=f"Successfully made favicon hash of {url}! " - f"Use https://www.shodan.io/search?query=http.favicon.hash:{favicon_hash}", - ) +result = run_module( + Runner, args=argv, arg_name="url", arg_default="https://www.intel.com/" +) +pprint(result) diff --git a/src/scripts/recon/shodan_favicon_hash/module.py b/src/scripts/recon/shodan_favicon_hash/module.py new file mode 100644 index 0000000..3acbaee --- /dev/null +++ b/src/scripts/recon/shodan_favicon_hash/module.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +from base64 import encodebytes + +from mmh3 import hash +from requests import get +from requests.exceptions import RequestException + +from src.core.base.recon import ReconRunner, PossibleKeys +from src.core.utils.response import ScriptResponse +from src.core.utils.validators import validate_kwargs + + +class Runner(ReconRunner): + def __init__(self, logger: str = __name__): + super(Runner, self).__init__(logger) + + @validate_kwargs(PossibleKeys.KEYS) + def run(self, *args, **kwargs): + """ + Returns hash of favicon.ico of given url. + :param args: args from core runner + :param kwargs: kwargs from core runner + :return: ScriptResponse message + """ + url = kwargs.get("url") + + try: + response = get(f"{url}/favicon.ico") + except RequestException as req_err: + return ScriptResponse.error( + result=None, + message=f"Can't connect to {url}!" f"Error message: {req_err}", + ) + + favicon = encodebytes(response.content) + favicon_hash = hash(favicon) + + return ScriptResponse.success( + result=favicon_hash, + message=f"Successfully made favicon hash of {url}! " + f"Use https://www.shodan.io/search?query=http.favicon.hash:{favicon_hash}", + )