hybrid_analysis.py

import urllib.request
import json
import datetime
import os
import argparse
import time
import sys
import jinja2
import shutil

hash_log_path = "HA - Observed Instances.hlog"
feed_log_dir = "."


def get_HA_feed():
    url = "https://www.hybrid-analysis.com/feed?json"
    dt_now = datetime.datetime.now().strftime('%b %d %Y - %H %M %S')
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'
    req = urllib.request.Request(
        url,
        data=None,
        headers={
            'User-Agent': user_agent
        }
    )
    response = urllib.request.urlopen(req)
    content = response.read().decode('utf-8')
    return content, dt_now


def save_feed(output_dir_path, content, retrieval_time):
    filename = 'HA - {}.rlog'.format(retrieval_time)
    with open(os.path.join(output_dir_path, filename), 'w', encoding='utf-8') as rlog:
        rlog.write(content)
    return filename


def record_file_hashes(hash_values, retrieval_time):
    with open(hash_log_path, "a", encoding="utf-8") as log_file:
        log_file.write("# {}\n".format(retrieval_time))
        for hash in hash_values:
            log_file.write(hash + "\n")


def load_file_hashes():
    hash_values = set()
    if os.path.isfile(hash_log_path):
        with open(hash_log_path, "r", encoding="utf-8") as log_file:
            for line in log_file:
                line = line.strip()
                if line.startswith("#") is False:
                    hash_values.add(line)
    return hash_values


def extract_info(content, observed_hashes):
    commands = {}
    file_hashes = set()

    obj = json.loads(content)


    for instance in obj['data']:
        sha256 = instance["sha256"]
        # check whether we has seen this instance before
        if sha256 not in observed_hashes:

            # update the list of observed hashes
            observed_hashes.add(sha256)
            report_url = instance["reporturl"]
            thread_score = instance["threatscore"] if 'threatscore' in instance else ""
            analysis_time = instance["analysis_start_time"] if 'analysis_start_time' in instance else ""
            file_type = instance["type"] if 'type' in instance else ""
            environment = instance["environmentDescription"] if 'environmentDescription' in instance else ""
            vt_detect = instance["vt_detect"] if 'vt_detect' in instance else ""
            instance_rec = ("https://www.hybrid-analysis.com" + report_url, sha256,
                                                    analysis_time, environment, file_type, thread_score,
                                                    vt_detect)
            file_hashes.add(sha256)
            for command in instance['process_list']:
                if 'normalizedpath' in command:
                    if 'WINDIR' in command['normalizedpath'] or 'PROGRAMFILES' in command['normalizedpath']:
                        if command['normalizedpath'] not in commands:
                            commands[command['normalizedpath']] = {}

                        if command['commandline'] not in commands[command['normalizedpath']]:
                            commands[command['normalizedpath']][command['commandline']] = set()
                        commands[command['normalizedpath']][command['commandline']].add(instance_rec)

    record_file_hashes(file_hashes, retrieval_time)

    return commands, file_hashes


def load_command_stat():
    stat = "{}"
    stat_file_path = "command_stat"
    if os.path.isfile(stat_file_path):
        with open(stat_file_path, "r", encoding="utf-8") as stat_file:
            stat = stat_file.read()
    return json.load(stat)


def copytree(src, dst, symlinks=False, ignore=None):
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            if not os.path.exists(d):
                os.makedirs(d)
            copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

def parse_arguments():

    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--daemon", default=False, action='store_true', help="Determines whether it must run forever")
    parser.add_argument("-c", "--cycle", default=60, type=int, help="Determines the duration (in mins) of sleep for each cycle")
    parser.add_argument("-t", "--type", default="text", type=str, help="Determines the format of output")
    parser.add_argument("-o", "--outputdir", default=None, help="Saves the output in the specified directory")
    args = parser.parse_known_args()

    return args[0].daemon, args[0].cycle,  args[0].type, args[0].outputdir


if __name__ == "__main__":

    daemon, cycle, type, output_path = parse_arguments()

    output_file = sys.stdout
    if output_path is not None:

        if not os.path.exists(output_path):
            os.makedirs(output_path)
        if type == "text":
            output_file = open(os.path.join(output_path, "output.log"), "a+", encoding="utf-8")
    else:
        output_path = "./"

    try:
        # get the hashes of all instances that we has seen so far
        observed_hashes = load_file_hashes()

        while True:
            try:
                content, retrieval_time = get_HA_feed()

                now = datetime.datetime.now()

                file_name = save_feed(feed_log_dir, content, retrieval_time)

                commands, observed_file_hashes = extract_info(content, observed_hashes)

                print("The feed was downloaded from hybrid-analysis.com at {} (New instances:{})"
                      .format(str(now), len(observed_file_hashes)))

                if type=="text":
                    for command, commandlines in commands.items():
                        print("\n######################################## " + str(now), file=output_file)
                        print(command + "", file=output_file)
                        for commandline in commandlines:
                            print("\t" + str(commandline), file=output_file)
                            for instance in commandlines[commandline]:
                                print("\t\t" + '\t'.join([str(x) for x in instance]), file=output_file)
                elif type=="html":
                    templateLoader = jinja2.FileSystemLoader(searchpath="./html template")
                    templateEnv = jinja2.Environment(loader=templateLoader)
                    TEMPLATE_FILE = "index.html"
                    template = templateEnv.get_template(TEMPLATE_FILE)
                    result = template.render(commands=commands)

                    copytree("./html template", output_path)
                    os.remove(os.path.join(output_path, "index.html"))
                    open(os.path.join(output_path, 'HA - {}.html'.format(retrieval_time)), 'w', encoding='utf-8').write(result)

                if not daemon:
                    break
            except Exception as e:
                print("The following error has occured during this cycle, will restart after some sleeping:\n" + str(e))

            time.sleep(cycle * 60)
    except Exception as e:
        print("Unrecoverable error has occurred, the program is going to terminate:\n" + str(e))