AutoGenerateFull.py

## LIBRARIES ##
import datetime
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from poibin import PoiBin
import pytest

## IMPORT ML MODELS ##
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn import svm
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, log_loss, roc_auc_score, confusion_matrix, precision_recall_fscore_support
import xgboost as xgb


## CREATE FILE ##
#This section initializes the file, gives it a title and a timestap
now = datetime.datetime.now()
unique_report = input('Enter Unique Report Identifier - ')
fhand = open('OutcomeReport{}.txt'.format(unique_report), 'w+')

# fhand_meta = open('MetaInfoForOutcomeReport{}.csv'.format(unique_report), 'w+')
# with open('MetaInfoForOutcomeReport{}.csv'.format(unique_report), mode='w+') as csv_file:
#     writer = csv.writer(csv_file, delimiter=',')

report_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
fhand.write('SCOTUS Machine Learning Models Outcome Report {} \n\n'.format(unique_report))
fhand.write(report_time)
fhand.write('\n\n')

## DATA IMPORT & CLEANING ##
# setting random seed
np.random.seed(9)

# import data and do initial cleaning
og_data = pd.read_csv('SCDB_2018_01_justiceCentered_Citation.csv', encoding = 'ISO-8859-1')
og_data = og_data.drop(columns = ['justice', 'docketId', 'caseIssuesId', 'voteId', 'dateDecision',
                                        'usCite', 'sctCite', 'ledCite', 'lexisCite',
                                        'docket', 'caseName', 'petitionerState', 'respondentState',
                                        'adminActionState', 'caseOriginState',
                                        'caseSourceState', 'declarationUncon',
                                        'caseDispositionUnusual', 'partyWinning', 'voteUnclear',
                                        'decisionDirectionDissent', 'authorityDecision1', 'authorityDecision2',
                                        'lawSupp', 'lawMinor', 'majOpinWriter', 'majOpinAssigner',
                                        'splitVote','firstAgreement', 'secondAgreement',
                                        'dateArgument', 'dateRearg', 'petitioner', 'respondent',
                                        'term', 'caseDisposition', 'decisionDirection',
                                        'majVotes', 'minVotes', 'majority', 'vote', 'opinion',
                                        'precedentAlteration', 'issueArea'])

# drop rows null for target column, fill in other nulls, and shift targrt column so sklearn recognizes it as binary
og_data = og_data.dropna(subset=['direction'])
og_data = og_data.fillna(int(999))
d2 = {1: 0, 2: 1}
og_data['direction'] = og_data['direction'].map(d2)
for c in og_data.columns:
    og_data[c] = og_data[c].astype('category')
not_to_dummy = ['caseId', 'justiceName', 'direction']
wd_columns_to_dummy = list(og_data.columns)
for n in not_to_dummy:
    wd_columns_to_dummy.remove(n)

# save list of "features used"
fhand.write('Features_used: {} \n\n'.format(wd_columns_to_dummy))

og_data = pd.get_dummies(og_data, columns = wd_columns_to_dummy)

# splitting all case data into top-level train and test sets
full_cases = pd.read_csv('SCDB_2018_01_caseCentered_Citation.csv', encoding = 'ISO-8859-1')
full_cases = full_cases.dropna(subset=['decisionDirection'])
full_cases = full_cases.fillna(int(999))
d = {1: 0, 2: 1, 3: 3}
full_cases['decisionDirection'] = full_cases['decisionDirection'].map(d)
# the line below is not strictly necessary as the formation of justice-level training data
# generally removes rows with no value for 'direction,' and these rows are precisely the ones with
# the value 3 in the 'decisionDirection' column
full_cases = full_cases[full_cases['decisionDirection'].isin([0,1])]
cases = full_cases['caseId']
full_cases_target = full_cases['decisionDirection']
full_cases_data = full_cases[wd_columns_to_dummy]
for c in full_cases_data.columns:
    full_cases_data[c] = full_cases_data[c].astype('category')
full_cases_data = pd.get_dummies(full_cases_data)
full_cases_train, full_cases_test, full_cases_train_target, full_cases_test_target, master_train_case, master_test_case = train_test_split(full_cases_data, full_cases_target, cases)
# print("full_cases_train: ", full_cases_train.shape)
# print("full_cases_test: ", full_cases_test.shape)
# print("master_train_case: ", master_train_case.shape)
# print("master_test_case: ", master_test_case.shape)
test_outcomes = pd.DataFrame(data = full_cases_test_target.values, index = master_test_case.values)
# print("test_outcomes: ", test_outcomes.shape)

# case-centered model:
case_forest = RandomForestClassifier(n_estimators = 2000, max_depth = 12)
case_forest.fit(full_cases_train, full_cases_train_target)
case_forest_train_predict = case_forest.predict(full_cases_train)
case_forest_test_predict = case_forest.predict(full_cases_test)
case_forest_train_score = case_forest.score(full_cases_train, full_cases_train_target)
case_forest_test_score = case_forest.score(full_cases_test, full_cases_test_target)
case_forest_train_probs = case_forest.predict_proba(full_cases_train)
case_forest_test_probs = case_forest.predict_proba(full_cases_test)
case_forest_train_log_loss = log_loss(full_cases_train_target, case_forest_train_probs[:,1])
case_forest_test_log_loss = log_loss(full_cases_test_target, case_forest_test_probs[:,1])
case_forest_train_roc_auc = roc_auc_score(full_cases_train_target, case_forest_train_probs[:,1])
case_forest_test_roc_auc = roc_auc_score(full_cases_test_target, case_forest_test_probs[:,1])
print('\nCase-based train accuracy: ', case_forest_train_score)
print('\nCase-based test accuracy: ', case_forest_test_score)
print('\nCase-based train AUC: ', case_forest_train_roc_auc)
print('\nCase-based test AUC: ', case_forest_test_roc_auc)
print('\nCase-based train log-loss: ', case_forest_train_log_loss)
print('\nCase-based test log-loss: ', case_forest_test_log_loss)
case_con_matrix = confusion_matrix(full_cases_test_target, case_forest_test_predict)
print('\nCase-based test confusion Matrix:\n', case_con_matrix)
precision, recall, fscore, support = precision_recall_fscore_support(full_cases_test_target, case_forest_test_predict)
percent_conservative = support[0]/(support[0] + support[1])
print('\nBased on ', support[0], ' conservative test decsions and ', support[1], ' liberal ones (', percent_conservative, ' conservative):')
print('\nConservatism Precision: ', precision[0], '\nConservatism Recall: ', recall[0], '\nConservatism F1: ', fscore[0])
print('\nLiberalism Precision: ', precision[1], '\nConservatism Recall: ', recall[1], '\nLiberalism F1: ', fscore[1])

fhand.write("Case-based train accuracy: {}\n\n".format(case_forest_train_score))
fhand.write("Case-based test accuracy: {}\n\n".format(case_forest_test_score))
fhand.write("Case-based train AUC: {}\n\n".format(case_forest_train_roc_auc))
fhand.write("Case-based test AUC: {}\n\n".format(case_forest_test_roc_auc))
fhand.write("Case-based train log-loss: {}\n\n".format(case_forest_train_log_loss))
fhand.write("Case-based test log-loss: {}\n\n".format(case_forest_test_log_loss))
fhand.write("Case-based test confusion Matrix: {}\n\n".format(case_con_matrix))
fhand.write('Based on {} conservative test decsions and {} liberal ones ({} conservative):'.format(support[0],support[1],percent_conservative))
fhand.write('\nConservatism Precision: {}\nConservatism Recall: {}\nConservatism F1: {}'.format(precision[0],recall[0],fscore[0]))
fhand.write('\nLiberalism Precision: {}\nLiberalism Recall: {}\nLiberalism F1: {}'.format(precision[1],recall[1],fscore[1]))

## INTIALIZING JUSITCE DATA ##
justices = list(og_data.justiceName.unique())
# use shorter list below for testing purposes
# justices = ['RBGinsburg', 'AScalia', 'SAAlito']

# properly narrow data for later ensemble
working_train_data = og_data[og_data['caseId'].isin(list(master_train_case.values))]
working_test_data = og_data[og_data['caseId'].isin(list(master_test_case.values))]

# used at the end for ensemble method
master_probas = pd.DataFrame(columns = justices, index = master_test_case.values)

# Create list to hold meta_information lists for eventual DataFrame (and export)
rounds_info_master = []

## MACHINE LEARNING MODELS - CLASSIFICATION ##

fhand.write('Models will be run for {} justices\n\n'.format(len(justices)))
now = datetime.datetime.now()
report_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
fhand.write(report_time)

fhand.write('Random Forest Classifier tuned on:\n')
fhand.write('n-estimators = [50, 250, 500]\n')
fhand.write('max_depth = [5, 10, 15, 20, 25, 30]\n')
fhand.write('for the roc_auc metric.\n\n')

# fhand.write('XGBoostClassifier tuned on:\n')
# fhand.write('alpha = [0.001, 0.01, 0.1, 0.2]\n')
# fhand.write('n-estimators = [100, 200, 300]\n')
# fhand.write('max_depth = [1, 2, 4, 6]\n')
# fhand.write('for the roc_auc metric.\n\n')

fhand.write('AdaBoost Classifier tuned on:\n')
fhand.write('alpha = [0.001, 0.01, 0.1]\n')
fhand.write('n-estimators = [100, 200, 300]\n')
fhand.write('max_depth = [1, 3, 6]\n')
fhand.write('for the roc_auc metric.\n\n')

fhand.write('Support Vector Machine tuned on:\n')
fhand.write('kernel = [linear, rbf, sigmoid]\n')
fhand.write('c_value = [1, 5, 10, 25, 50, 75, 100]\n\n')

# fhand.write('Logisstic Regression tuned on:\n')
# fhand.write('kernel = [linear, rbf, sigmoid]\n')
# fhand.write('c_value = [1, 5, 10, 25, 50, 75, 100]\n\n')

model_run_count = 0
print('')
print('Start Time: ')
print(report_time)

for i in range(len(justices)):

    fhand.write('************************************************')
    fhand.write("\n\n")

    current_justice = justices[model_run_count]

    model_run_count += 1

    fhand.write('Model Set {} - Justice {}\n\n'.format(model_run_count, current_justice))

    # BUILD JUSTICE DATAFRAME #
    current_justice_train_df = working_train_data[working_train_data['justiceName'] == current_justice]
    current_justice_test_df = working_test_data[working_test_data['justiceName'] == current_justice]
    case_test = current_justice_test_df['caseId']
    current_justice_train_df = current_justice_train_df.drop(columns = ['caseId', 'justiceName'])
    current_justice_test_df = current_justice_test_df.drop(columns = ['caseId', 'justiceName'])

    #pull out target vector
    current_justice_target_train = current_justice_train_df['direction']
    current_justice_data_train = current_justice_train_df.drop(columns = ['direction'])
    current_justice_target_test = current_justice_test_df['direction']
    current_justice_data_test = current_justice_test_df.drop(columns = ['direction'])

    # INTIALIZING MODELS #

    ### Random Forest ###
    forest = RandomForestClassifier(n_estimators = 100, max_depth = 15)
    forest.fit(current_justice_data_train, current_justice_target_train)

    # Initial Outcome Metrics
    forest_initial_train_score = forest.score(current_justice_data_train, current_justice_target_train)
    forest_initial_test_score = forest.score(current_justice_data_test, current_justice_target_test)

    initial_forest_train_probs = forest.predict_proba(current_justice_data_train)
    initial_forest_test_probs = forest.predict_proba(current_justice_data_test)

    initial_forest_train_predict = forest.predict(current_justice_data_train)
    initial_forest_test_predict = forest.predict(current_justice_data_test)

    forest_initial_train_log_loss = log_loss(current_justice_target_train, initial_forest_train_probs[:,1])
    forest_initial_test_log_loss = log_loss(current_justice_target_test, initial_forest_test_probs[:,1])

    forest_initial_train_roc_auc = roc_auc_score(current_justice_target_train, initial_forest_train_probs[:,1])
    forest_initial_test_roc_auc = roc_auc_score(current_justice_target_test, initial_forest_test_probs[:,1])

    # Hyperparamater Tuning

    param_grid_forest = {'n_estimators' : [50, 250, 500, 1000], 'max_depth' : [5, 10, 15, 25]}

    # kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)

    grid_search = GridSearchCV(forest, param_grid_forest, scoring = "accuracy", n_jobs = -1, cv = 4)
    grid_result = grid_search.fit(current_justice_data_train, current_justice_target_train)

    # Interpreting results
    forest_best_score = grid_result.best_score_
    forest_best_params = grid_result.best_params_

    # Reintializing model with best parameters

    forest = RandomForestClassifier(max_depth = forest_best_params['max_depth'], n_estimators = forest_best_params['n_estimators'])

    forest.fit(current_justice_data_train, current_justice_target_train)

    # Final Metrics

    forest_tuned_train_score = forest.score(current_justice_data_train, current_justice_target_train)
    forest_tuned_test_score = forest.score(current_justice_data_test, current_justice_target_test)

    tuned_forest_train_probs= forest.predict_proba(current_justice_data_train)
    tuned_forest_test_probs= forest.predict_proba(current_justice_data_test)
    probs_series = pd.DataFrame(data = tuned_forest_test_probs[:,1], index = case_test.index)
    probs_with_ids = pd.concat([probs_series, case_test], axis = 1)
    probs_with_ids.rename(columns={0:'probability'}, inplace = True)
    for ind, row in probs_with_ids.iterrows():
        case = row['caseId']
        probabil = row['probability']
        master_probas[current_justice].loc[case] = probabil

    tuned_forest_train_predict = forest.predict(current_justice_data_train)
    tuned_forest_test_predict = forest.predict(current_justice_data_test)

    forest_tuned_train_log_loss = log_loss(current_justice_target_train, tuned_forest_train_probs)
    forest_tuned_test_log_loss = log_loss(current_justice_target_test, tuned_forest_test_probs)

    forest_tuned_train_roc_auc = roc_auc_score(current_justice_target_train, tuned_forest_train_probs[:,1])
    forest_tuned_test_roc_auc = roc_auc_score(current_justice_target_test, tuned_forest_test_probs[:,1])

    # Write to file

    fhand.write("Random Forest Model Results\n\n")

    now = datetime.datetime.now()
    report_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
    fhand.write(report_time)
    fhand.write("\n\n")

    fhand.write("On default settings, Random Forest accuracy score on training data was {}\n\n".format(forest_initial_train_score))

    fhand.write("On default settings, Random Forest accuracy score on test data was {}\n\n".format(forest_initial_test_score))

    fhand.write("On default settings, Random Forest logloss on training set was {}\n\n".format(forest_initial_train_log_loss))

    fhand.write("On default settings, Random Forest logloss on test set was {}\n\n".format(forest_initial_test_log_loss))

    fhand.write("On default settings, Random Forest roc_auc on training set was {}\n\n".format(forest_initial_train_roc_auc))

    fhand.write("On default settings, Random Forest roc_auc on test set was {}\n\n".format(forest_initial_test_roc_auc))

    fhand.write("The best roc_auc score achieved by GridSearchCV was {}\nIt was achieved by setting max depth to {} and n_estimators to {}\n\n".format(forest_best_score, forest_best_params['max_depth'], forest_best_params['n_estimators']))

    fhand.write("Once tuned, Random Forest accuracy score on training data was {}\n\n".format(forest_tuned_train_score))

    fhand.write("Once tuned, Random Forest accuracy score on test data was {}\n\n".format(forest_tuned_test_score))

    fhand.write("Once tuned, Random Forest logloss on training set was {}\n\n".format(forest_tuned_train_log_loss))

    fhand.write("Once tuned, Random Forest logloss on test set was {}\n\n".format(forest_tuned_test_log_loss))

    fhand.write("Once tuned, Random Forest roc_auc on training set was {}\n\n".format(forest_tuned_train_roc_auc))

    fhand.write("Once tuned, Random Forest roc_auc on test set was {}\n\n".format(forest_tuned_test_roc_auc))

    fhand.write('-----------------------------------------')
    fhand.write("\n\n")

## XGBoost is ready but currently not on; needs work

    # ### XGBoost ###
    #
    # # Initialize Model
    #
    # xgboost = xgb.XGBClassifier()
    # xgboost.fit(current_justice_data_train, current_justice_target_train)
    #
    # # Initial Outcome Metrics
    #
    # xgboost_initial_train_score = xgboost.score(current_justice_data_train, current_justice_target_train)
    # xgboost_initial_test_score = xgboost.score(current_justice_data_test, current_justice_target_test)
    #
    # initial_xgboost_train_probs = xgboost.predict_proba(current_justice_data_train)
    # initial_xgboost_test_probs = xgboost.predict_proba(current_justice_data_test)
    #
    # initial_xgboost_train_predict = xgboost.predict(current_justice_data_train)
    # initial_xgboost_test_predict = xgboost.predict(current_justice_data_test)
    #
    # xgboost_initial_train_log_loss = log_loss(current_justice_target_train, initial_xgboost_train_probs)
    # xgboost_initial_test_log_loss = log_loss(current_justice_target_test, initial_xgboost_test_probs)
    #
    # xgboost_initial_train_roc_auc = roc_auc_score(current_justice_target_train, initial_xgboost_train_probs[:,1])
    # xgboost_initial_test_roc_auc = roc_auc_score(current_justice_target_test, initial_xgboost_test_probs[:,1])
    #
    # # Hyperparamater Tuning
    #
    # xgboost_alpha = [ 0.01, 0.1]
    # xgboost_n_estimators = [100, 200, 300]
    # xgboost_max_depth = [1, 3, 6]
    #
    # param_grid_xgboost = dict(n_estimators=xgboost_n_estimators, max_depth=xgboost_max_depth, learning_rate = xgboost_alpha)
    #
    # kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)
    #
    # grid_search = GridSearchCV(xgboost, param_grid_xgboost, scoring="roc_auc", n_jobs=-1, cv=kfold)
    # grid_result = grid_search.fit(current_justice_data_train, current_justice_target_train)
    #
    # # Interpreting results
    # xgboost_best_score = grid_result.best_score_
    # xgboost_best_params = grid_result.best_params_
    #
    # # Reintializing model with best parameters
    #
    # xgboost = xgb.XGBClassifier(max_depth = xgboost_best_params['max_depth'], n_estimators = xgboost_best_params['n_estimators'], learning_rate = xgboost_best_params['learning_rate'])
    #
    # xgboost.fit(current_justice_data_train, current_justice_target_train)
    #
    # # Final Metrics
    #
    # xgboost_tuned_train_score = xgboost.score(current_justice_data_train, current_justice_target_train)
    # xgboost_tuned_test_score = xgboost.score(current_justice_data_test, current_justice_target_test)
    #
    # tuned_xgboost_train_probs= xgboost.predict_proba(current_justice_data_train)
    # tuned_xgboost_test_probs= xgboost.predict_proba(current_justice_data_test)
    #
    # tuned_xgboost_train_predict = xgboost.predict(current_justice_data_train)
    # tuned_xgboost_test_predict = xgboost.predict(current_justice_data_test)
    #
    # xgboost_tuned_train_log_loss = log_loss(current_justice_target_train, tuned_xgboost_train_probs)
    # xgboost_tuned_test_log_loss = log_loss(current_justice_target_test, tuned_xgboost_test_probs)
    #
    # xgboost_tuned_train_roc_auc = roc_auc_score(current_justice_target_train, tuned_xgboost_train_probs[:,1])
    # xgboost_tuned_test_roc_auc = roc_auc_score(current_justice_target_test, tuned_xgboost_test_probs[:,1])
    #
    # # Write to file
    #
    # fhand.write("XGBoost Model Results\n\n")
    #
    # now = datetime.datetime.now()
    # report_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
    # fhand.write(report_time)
    # fhand.write("\n\n")
    #
    # fhand.write("On default settings, XGBoost accuracy score on training data was {}\n\n".format(xgboost_initial_train_score))
    #
    # fhand.write("On default settings, XGBoost accuracy score on test data was {}\n\n".format(xgboost_initial_test_score))
    #
    # fhand.write("On default settings, XGBoost logloss on training set was {}\n\n".format(xgboost_initial_train_log_loss))
    #
    # fhand.write("On default settings, XGBoost logloss on test set was {}\n\n".format(xgboost_initial_test_log_loss))
    #
    # fhand.write("On default settings, XGBoost roc_auc on training set was {}\n\n".format(xgboost_initial_train_roc_auc))
    #
    # fhand.write("On default settings, XGBoost roc_auc on test set was {}\n\n".format(xgboost_initial_test_roc_auc))
    #
    # fhand.write("The best roc_auc score achieved by GridSearchCV was {}\nIt was achieved by setting alpha to {}, max depth to {}, and n_estimators to {}\n\n".format(xgboost_best_score, xgboost_best_params['learning_rate'], xgboost_best_params['max_depth'], xgboost_best_params['n_estimators']))
    #
    # fhand.write("Once tuned, XGBoost accuracy score on training data was {}\n\n".format(xgboost_tuned_train_score))
    #
    # fhand.write("Once tuned, XGBoost accuracy score on test data was {}\n\n".format(xgboost_tuned_test_score))
    #
    # fhand.write("Once tuned, XGBoost logloss on training set was {}\n\n".format(xgboost_tuned_train_log_loss))
    #
    # fhand.write("Once tuned, XGBoost logloss on test set was {}\n\n".format(xgboost_tuned_test_log_loss))
    #
    # fhand.write("Once tuned, XGBoost roc_auc on training set was {}\n\n".format(xgboost_tuned_train_roc_auc))
    #
    # fhand.write("Once tuned, XGBoost roc_auc on test set was {}\n\n".format(xgboost_tuned_test_roc_auc))
    #
    # fhand.write('-----------------------------------------')
    # fhand.write("\n\n")



    ### AdaBoost ###

    # Initialize Model

    adaboost = AdaBoostClassifier()
    adaboost.fit(current_justice_data_train, current_justice_target_train)

    # Initial Outcome Metrics

    adaboost_initial_train_score = adaboost.score(current_justice_data_train, current_justice_target_train)
    adaboost_initial_test_score = adaboost.score(current_justice_data_test, current_justice_target_test)

    initial_adaboost_train_probs = adaboost.predict_proba(current_justice_data_train)
    initial_adaboost_test_probs = adaboost.predict_proba(current_justice_data_test)

    initial_adaboost_train_predict = adaboost.predict(current_justice_data_train)
    initial_adaboost_test_predict = adaboost.predict(current_justice_data_test)

    adaboost_initial_train_log_loss = log_loss(current_justice_target_train, initial_adaboost_train_probs)
    adaboost_initial_test_log_loss = log_loss(current_justice_target_test, initial_adaboost_test_probs)

    adaboost_initial_train_roc_auc = roc_auc_score(current_justice_target_train, initial_adaboost_train_probs[:,1])
    adaboost_initial_test_roc_auc = roc_auc_score(current_justice_target_test, initial_adaboost_test_probs[:,1])

    # Hyperparamater Tuning

    param_grid_adaboost = {'learning_rate' : [0.001, 0.01, 0.1], 'n_estimators' : [100, 300, 600]}

    # kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)

    grid_search = GridSearchCV(adaboost, param_grid_adaboost, scoring="roc_auc", n_jobs=-1, cv=4)
    grid_result = grid_search.fit(current_justice_data_train, current_justice_target_train)

    # Interpreting results
    adaboost_best_score = grid_result.best_score_
    adaboost_best_params = grid_result.best_params_

    # Reintializing model with best parameters

    adaboost = AdaBoostClassifier(n_estimators = adaboost_best_params['n_estimators'], learning_rate = adaboost_best_params['learning_rate'])

    adaboost.fit(current_justice_data_train, current_justice_target_train)

    # Final Metrics

    adaboost_tuned_train_score = adaboost.score(current_justice_data_train, current_justice_target_train)
    adaboost_tuned_test_score = adaboost.score(current_justice_data_test, current_justice_target_test)

    tuned_adaboost_train_probs= adaboost.predict_proba(current_justice_data_train)
    tuned_adaboost_test_probs= adaboost.predict_proba(current_justice_data_test)

    tuned_adaboost_train_predict = adaboost.predict(current_justice_data_train)
    tuned_adaboost_test_predict = adaboost.predict(current_justice_data_test)

    adaboost_tuned_train_log_loss = log_loss(current_justice_target_train, tuned_adaboost_train_probs)
    adaboost_tuned_test_log_loss = log_loss(current_justice_target_test, tuned_adaboost_test_probs)

    adaboost_tuned_train_roc_auc = roc_auc_score(current_justice_target_train, tuned_adaboost_train_probs[:,1])
    adaboost_tuned_test_roc_auc = roc_auc_score(current_justice_target_test, tuned_adaboost_test_probs[:,1])

    # Write to file

    fhand.write("AdaBoost Model Results\n\n")

    now = datetime.datetime.now()
    report_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
    fhand.write(report_time)
    fhand.write("\n\n")

    fhand.write("On default settings, AdaBoost accuracy score on training data was {}\n\n".format(adaboost_initial_train_score))

    fhand.write("On default settings, AdaBoost accuracy score on test data was {}\n\n".format(adaboost_initial_test_score))

    fhand.write("On default settings, AdaBoost logloss on training set was {}\n\n".format(adaboost_initial_train_log_loss))

    fhand.write("On default settings, AdaBoost logloss on test set was {}\n\n".format(adaboost_initial_test_log_loss))

    fhand.write("On default settings, AdaBoost roc_auc on training set was {}\n\n".format(adaboost_initial_train_roc_auc))

    fhand.write("On default settings, AdaBoost roc_auc on test set was {}\n\n".format(adaboost_initial_test_roc_auc))

    fhand.write("The best roc_auc score achieved by GridSearchCV was {}\nIt was achieved by setting learning_rate to {} and n_estimators to {}\n\n".format(adaboost_best_score, adaboost_best_params['learning_rate'], adaboost_best_params['n_estimators']))

    fhand.write("Once tuned, AdaBoost accuracy score on training data was {}\n\n".format(adaboost_tuned_train_score))

    fhand.write("Once tuned, AdaBoost accuracy score on test data was {}\n\n".format(adaboost_tuned_test_score))

    fhand.write("Once tuned, AdaBoost logloss on training set was {}\n\n".format(adaboost_tuned_train_log_loss))

    fhand.write("Once tuned, AdaBoost logloss on test set was {}\n\n".format(adaboost_tuned_test_log_loss))

    fhand.write("Once tuned, AdaBoost roc_auc on training set was {}\n\n".format(adaboost_tuned_train_roc_auc))

    fhand.write("Once tuned, AdaBoost roc_auc on test set was {}\n\n".format(adaboost_tuned_test_roc_auc))

    fhand.write('-----------------------------------------')
    fhand.write("\n\n")

    ### Support Vector Machine ###

    # Intialize Model

    svm_model = svm.SVC(probability=True, random_state=7)
    svm_model.fit(current_justice_data_train, current_justice_target_train)

    # Hyperparamater Tuning

    svm_model_kernel = ["linear", "rbf"]
    svm_model_C_value = [1, 5, 10, 25, 50, 75, 100]

    param_grid_svm_model = dict(C = svm_model_C_value, kernel = svm_model_kernel)

    # kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)

    grid_search = GridSearchCV(svm_model, param_grid_svm_model, scoring="roc_auc",  n_jobs=-1, cv=4)
    grid_result = grid_search.fit(current_justice_data_train, current_justice_target_train)

    # Interpreting results
    svm_model_best_score = grid_result.best_score_
    svm_model_best_params = grid_result.best_params_

    # Reintializing model with best parameters

    svm_model = svm.SVC(C = svm_model_best_params['C'], kernel = svm_model_best_params['kernel'], probability=True, random_state=7)

    svm_model.fit(current_justice_data_train, current_justice_target_train)

    # Final Metrics

    svm_model_tuned_train_score = svm_model.score(current_justice_data_train, current_justice_target_train)
    svm_model_tuned_test_score = svm_model.score(current_justice_data_test, current_justice_target_test)

    tuned_svm_model_train_probs= svm_model.predict_proba(current_justice_data_train)
    tuned_svm_model_test_probs= svm_model.predict_proba(current_justice_data_test)

    svm_model_tuned_train_log_loss = log_loss(current_justice_target_train, tuned_svm_model_train_probs)
    svm_model_tuned_test_log_loss = log_loss(current_justice_target_test, tuned_svm_model_test_probs)

    svm_model_tuned_train_roc_auc = roc_auc_score(current_justice_target_train, tuned_svm_model_train_probs[:,1])
    svm_model_tuned_test_roc_auc = roc_auc_score(current_justice_target_test, tuned_svm_model_test_probs[:,1])


    fhand.write("Support Vector Machine Results\n\n")

    now = datetime.datetime.now()
    report_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
    fhand.write(report_time)
    fhand.write("\n\n")

    fhand.write("The best roc_auc score achieved by GridSearchCV was {}\nIt was achieved by setting C to {} and kernel type to {}\n\n".format(svm_model_best_score, svm_model_best_params['C'], svm_model_best_params['kernel']))

    fhand.write("Once tuned, SVM accuracy score on training data was {}\n\n".format(svm_model_tuned_train_score))

    fhand.write("Once tuned, SVM accuracy score on test data was {}\n\n".format(svm_model_tuned_test_score))

    fhand.write("Once tuned, SVM logloss on training set was {}\n\n".format(svm_model_tuned_train_log_loss))

    fhand.write("Once tuned, SVM logloss on test set was {}\n\n".format(svm_model_tuned_test_log_loss))

    fhand.write("Once tuned, SVM roc_auc on training set was {}\n\n".format(svm_model_tuned_train_roc_auc))

    fhand.write("Once tuned, SVM roc_auc on test set was {}\n\n".format(svm_model_tuned_test_roc_auc))

    fhand.write('-----------------------------------------')
    fhand.write("\n\n")


    ### Logistic Regression ###

    #
    # fhand.write("Logistic Regression Results\n\n")
    #
    # now = datetime.datetime.now()
    # report_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
    # fhand.write(report_time)
    # fhand.write("\n\n")


    # SAVE JUSTICE SPECIFIC OUTCOMES #

    # write meta analysis to a csv
    round_info = [current_justice, model_run_count, forest_initial_train_score, forest_initial_test_score, forest_initial_train_log_loss, forest_initial_test_log_loss, forest_initial_train_roc_auc, forest_initial_test_roc_auc, forest_best_score, forest_best_params['max_depth'], forest_best_params['n_estimators'], forest_tuned_train_score, forest_tuned_test_score, forest_tuned_train_log_loss, forest_tuned_test_log_loss, forest_tuned_train_roc_auc, forest_tuned_test_roc_auc, adaboost_initial_train_score, adaboost_initial_test_score, adaboost_initial_train_log_loss, adaboost_initial_test_log_loss, adaboost_initial_train_roc_auc, adaboost_initial_test_roc_auc, adaboost_best_score, adaboost_best_params['learning_rate'], adaboost_best_params['n_estimators'],
    adaboost_tuned_train_score, adaboost_tuned_test_score, adaboost_tuned_train_log_loss, adaboost_tuned_test_log_loss, adaboost_tuned_train_roc_auc, adaboost_tuned_test_roc_auc, svm_model_best_score, svm_model_best_params['C'], svm_model_best_params['kernel'], svm_model_tuned_train_score, svm_model_tuned_test_score, svm_model_tuned_train_log_loss, svm_model_tuned_test_log_loss, svm_model_tuned_train_roc_auc, svm_model_tuned_test_roc_auc]

    rounds_info_master.append(round_info)

#xgboost meta tags

# xgboost_initial_train_score, xgboost_initial_test_score, xgboost_initial_train_log_loss, xgboost_initial_test_log_loss, xgboost_initial_train_roc_auc, xgboost_initial_test_roc_auc, xgboost_best_score, xgboost_best_params['learning_rate'], xgboost_best_params['max_depth'],
# xgboost_best_params['n_estimators'], xgboost_tuned_train_score, xgboost_tuned_test_score, xgboost_tuned_train_log_loss, xgboost_tuned_test_log_loss, xgboost_tuned_train_roc_auc, xgboost_tuned_test_roc_auc,



    #create lists of feature coefficients, then add id column info

    model_type = "RFT"
    forest_feature_import = list(forest.feature_importances_)
    forest_feature_import_with_id = ['{}-{}-{}'.format(current_justice, model_type, model_run_count)] + forest_feature_import

    # model_type = "XGB"
    # xgb_feature_import = list(xgboost.feature_importances_)
    # xgb_feature_import_with_id = ['{}-{}-{}'.format(current_justice, model_type, model_run_count)] + xgb_feature_import

    model_type = "ADA"
    ada_feature_import = list(adaboost.feature_importances_)
    ada_feature_import_with_id = ['{}-{}-{}'.format(current_justice, model_type, model_run_count)] + ada_feature_import

    # model_type = "SVM"
    # svm_feature_import = list(svm_model.feature_importances_)
    # svm_feature_import_with_id = ['{}-{}-{}'.format(current_justice, model_type, model_run_count)] + svm_feature_import

    # model_type = "LGR"
    # lgr_feature_import = list(logreg.feature_importances_)
    # lgr_feature_import_with_id = ['{}-{}-{}'.format(current_justice, model_type, model_run_count)] + lgr_feature_import


    feature_info_master = []

    feature_columns_info_master =[]

    feature_info_master.append(forest_feature_import_with_id)
    # feature_info_master.append(xgb_feature_import_with_id)
    feature_info_master.append(ada_feature_import_with_id)
    # feature_info_master.append(svm_feature_import_with_id)
    # feature_info_master.append(lgr_feature_import_with_id)

    features_as_a_list = list(current_justice_data_train.columns)
    features_master_columns = ['ID'] + features_as_a_list
    feature_columns_info_master.append(features_master_columns)

    feature_master = pd.DataFrame.from_records(feature_info_master, columns = feature_columns_info_master)
    feature_master.to_csv('OutcomeReport_{}_FeatureImportInfo{}.csv'.format(unique_report, current_justice), mode = 'w+')

    print("Round {} - Justice {} Done.\n".format(model_run_count, current_justice))

    now = datetime.datetime.now()
    report_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
    print("At:" + report_time)

    # fhand.close()



#Creating meta_information DataFrame for export to csv

meta_master = pd.DataFrame.from_records(rounds_info_master, columns = ['JusticeName', 'Round',
                    'forest_initial_train_score', 'forest_initial_test_score', 'forest_initial_train_log_loss', 'forest_initial_test_log_loss', 'forest_initial_train_roc_auc', 'forest_initial_test_roc_auc', 'forest_best_score', 'forest_best_max_depth' , 'forest_best_n_estimators', 'forest_tuned_train_score', 'forest_tuned_test_score', 'forest_tuned_train_log_loss', 'forest_tuned_test_log_loss', 'forest_tuned_train_roc_auc', 'forest_tuned_test_roc_auc', 'adaboost_initial_train_score', 'adaboost_initial_test_score', 'adaboost_initial_train_log_loss', 'adaboost_initial_test_log_loss', 'adaboost_initial_train_roc_auc',
                    'adaboost_initial_test_roc_auc', 'adaboost_best_score', 'adaboost_best_learning_rate', 'adaboost_best_n_estimators', 'adaboost_tuned_train_score', 'adaboost_tuned_test_score', 'adaboost_tuned_train_log_loss', 'adaboost_tuned_test_log_loss', 'adaboost_tuned_train_roc_auc', 'adaboost_tuned_test_roc_auc', 'svm_model_best_score', 'svm_model_best_c', 'svm_model_best_kernel', 'svm_model_tuned_train_score', 'svm_model_tuned_test_score', 'svm_model_tuned_train_log_loss', 'svm_model_tuned_test_log_loss', 'svm_model_tuned_train_roc_auc', 'svm_model_tuned_test_roc_auc'])


#xgboost meta column tags

# 'xgboost_initial_test_score', 'xgboost_initial_train_log_loss', 'xgboost_initial_test_log_loss', 'xgboost_initial_train_roc_auc',
# 'xgboost_initial_test_roc_auc', 'xgboost_best_score', 'xgboost_best_alpha', 'xgboost_best_max_depth', 'xgboost_best_n_estimators', 'xgboost_tuned_train_score', 'xgboost_tuned_test_score', 'xgboost_tuned_train_log_loss', 'xgboost_tuned_test_log_loss', 'xgboost_tuned_train_roc_auc', 'xgboost_tuned_test_roc_auc',

meta_master.to_csv('OutcomeReport{}MetaInfo.csv'.format(unique_report), mode = 'w+')

# feature_master.to_csv()


for i in range(len(justices)):
    feature_master = pd.DataFrame.from_records(feature_info_master, columns = feature_columns_info_master)
    feature_master.to_csv('OutcomeReport_{}_FeatureImportInfo{}.csv'.format(unique_report, current_justice), mode = 'w+')


master_probas = master_probas.fillna(2)
ps = dict.fromkeys(list(master_probas.index.values), 0)
for ind, row in master_probas.iterrows():
    lista = []
    for c in master_probas.columns:
        if row[c] != 2:
            lista.append(row[c])
    ps[ind] = lista
outcomes = {}
for k in ps.keys():
    pb = PoiBin(ps[k])
    if len(ps[k]) == 9:
        outcomes[k] = sum(pb.pmf([5, 6, 7, 8, 9]))
    elif len(ps[k]) == 8:
        outcomes[k] = sum(pb.pmf([5, 6, 7, 8]))
    elif len(ps[k]) == 7:
        outcomes[k] = sum(pb.pmf([4, 5, 6, 7]))
    elif len(ps[k]) == 6:
        outcomes[k] = sum(pb.pmf([4, 5, 6]))
    elif len(ps[k]) == 5:
        outcomes[k] = sum(pb.pmf([3, 4, 5]))
    elif len(ps[k]) == 4:
        outcomes[k] = sum(pb.pmf([3, 4]))
    elif len(ps[k]) == 3:
        outcomes[k] = sum(pb.pmf([2, 3]))
    elif len(ps[k]) == 2:
        outcomes[k] = sum(pb.pmf([2]))
# as it happens, the minimum number of justices to vote in a case is 5

# print("\n\nOutcomes: ", outcomes)

probs = []
case_outcomes = []
for k,v in outcomes.items():
    probs.append(v)
    case_outcomes.append(test_outcomes.loc[k])

predicted = []
for prob in probs:
    if prob > 0.5:
        val = 1
    else:
        val = 0
    predicted.append(val)

ensemble_acc = accuracy_score(case_outcomes, predicted)
print("\nJustice-based test accuracy: ", ensemble_acc)

ensemble_auc = roc_auc_score(case_outcomes, probs)
print("\nJustice-based test AUC: ", ensemble_auc)

probs2 = np.array(probs)
case_outcomes2 = np.array(case_outcomes)
ensemble_ll = log_loss(case_outcomes2, probs2)
print("\nJustice-based test log-loss: ", ensemble_ll)

cnf_matrix = confusion_matrix(case_outcomes, predicted)
print('\nJustice-based test confusion Matrix:\n',cnf_matrix)

precision, recall, fscore, support = precision_recall_fscore_support(case_outcomes, predicted)
percent_conservative = support[0]/(support[0] + support[1])
print('\nBased on ', support[0], ' conservative test decsions and ', support[1], ' liberal ones (', percent_conservative, ' conservative):')
print('\nConservatism Precision: ', precision[0], '\nConservatism Recall: ', recall[0], '\nConservatism F1: ', fscore[0])
print('\nLiberalism Precision: ', precision[1], '\nConservatism Recall: ', recall[1], '\nLiberalism F1: ', fscore[1])

finish_time = now.strftime("%m-%d-%Y %I:%M:%S %p")
print('\n\n', finish_time)

fhand.write("Justice-based test accuracy: {}\n\n".format(ensemble_acc))
fhand.write("Justice-based test AUC: {}\n\n".format(ensemble_auc))
fhand.write("Justice-based test log-loss: {}\n\n".format(ensemble_ll))
fhand.write("Justice-based test confusion Matrix: {}\n\n".format(cnf_matrix))
fhand.write('Based on {} conservative test decsions and {} liberal ones ({} conservative):'.format(support[0],support[1],percent_conservative))
fhand.write('\nConservatism Precision: {}\nConservatism Recall: {}\nConservatism F1: {}'.format(precision[0],recall[0],fscore[0]))
fhand.write('\nLiberalism Precision: {}\nLiberalism Recall: {}\nLiberalism F1: {}'.format(precision[1],recall[1],fscore[1]))

fhand.close()



## MACHINE LEARNING MODELS - REGRESSION ##
# Using Miller-Quinn scores with accuracy data of justices