def decontracted(phrase):
    # specific
    phrase = re.sub(r"won['’‘`]t", "will not", phrase)
    phrase = re.sub(r"can['’‘`]t", "can not", phrase)
    phrase = re.sub(r"ain['’‘`]t", "am not", phrase)

    # general
    phrase = re.sub(r"n['’‘`]t", " not", phrase)
    phrase = re.sub(r"['’‘`]re", " are", phrase)
    phrase = re.sub(r"['’‘`]s", " is", phrase)
    phrase = re.sub(r"['’‘`]d", " would", phrase)
    phrase = re.sub(r"['’‘`]ll", " will", phrase)
    phrase = re.sub(r"['’‘`]t", " not", phrase)
    phrase = re.sub(r"['’‘`]ve", " have", phrase)
    phrase = re.sub(r"['’‘`]m", " am", phrase)

    #phrase = re.sub('([.;!?])', r' \1 ', phrase)
    phrase = re.sub(r'[^\w.?!;]', ' ', phrase)
    phrase = re.sub(' +', ' ', phrase)
    sentences = re.split('([.;!?] *)', phrase)

    return ' '.join([i.capitalize() for i in sentences])

nlp = en_core_web_lg.load(disable=['parser'])
nlp.max_length = 5000000

import re


def prepare_text(text, drop_words=[]):
    pat = re.compile(r'[^A-Za-z]+')
    cleared_text = re.sub(pat, ' ', decontracted(text))
    nlp_doc = nlp(cleared_text.strip())
    pos_drop_dict = ['ADP', 'AUX', 'CONJ', 'CCONJ', 'DET', 'PUNCT', 'SYM', 'X', 'SPACE', 'NUM']
    prepared_text = ''
    for token in nlp_doc:
        if token.pos_ in pos_drop_dict:
            continue
        elif len(token) <= 3:
            continue
        elif token.lemma_ in drop_words:
            continue

        else:
            prepared_text += str(token).lower() + ' '
    return prepared_text[:-1]

'provision unenforceable': {'provision unenforceable',
  'provisions unenforceable',
  'unenforceable provision',
  'unenforceable provisions'},
 'state unenforceable': {'unenforceable state'},
 'make state': {'made state',
  'made states',
  'make state',
  'making state',
  'making stated',
  'state made',
  'state make',
  'state makes',
  'stated made',
  'stated make',
  'states made',
  'states make',
  'states making',
  'stating making'},
  ...

matrices_dict = {}
for cat_id in final_cat_id:
    matrices_dict[cat_id] = np.load(f'../RES/relevance_matrices/{cat_id}_relevance_matrix_new.npy')

matrices_dict['16'].shape

>>> (1059, 13889)

len(final_substrings['16'])

>>> 13889

pd.DataFrame(matrices_dict['16'], columns=final_substrings['16'])

bow_matrices = {}
bow_phrases = {}
for cat_id in tqdm(final_cat_id):
    phrases = list(sorted(set(substring_dictionaries_reverse[cat_id][w] for w in final_substrings[cat_id])))
    matrix = matrices_dict[cat_id]

    bow_matrix = []
    for phrase in tqdm(phrases, leave=False):
        idx = []
        for p in substring_dictionaries[cat_id][phrase]:
            if p in final_substrings[cat_id]:
                idx.append(final_substrings[cat_id].index(p))
        bow_matrix.append(matrix[:, idx].max(axis=1))
    bow_matrices[cat_id] = bow_matrix
    bow_phrases[cat_id] = phrases

def get_most_relevant_substrings(matrix, substrings, thrsh=0.3, max_num=100, rel_fraction=0.05):
    assert matrix.shape[1] == len(substrings)
    phrase_vals = matrix[:, 0]

    phrase_meanval = []
    for phrase_vals in matrix.T:
        phrase_meanval.append(np.mean(np.sort(phrase_vals)[-max_num:]))
    phrase_meanval = np.array(phrase_meanval)

    tmp_idx = np.where(phrase_meanval > thrsh)[0]

    most_relevant_idx = tmp_idx[np.argsort(phrase_meanval[tmp_idx])[-int(tmp_idx.shape[0] * rel_fraction):]]
    return pd.DataFrame(
        {'phrase': substrings[most_relevant_idx][::-1], 'mean_score': phrase_meanval[most_relevant_idx][::-1]}
    )
rel_phrases_dict = {}
for cat_id in tqdm(final_cat_id):
    matrix = bow_matrices[cat_id]
    substrings = np.array(bow_phrases[cat_id])
    rel_phrases_dict[cat_id] = get_most_relevant_substrings(matrix, substrings, max_num=50)
print(rel_phrases_dict['16'])    
>>>
                           phrase  mean_score
0         arbitration arbitration    0.552296
1      arbitration interpretation    0.541906
2    fraudulent misrepresentation    0.532886
3        agreement interpretation    0.529315
4         fraud misrepresentation    0.528854
..                            ...         ...
333             arbitration scope    0.453503
334       arbitration controversy    0.453503
335          clause unenforceable    0.453357
336            agreement employee    0.453308
337              agreement follow    0.453300

[338 rows x 2 columns]

word_frequency_dict = {}
for cat_id in final_cat_id:
    w_f = Counter(' '.join(rel_phrases_dict[cat_id].phrase).split())
    word_frequency_dict[cat_id] = pd.DataFrame(
        {'word': w_f.keys(), 'frequency': w_f.values()}).sort_values('frequency', ascending=False)
word_frequency_dict['16'].query('frequency > 6')

>>>
                word  frequency
0        arbitration         63
1       jurisdiction         58
2          agreement         39
3     interpretation         26
4               that         10
5      determination          9
6          provision          9
7  misrepresentation          8
8        termination          8
9           contract          7

doc_to_word_dict = {}
for cat_id in tqdm(final_cat_id):
    doc_to_word_dict[cat_id] = get_corelevance_matrix(
        prepared_texts[cat_id], word_frequency_dict[cat_id].query('frequency > 6').word, k=1
    )
    
relevant_doc_id_dict = {}
for cat_id in final_cat_id:
    doc_to_word = doc_to_word_dict[cat_id]
    word_frequency = word_frequency_dict[cat_id].query('frequency > 6')
    relevant_doc_id_dict[cat_id] = np.argsort(
        doc_to_word.max(axis=1)
    )[-int(doc_to_word.shape[0] * 0.1):][::-1]

print(category_phrases['16'])
>>>
                        phrase  mean_score
0            judgment judgment    0.415170
1        agreement termination    0.412065
2           agreement judgment    0.402458
3       determination district    0.401826
4       contract determination    0.399752
5          agreement agreement    0.398007
6         agreement obligation    0.397804
7     agreement interpretation    0.397457
8          agreement provision    0.397129
9        provision termination    0.392126
10    interpretation provision    0.389592
11             judgment motion    0.389397
12        contract termination    0.388398
13       agreement contractual    0.387740
14          agreement question    0.387402
15  application interpretation    0.387228
16  contractual interpretation    0.386429
17      contractual obligation    0.385691
18         court determination    0.384388
19       agreement application    0.384235
20         provision provision    0.384006
21           contract judgment    0.383274
22          provision question    0.383173
23   interpretation reasonable    0.382953
24     contract interpretation    0.382451
25        determination regard    0.382304
26      agreement construction    0.382123
27          determination make    0.381870
28           decision judgment    0.381465
29          agreement contract    0.380993
30        district termination    0.379727
31          agreement district    0.379490
32        agreement reasonable    0.378426
33          addition agreement    0.377187
34      consideration district    0.376678
35       contractual provision    0.376262
36           question question    0.374988
37        provision reasonable    0.374732
38         agreement violation    0.374507
39       determination factual    0.374282
40           agreement include    0.373976
41           agreement provide    0.373344
42             action judgment    0.372167
43          determination upon    0.371592
44          citation quotation    0.371462
45        district requirement    0.371067
46          district provision    0.370348
47          contract provision    0.369338
48        limitation provision    0.368249
49            date termination    0.367801
50          obligation provide    0.367627
51    district reconsideration    0.367293
52         district litigation    0.366763
53  enforcement interpretation    0.365494
54      motion reconsideration    0.365338
**********

new_relevance_matrix_dict = {}
for cat_id in final_cat_id:
    rel_phrases = rel_phrases_dict_[cat_id]
    matrix = bow_matrices[cat_id]
    substrings = bow_phrases[cat_id]

    new_relevance_matrix_dict[cat_id] = matrix[:, [substrings.index(el) for el in rel_phrases.phrase.to_list()]]

corelevance_matrix_dict = {}
for cat_id in final_cat_id:
    rel_matrix = new_relevance_matrix_dict[cat_id]
    n_relevant_for_texts = (rel_matrix > 0.28).sum(axis=1)
    corelevance_matrix_dict[cat_id] = (rel_matrix.T / n_relevant_for_texts) @ rel_matrix

def transition_idx(cbs, thrsh=1.5):
    trs = np.where((cbs[:-1] / cbs[1:]) >= thrsh)[0]
    if len(trs) == 0:
        return len(cbs)
    return trs[0] + 1

def threshold_idx(cbs, thrsh=0.01):
    trs = np.where(cbs < thrsh)[0]
    if len(trs) == 0:
        return len(cbs)
    return trs[0]

for cat_id in final_cat_id:
    original_stdout = sys.stdout
    rel_phrases = rel_phrases_dict_[cat_id]
    annotations = rel_phrases.phrase
    B, member, contrib, intensity, lat, tt = faddis_dict[cat_id]
    idx = np.argsort(contrib)[::-1]
    i_ = min(
        threshold_idx(contrib[idx], 0.01),  # threshold 
        transition_idx(contrib[idx], 1.5)  # transition
    )
    with open(f'../RES/fuzzy_clusters/{cat_id}_clusters.txt', 'w', encoding='utf-8') as f:
        sys.stdout = f
        print(f'Category {cat_id};', f'{i_} clusters')
        print('__' * 50)
        for cl_idx, (cluster, cb) in enumerate(zip(member[:, idx].T, contrib[idx])):
            print(f'Cluster {cl_idx}; contribution {cb:.4f}:')
            print([(phrase, f"{val:.4f}") for phrase, val in list(sorted(zip(annotations, cluster.flat),
                                                                         key=itemgetter(1), reverse=True)) if
                   val > 0.21])
            print('\n')
    sys.stdout = original_stdout

faddis_dict = np.load('faddis_dict.npy', allow_pickle=True).item()
B, member, contrib, intensity, lat, tt = faddis_dict['16']