From ef71611c11394438bafbb67157ec5d5272f06eb4 Mon Sep 17 00:00:00 2001 From: Brandon Miller Date: Wed, 14 Jun 2023 16:16:42 -0400 Subject: [PATCH 1/5] Logging to Excel file when an item is locked. --- link_fixer.py | 46 ++++++++++++++++++++++++++++++++++++++++++++-- requirements.txt | 1 + 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/link_fixer.py b/link_fixer.py index d060a65..98698da 100644 --- a/link_fixer.py +++ b/link_fixer.py @@ -1,12 +1,14 @@ import configparser import datetime import getpass +import json import logging import os import sys import time import warnings import urllib.parse as urlparse +import openpyxl from bs4 import BeautifulSoup from halo import Halo @@ -208,6 +210,27 @@ def get_synced_item(item_id, project_id): return None +def start_workbook(): + # Create workbook using openpyxl + workbook = openpyxl.Workbook() + sheet = workbook.active + + # Write the headers + sheet['A1'] = "Item Name" + sheet['B1'] = "Item ID" + sheet['C1'] = "Broken Link/Description" + + return workbook + + +def log_locked_items(workbook, item_name, item_id, description): + sheet = workbook.active + # Check for item lock + if client.get_item_lock(item_id): + row = [item_name, item_id, description] + sheet.append(row) + + # link fixer script, will identify broken links from old projects, and correct the links # a link to the past if __name__ == '__main__': @@ -216,6 +239,8 @@ def get_synced_item(item_id, project_id): logger = init_logger() start_time = time.time() + workbook = start_workbook() # Opens Excel file to write locked items to + logger.info('Running link fixer script') config = configparser.ConfigParser() @@ -272,10 +297,20 @@ def get_synced_item(item_id, project_id): """ STEP TWO - iterate over all the retrieved items and find bad links """ + broken_link_map = {} for item in items: + item_name = item.get('name') item_id = item.get('id') fields = item.get('fields') + name_field_for_excel = item['fields']['name'] + link_in_description = item['fields']['description'] + + # Getting lock properties, so we don't need to call the API multiple times for Excel logging + item_lock_properties = (item.get('lock')) + item_lock_json = json.dumps(item_lock_properties) + item_lock_dict = json.loads(item_lock_json) + for key in fields: original_value = fields[key] value = fields[key] @@ -419,6 +454,11 @@ def get_synced_item(item_id, project_id): corrected_hyperlink_string = corrected_hyperlink_string.replace('docId=' + str(linked_item_id), 'docId=' + str( corrected_item_id)) + # Before we replace the hyperlink, let's check if it's locked and log it to Excel if so + if item_lock_dict['locked']: + print("Item was locked and has a broken link. Logging to Excel File...\n") + log_locked_items(workbook, name_field_for_excel, item_id, link_in_description) + # if we have made it this far then let's go ahead and replace the hyperlink if hyperlink_string in value: value = value.replace(hyperlink_string, corrected_hyperlink_string) @@ -442,7 +482,7 @@ def get_synced_item(item_id, project_id): # we have a bad link for this item? if bad_link_found: - # let's build out an object of all the data we car about for patching and logging + # let's build out an object of all the data we care about for patching and logging broken_link_data = { 'fieldName': key, 'newValue': value, @@ -456,6 +496,8 @@ def get_synced_item(item_id, project_id): broken_list.append(broken_link_data) broken_link_map[item_id] = broken_list + workbook.save("locked_items.xlsx") + """ STEP THREE - fix and log all broken hyperlinks """ @@ -504,4 +546,4 @@ def get_synced_item(item_id, project_id): # were done here elapsed_time = '%.2f' % (time.time() - start_time) - print('total execution time: ' + elapsed_time + ' seconds') + print('total execution time: ' + elapsed_time + ' seconds') \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 44fd689..d68706f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ soupsieve==2.2.1 spinners==0.0.24 termcolor==1.1.0 urllib3==1.26.5 +openpyxl~=3.1.2 \ No newline at end of file From 6b892a2138fb74190ca97480fb5b4e2c093df409 Mon Sep 17 00:00:00 2001 From: Brandon Miller Date: Fri, 16 Jun 2023 09:57:39 -0400 Subject: [PATCH 2/5] One row per locked item and some other code cleanup --- link_fixer.py | 52 ++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/link_fixer.py b/link_fixer.py index 98698da..e505b71 100644 --- a/link_fixer.py +++ b/link_fixer.py @@ -218,17 +218,19 @@ def start_workbook(): # Write the headers sheet['A1'] = "Item Name" sheet['B1'] = "Item ID" - sheet['C1'] = "Broken Link/Description" + sheet['C1'] = "Field with Broken Link" + sheet['D1'] = "Broken Hyperlink" return workbook -def log_locked_items(workbook, item_name, item_id, description): +def log_locked_items(workbook, item_name, item_id, key, hyperlink): sheet = workbook.active # Check for item lock if client.get_item_lock(item_id): - row = [item_name, item_id, description] + row = [item_name, item_id, key, hyperlink] sheet.append(row) + logger.info("Item {} is locked and was added to the Excel workbook.") # link fixer script, will identify broken links from old projects, and correct the links @@ -297,10 +299,8 @@ def log_locked_items(workbook, item_name, item_id, description): """ STEP TWO - iterate over all the retrieved items and find bad links """ - broken_link_map = {} for item in items: - item_name = item.get('name') item_id = item.get('id') fields = item.get('fields') name_field_for_excel = item['fields']['name'] @@ -454,10 +454,6 @@ def log_locked_items(workbook, item_name, item_id, description): corrected_hyperlink_string = corrected_hyperlink_string.replace('docId=' + str(linked_item_id), 'docId=' + str( corrected_item_id)) - # Before we replace the hyperlink, let's check if it's locked and log it to Excel if so - if item_lock_dict['locked']: - print("Item was locked and has a broken link. Logging to Excel File...\n") - log_locked_items(workbook, name_field_for_excel, item_id, link_in_description) # if we have made it this far then let's go ahead and replace the hyperlink if hyperlink_string in value: @@ -480,23 +476,29 @@ def log_locked_items(workbook, item_name, item_id, description): hyperlink_string = start_link + encoded_name + end_link value = value.replace(hyperlink_string, corrected_hyperlink_string) - # we have a bad link for this item? - if bad_link_found: - # let's build out an object of all the data we care about for patching and logging - broken_link_data = { - 'fieldName': key, - 'newValue': value, - 'oldValue': original_value, - 'counter': str(bad_link_count) - } - broken_list = broken_link_map.get(item_id) - if broken_list is None: - broken_list = [broken_link_data] - else: - broken_list.append(broken_link_data) - broken_link_map[item_id] = broken_list + # we have a bad link for this item? + if bad_link_found: + # Before we replace the hyperlink, let's check if it's locked and log it to Excel if so + if item_lock_dict['locked']: + print("Item was locked and has a broken link. Logging to Excel File...\n") + log_locked_items(workbook, name_field_for_excel, str(item_id), key, str(hyperlink)) - workbook.save("locked_items.xlsx") + # let's build out an object of all the data we care about for patching and logging + else: + broken_link_data = { + 'fieldName': key, + 'newValue': value, + 'oldValue': original_value, + 'counter': str(bad_link_count) + } + broken_list = broken_link_map.get(item_id) + if broken_list is None: + broken_list = [broken_link_data] + else: + broken_list.append(broken_link_data) + broken_link_map[item_id] = broken_list + + workbook.save("locked_items.xlsx") """ STEP THREE - fix and log all broken hyperlinks From 1881bc74196ae822d506682cce64158937043719 Mon Sep 17 00:00:00 2001 From: Brandon Miller Date: Thu, 22 Jun 2023 16:43:27 -0400 Subject: [PATCH 3/5] Removed tuple, added functionality to check for locked items after script starts, and broken link map adjustments/logging for that. --- link_fixer.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/link_fixer.py b/link_fixer.py index e505b71..aa6391b 100644 --- a/link_fixer.py +++ b/link_fixer.py @@ -307,9 +307,9 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): link_in_description = item['fields']['description'] # Getting lock properties, so we don't need to call the API multiple times for Excel logging - item_lock_properties = (item.get('lock')) - item_lock_json = json.dumps(item_lock_properties) - item_lock_dict = json.loads(item_lock_json) + item_lock_properties = item.get('lock') + # item_lock_json = json.dumps(item_lock_properties) + # item_lock_dict = json.loads(item_lock_json) for key in fields: original_value = fields[key] @@ -479,7 +479,7 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): # we have a bad link for this item? if bad_link_found: # Before we replace the hyperlink, let's check if it's locked and log it to Excel if so - if item_lock_dict['locked']: + if item_lock_properties['locked']: print("Item was locked and has a broken link. Logging to Excel File...\n") log_locked_items(workbook, name_field_for_excel, str(item_id), key, str(hyperlink)) @@ -489,7 +489,9 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): 'fieldName': key, 'newValue': value, 'oldValue': original_value, - 'counter': str(bad_link_count) + 'counter': str(bad_link_count), + 'name': str(name_field_for_excel), + 'itemId': str(item_id) } broken_list = broken_link_map.get(item_id) if broken_list is None: @@ -498,8 +500,6 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): broken_list.append(broken_link_data) broken_link_map[item_id] = broken_list - workbook.save("locked_items.xlsx") - """ STEP THREE - fix and log all broken hyperlinks """ @@ -533,12 +533,20 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): # let's try and patch this data try: client.patch_item(item_id, patch_list) - logger.info('Successfully patched item [' + str(item_id) + ']') - + logger.info('Successfully patched item [' + b.get('name') + ']') except APIException as error: - # failed to patch - logger.error('Failed to patched item [' + str(item_id) + ']') - logger.error('API exception response: ' + str(error)) + if "locked" in str(error): + try: + log_locked_items(workbook, str(b.get('name')), str(b.get('itemId')), str(b.get('fieldName')), + str(logger_old_value)) + print("Log locked items method successful for " + str(b.get('name'))) + except Exception as e: + logger.error('Failed to log locked items for [' + str(b.get('name')) + ']') + logger.error('Error: ' + str(e)) + else: + # Failed to patch + logger.error('Failed to patch item [' + str(b.get('name')) + ']') + logger.error('API exception response: ' + str(error)) bar.next() bar.finish() @@ -546,6 +554,8 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): else: logger.info('There are zero links to be corrected, exiting...') + workbook.save("locked_items.xlsx") + # were done here elapsed_time = '%.2f' % (time.time() - start_time) - print('total execution time: ' + elapsed_time + ' seconds') \ No newline at end of file + print('total execution time: ' + elapsed_time + ' seconds') From 0ded5e1ba1df0979e06d4bb025c39742323ea0a3 Mon Sep 17 00:00:00 2001 From: Brandon Miller Date: Fri, 23 Jun 2023 10:13:57 -0400 Subject: [PATCH 4/5] Small logging change request to format item name. --- link_fixer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/link_fixer.py b/link_fixer.py index aa6391b..09dccc4 100644 --- a/link_fixer.py +++ b/link_fixer.py @@ -230,7 +230,7 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): if client.get_item_lock(item_id): row = [item_name, item_id, key, hyperlink] sheet.append(row) - logger.info("Item {} is locked and was added to the Excel workbook.") + logger.info("Item {} is locked and was added to the Excel workbook.".format(item_name)) # link fixer script, will identify broken links from old projects, and correct the links From ccf51e4ffba133ba29c79f35ece3339508897d10 Mon Sep 17 00:00:00 2001 From: Brandon Miller Date: Mon, 26 Jun 2023 16:39:12 -0400 Subject: [PATCH 5/5] Updated jamaclient version and fixed print statements to be logger.info statements --- link_fixer.py | 14 +++++++------- requirements.txt | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/link_fixer.py b/link_fixer.py index 09dccc4..bef8202 100644 --- a/link_fixer.py +++ b/link_fixer.py @@ -33,9 +33,9 @@ def init_jama_client(): return jama_client except APIException: # we cant do things without the API so let's kick out of the execution. - print('Error: invalid Jama credentials, check they are valid in the config.ini file.') + logger.info('Error: invalid Jama credentials, check they are valid in the config.ini file.') except: - print('Failed to authenticate to <' + get_instance_url(credentials_dict) + '>') + logger.info('Failed to authenticate to <' + get_instance_url(credentials_dict) + '>') response = input('\nWould you like to manually enter server credentials?\n') response = response.lower() @@ -294,7 +294,7 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): spinner.start() items = client.get_items(project_id) spinner.stop() - print('Retrieving ' + str(len(items)) + ' items from project ID:[' + str(project_id) + ']') + logger.info('Retrieving ' + str(len(items)) + ' items from project ID:[' + str(project_id) + ']') """ STEP TWO - iterate over all the retrieved items and find bad links @@ -480,7 +480,7 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): if bad_link_found: # Before we replace the hyperlink, let's check if it's locked and log it to Excel if so if item_lock_properties['locked']: - print("Item was locked and has a broken link. Logging to Excel File...\n") + logger.info("Item was locked and has a broken link. Logging to Excel File...\n") log_locked_items(workbook, name_field_for_excel, str(item_id), key, str(hyperlink)) # let's build out an object of all the data we care about for patching and logging @@ -539,7 +539,7 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): try: log_locked_items(workbook, str(b.get('name')), str(b.get('itemId')), str(b.get('fieldName')), str(logger_old_value)) - print("Log locked items method successful for " + str(b.get('name'))) + logger.info("Log locked items method successful for " + str(b.get('name'))) except Exception as e: logger.error('Failed to log locked items for [' + str(b.get('name')) + ']') logger.error('Error: ' + str(e)) @@ -550,7 +550,7 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): bar.next() bar.finish() - print('updated ' + str(len(broken_link_map)) + ' link(s)') + logger.info('updated ' + str(len(broken_link_map)) + ' link(s)') else: logger.info('There are zero links to be corrected, exiting...') @@ -558,4 +558,4 @@ def log_locked_items(workbook, item_name, item_id, key, hyperlink): # were done here elapsed_time = '%.2f' % (time.time() - start_time) - print('total execution time: ' + elapsed_time + ' seconds') + logger.info('total execution time: ' + elapsed_time + ' seconds') diff --git a/requirements.txt b/requirements.txt index d68706f..fe9ef70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ halo==0.0.31 idna==2.10 log-symbols==0.0.14 progress==1.5 -py-jama-rest-client==1.16.0 +py-jama-rest-client==1.17.1 requests==2.25.1 six==1.16.0 soupsieve==2.2.1