diff --git a/openshift_metrics/merge.py b/openshift_metrics/merge.py index 4a89040..6c767ef 100644 --- a/openshift_metrics/merge.py +++ b/openshift_metrics/merge.py @@ -2,6 +2,7 @@ Merges metrics from files and produces reports by pod and by namespace """ +import logging import os import argparse from datetime import datetime, UTC @@ -13,6 +14,9 @@ from openshift_metrics import utils, invoice from openshift_metrics.metrics_processor import MetricsProcessor +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + def compare_dates(date_str1, date_str2): """Returns true is date1 is earlier than date2""" date1 = datetime.strptime(date_str1, "%Y-%m-%d") @@ -96,14 +100,15 @@ def main(): elif compare_dates(report_end_date, metrics_from_file["end_date"]): report_end_date = metrics_from_file["end_date"] - print(report_start_date) - print(report_end_date) + logger.info(f"Generating report from {report_start_date} to {report_end_date}") + report_start_date = datetime.strptime(report_start_date, "%Y-%m-%d") report_end_date = datetime.strptime(report_end_date, "%Y-%m-%d") report_month = datetime.strftime(report_start_date, "%Y-%m") if args.use_nerc_rates: + logger.info("Using nerc rates.") nerc_data = nerc_rates.load_from_url() rates = invoice.Rates( cpu=Decimal(nerc_data.get_value_at("CPU SU Rate", report_month)), @@ -130,7 +135,7 @@ def main(): pod_report_file = f"Pod NERC OpenShift {report_month}.csv" if report_start_date.month != report_end_date.month: - print("Warning: The report spans multiple months") + logger.warning("The report spans multiple months") report_month += " to " + datetime.strftime(report_end_date, "%Y-%m") condensed_metrics_dict = processor.condense_metrics( diff --git a/openshift_metrics/openshift_prometheus_metrics.py b/openshift_metrics/openshift_prometheus_metrics.py index 679bb24..e82a3d9 100755 --- a/openshift_metrics/openshift_prometheus_metrics.py +++ b/openshift_metrics/openshift_prometheus_metrics.py @@ -18,11 +18,15 @@ import os import sys import json +import logging from openshift_metrics import utils from openshift_metrics.prometheus_client import PrometheusClient from openshift_metrics.metrics_processor import MetricsProcessor +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + CPU_REQUEST = 'kube_pod_resource_request{unit="cores"} unless on(pod, namespace) kube_pod_status_unschedulable' MEMORY_REQUEST = 'kube_pod_resource_request{unit="bytes"} unless on(pod, namespace) kube_pod_status_unschedulable' GPU_REQUEST = 'kube_pod_resource_request{resource=~"nvidia.com.*"} unless on(pod, namespace) kube_pod_status_unschedulable' @@ -71,8 +75,7 @@ def main(): else: output_file = f"metrics-{report_start_date}-to-{report_end_date}.json" - print(f"Generating report starting {report_start_date} and ending {report_end_date} in {output_file}") - + logger.info(f"Generating report starting {report_start_date} and ending {report_end_date} in {output_file}") token = os.environ.get("OPENSHIFT_TOKEN") prom_client = PrometheusClient(openshift_url, token) @@ -98,6 +101,7 @@ def main(): node_labels = prom_client.query_metric(KUBE_NODE_LABELS, report_start_date, report_end_date) metrics_dict["gpu_metrics"] = MetricsProcessor.insert_node_labels(node_labels, gpu_request_metrics) except utils.EmptyResultError: + logger.info(f"No GPU metrics found for the period {report_start_date} to {report_end_date}") pass month_year = datetime.strptime(report_start_date, "%Y-%m-%d").strftime("%Y-%m") @@ -108,6 +112,7 @@ def main(): s3_location = f"data_{month_year}/metrics-{report_start_date}-to-{report_end_date}.json" with open(output_file, "w") as file: + logger.info(f"Writing metrics to {output_file}") json.dump(metrics_dict, file) if args.upload_to_s3: diff --git a/openshift_metrics/prometheus_client.py b/openshift_metrics/prometheus_client.py index c64708a..ef3a79c 100644 --- a/openshift_metrics/prometheus_client.py +++ b/openshift_metrics/prometheus_client.py @@ -1,10 +1,14 @@ import requests import time +import logging from urllib3.util.retry import Retry from requests.adapters import HTTPAdapter from openshift_metrics.utils import EmptyResultError +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + class PrometheusClient: def __init__(self, prometheus_url: str, token: str, step_min: int=15): self.prometheus_url = prometheus_url @@ -22,7 +26,7 @@ def query_metric(self, metric, start_date, end_date): session = requests.Session() session.mount("https://", HTTPAdapter(max_retries=retries)) - print(f"Retrieving metric: {metric}") + logger.info(f"Retrieving metric: {metric}") for _ in range(3): response = session.get(url, headers=headers, verify=True) @@ -33,7 +37,7 @@ def query_metric(self, metric, start_date, end_date): data = response.json()["data"]["result"] if data: break - print("Empty result set") + logger.warning("Empty result set") time.sleep(3) if not data: diff --git a/openshift_metrics/utils.py b/openshift_metrics/utils.py index 58e2cb2..9b8c1d4 100755 --- a/openshift_metrics/utils.py +++ b/openshift_metrics/utils.py @@ -17,10 +17,14 @@ import csv import requests import boto3 +import logging from openshift_metrics import invoice from decimal import Decimal +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + class EmptyResultError(Exception): """Raise when no results are retrieved for a query""" @@ -69,7 +73,7 @@ def upload_to_s3(file, bucket, location): aws_access_key_id=s3_key_id, aws_secret_access_key=s3_secret, ) - + logger.info(f"Uploading {file} to s3://{bucket}/{location}") response = s3.upload_file(file, Bucket=bucket, Key=location) @@ -104,7 +108,7 @@ def get_namespace_attributes(): def csv_writer(rows, file_name): """Writes rows as csv to file_name""" - print(f"Writing csv to {file_name}") + logger.info(f"Writing report to {file_name}") with open(file_name, "w") as csvfile: csvwriter = csv.writer(csvfile) csvwriter.writerows(rows)