Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use logging.config and do not pass CLI args as a parameter #73

Merged
merged 7 commits into from
Jun 6, 2024
Merged

Use logging.config and do not pass CLI args as a parameter #73

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7bdf162
Do not pass args and use logging.config
amaslenn Jun 5, 2024
28a2468
Use module-level logging for better configurability
amaslenn Jun 5, 2024
b22f3e8
Do not use print, use logging
amaslenn Jun 5, 2024
b7e3376
Use global logging
amaslenn Jun 5, 2024
6b5de44
Extend logging for DockerImageCacheManager
amaslenn Jun 5, 2024
d396185
Fix tests
amaslenn Jun 5, 2024
1ab4945
Merge branch 'main' into no-args-logging
amaslenn Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 110 additions & 52 deletions src/cloudai/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
import argparse
import asyncio
import logging
import os
import logging.config
import sys
from pathlib import Path
from typing import Optional

from cloudai import Installer, Parser, ReportGenerator, Runner

Expand All @@ -33,12 +35,37 @@ def setup_logging(log_file: str, log_level: str) -> None:
if not isinstance(numeric_level, int):
raise ValueError(f"Invalid log level: {log_level}")

logging.basicConfig(
filename=log_file,
level=numeric_level,
format="%(asctime)s - %(levelname)s - %(message)s",
filemode="w",
)
LOGGING_CONFIG = {
"version": 1,
"disable_existing_loggers": True,
"formatters": {
"standard": {"format": "%(asctime)s - %(levelname)s - %(message)s"},
"short": {"format": "[%(levelname)s] %(message)s"},
},
"handlers": {
"default": {
"level": log_level.upper(),
"formatter": "short",
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout",
},
"debug_file": {
"level": "DEBUG",
"formatter": "standard",
"class": "logging.FileHandler",
"filename": log_file,
"mode": "w",
},
},
"loggers": {
"": {
"handlers": ["default", "debug_file"],
"level": "DEBUG",
"propagate": False,
},
},
}
logging.config.dictConfig(LOGGING_CONFIG)


def parse_arguments() -> argparse.Namespace:
Expand Down Expand Up @@ -100,22 +127,26 @@ def parse_arguments() -> argparse.Namespace:
return parser.parse_args()


def handle_install_and_uninstall(args: argparse.Namespace) -> None:
def handle_install_and_uninstall(
mode: str, system_config_path: Path, test_template_path: Path, output_path: Optional[Path] = None
) -> None:
"""
Manage the installation or uninstallation process for CloudAI.

Based on user-specified mode, utilizing the Installer and Parser classes.

Args:
args (argparse.Namespace): Parsed command-line arguments containing
user preferences.
mode (str): The mode of operation (e.g., install, uninstall).
system_config_path (Path): The path to the system configuration file.
test_template_path (Path): The path to the test template configuration directory.
output_path (Optional[Path]): The path to the output directory.
"""
logging.info("Starting configuration parsing")
parser = Parser(args.system_config_path, args.test_template_path)
parser = Parser(str(system_config_path), str(test_template_path))
system, test_templates = parser.parse_system_and_templates()

if args.output_path:
system.output_path = os.path.abspath(args.output_path)
if output_path:
system.output_path = str(output_path.absolute())

system.update()

Expand All @@ -124,70 +155,81 @@ def handle_install_and_uninstall(args: argparse.Namespace) -> None:

installer = Installer(system)

if args.mode == "install":
if mode == "install":
logging.info("Installing test templates.")
if installer.is_installed(test_templates):
print("CloudAI is already installed.")
logging.info("CloudAI is already installed.")
else:
result = installer.install(test_templates)
if not result:
print(result)
sys.exit(1)
logging.error(result)
exit(1)

elif args.mode == "uninstall":
elif mode == "uninstall":
logging.info("Uninstalling test templates.")
result = installer.uninstall(test_templates)
if not result:
print(result)
logging.error(result)
sys.exit(1)


def handle_dry_run_and_run(args: argparse.Namespace) -> None:
def handle_dry_run_and_run(
mode: str,
system_config_path: Path,
test_template_path: Path,
test_path: Path,
test_scenario_path: Optional[Path] = None,
output_path: Optional[Path] = None,
) -> None:
"""
Execute the dry-run or run modes for CloudAI.

Includes parsing configurations, verifying installations, and executing test scenarios.

Args:
args (argparse.Namespace): Parsed command-line arguments containing
user preferences.
mode (str): The mode of operation (e.g., dry-run, run).
system_config_path (Path): The path to the system configuration file.
test_template_path (Path): The path to the test template configuration directory.
test_path (Path): The path to the test configuration directory.
test_scenario_path (Optional[Path]): The path to the test scenario file.
output_path (Optional[Path]): The path to the output directory.
"""
logging.info("Starting configuration parsing")
parser = Parser(
args.system_config_path,
args.test_template_path,
args.test_path,
args.test_scenario_path,
str(system_config_path),
str(test_template_path),
str(test_path),
str(test_scenario_path) if test_scenario_path else None,
)
system, test_templates, test_scenario = parser.parse()

if args.output_path:
system.output_path = os.path.abspath(args.output_path)
if output_path:
system.output_path = str(output_path.absolute())

system.update()

logging.info(f"System Name: {system.name}")
logging.info(f"Scheduler: {system.scheduler}")
logging.info(f"Test Scenario Name: {test_scenario.name}")

if args.mode == "run":
if mode == "run":
logging.info("Checking if test templates are installed.")
installer = Installer(system)
result = installer.is_installed(test_templates)
if not result:
print("CloudAI has not been installed. Please run install mode first.")
print(result)
sys.exit(1)
logging.error("CloudAI has not been installed. Please run install mode first.")
logging.error(result)
exit(1)

test_scenario.pretty_print()
logging.info(test_scenario.pretty_print())

runner = Runner(args.mode, system, test_scenario)
runner = Runner(mode, system, test_scenario)
asyncio.run(runner.run())

print(f"All test scenario results stored at: {runner.runner.output_path}")
logging.info(f"All test scenario results stored at: {runner.runner.output_path}")

if args.mode == "run":
print(
if mode == "run":
logging.info(
"All test scenario execution attempts are complete. Please review"
" the 'debug.log' file to confirm successful completion or to"
" identify any issues."
Expand All @@ -197,44 +239,60 @@ def handle_dry_run_and_run(args: argparse.Namespace) -> None:
generator.generate_report(test_scenario)


def handle_generate_report(args: argparse.Namespace) -> None:
def handle_generate_report(
system_config_path: Path,
test_template_path: Path,
test_path: Path,
output_path: Path,
test_scenario_path: Optional[Path] = None,
) -> None:
"""
Generate a report based on the existing configuration and test results.

Args:
args (argparse.Namespace): Parsed command-line arguments containing
user preferences.
system_config_path (Path): The path to the system configuration file.
test_template_path (Path): The path to the test template configuration directory.
test_path (Path): The path to the test configuration directory.
output_path (Path): The path to the output directory.
test_scenario_path (Optional[Path]): The path to the test scenario file.
"""
logging.info("Generating report based on system and test templates")
parser = Parser(
args.system_config_path,
args.test_template_path,
args.test_path,
args.test_scenario_path,
str(system_config_path),
str(test_template_path),
str(test_path),
str(test_scenario_path),
)
system, test_templates, test_scenario = parser.parse()

generator = ReportGenerator(args.output_path)
generator = ReportGenerator(str(output_path))
generator.generate_report(test_scenario)

print("Report generation completed.")
logging.info("Report generation completed.")


def main() -> None:
args = parse_arguments()

setup_logging(args.log_file, args.log_level)

if args.mode == "generate-report" and not args.output_path:
print("Error: --output_path is required when mode is generate-report.")
sys.exit(1)
system_config_path = Path(args.system_config_path)
test_template_path = Path(args.test_template_path)
test_path = Path(args.test_path)
test_scenario_path = Path(args.test_scenario_path) if args.test_scenario_path else None
output_path = Path(args.output_path) if args.output_path else None

if args.mode in ["install", "uninstall"]:
handle_install_and_uninstall(args)
handle_install_and_uninstall(args.mode, system_config_path, test_template_path, output_path=output_path)
elif args.mode in ["dry-run", "run"]:
handle_dry_run_and_run(args)
handle_dry_run_and_run(
args.mode, system_config_path, test_template_path, test_path, test_scenario_path, output_path
)
elif args.mode == "generate-report":
handle_generate_report(args)
if not output_path:
logging.error("Error: --output_path is required when mode is generate-report.")
exit(1)
handle_generate_report(system_config_path, test_template_path, test_path, output_path, test_scenario_path)


if __name__ == "__main__":
Expand Down
21 changes: 10 additions & 11 deletions src/cloudai/_core/base_installer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@ def __init__(self, system: System):
system (System): The system schema object.
"""
self.system = system
self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__)
self.logger.info(f"BaseInstaller initialized for {self.system.scheduler}.")
logging.info(f"BaseInstaller initialized for {self.system.scheduler}.")

def _is_binary_installed(self, binary_name: str) -> bool:
"""
Expand All @@ -56,7 +55,7 @@ def _is_binary_installed(self, binary_name: str) -> bool:
Returns:
bool: True if the binary is installed, False otherwise.
"""
self.logger.debug(f"Checking if binary '{binary_name}' is installed.")
logging.debug(f"Checking if binary '{binary_name}' is installed.")
return shutil.which(binary_name) is not None

def _check_prerequisites(self) -> InstallStatusResult:
Expand All @@ -68,7 +67,7 @@ def _check_prerequisites(self) -> InstallStatusResult:
Returns
InstallStatusResult: Result containing the status and any error message.
"""
self.logger.info("Checking for common prerequisites.")
logging.info("Checking for common prerequisites.")
return InstallStatusResult(True)

def is_installed(self, test_templates: Iterable[TestTemplate]) -> InstallStatusResult:
Expand All @@ -83,7 +82,7 @@ def is_installed(self, test_templates: Iterable[TestTemplate]) -> InstallStatusR
Returns:
InstallStatusResult: Result containing the installation status and error message if not installed.
"""
self.logger.info("Verifying installation status of test templates.")
logging.info("Verifying installation status of test templates.")
not_installed = {}
for test_template in test_templates:
result = test_template.is_installed()
Expand All @@ -105,7 +104,7 @@ def install(self, test_templates: Iterable[TestTemplate]) -> InstallStatusResult
Returns:
InstallStatusResult: Result containing the installation status and error message if any.
"""
self.logger.info("Starting installation of test templates.")
logging.info("Starting installation of test templates.")
prerequisites_result = self._check_prerequisites()
if not prerequisites_result:
return prerequisites_result
Expand All @@ -122,14 +121,14 @@ def install(self, test_templates: Iterable[TestTemplate]) -> InstallStatusResult
else:
install_results[test_template.name] = result.message
except Exception as e:
self.logger.error(f"Installation failed for {test_template.name}: {e}")
logging.error(f"Installation failed for {test_template.name}: {e}")
install_results[test_template.name] = str(e)

all_success = all(result == "Success" for result in install_results.values())
if all_success:
return InstallStatusResult(True, "All test templates installed successfully.")
else:
return InstallStatusResult(False, "Some test templates failed to install.", install_results)

return InstallStatusResult(False, "Some test templates failed to install.", install_results)

def uninstall(self, test_templates: Iterable[TestTemplate]) -> InstallStatusResult:
"""
Expand All @@ -141,7 +140,7 @@ def uninstall(self, test_templates: Iterable[TestTemplate]) -> InstallStatusResu
Returns:
InstallStatusResult: Result containing the uninstallation status and error message if any.
"""
self.logger.info("Uninstalling test templates.")
logging.info("Uninstalling test templates.")
uninstall_results = {}
with ThreadPoolExecutor() as executor:
futures = {executor.submit(test_template.uninstall): test_template for test_template in test_templates}
Expand All @@ -154,7 +153,7 @@ def uninstall(self, test_templates: Iterable[TestTemplate]) -> InstallStatusResu
else:
uninstall_results[test_template.name] = result.message
except Exception as e:
self.logger.error(f"Uninstallation failed for {test_template.name}: {e}")
logging.error(f"Uninstallation failed for {test_template.name}: {e}")
uninstall_results[test_template.name] = str(e)

all_success = all(result == "Success" for result in uninstall_results.values())
Expand Down
Loading