From 51c78028f1f982c891d9c009c3583ae185615127 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 10 Apr 2022 21:43:20 +0000 Subject: [PATCH] Use python logging --- alice-ci/src/alice/cli.py | 12 +++++--- alice-ci/src/alice/configparser.py | 21 ++++++-------- alice-ci/src/alice/runnerfactory.py | 19 +++++-------- alice-ci/src/alice/runners/pypirunner.py | 13 ++++----- alice-ci/src/alice/runners/pythonrunner.py | 33 +++++++++------------- alice-ci/src/alice/runners/pyutils.py | 16 +++++------ 6 files changed, 49 insertions(+), 65 deletions(-) diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py index bfe3efb..5cfc193 100644 --- a/alice-ci/src/alice/cli.py +++ b/alice-ci/src/alice/cli.py @@ -1,3 +1,4 @@ +import logging import os import argparse @@ -20,9 +21,8 @@ def parse_jobs(args): try: if len(args.env) > 0: envs = gen_env(args.env) - if args.verbose: - print(f"[Alice] Env vars from CLI: {envs}") - jobParser = ConfigParser(args.input, gen_env(args.env), args.verbose) + logging.debug(f"[Alice] Env vars from CLI: {envs}") + jobParser = ConfigParser(args.input, gen_env(args.env)) for step in args.steps: jobParser.execute(step) @@ -42,8 +42,12 @@ def main(): parser.add_argument("-i", "--input", default="alice-ci.yaml") parser.add_argument("-e", "--env", nargs='*', default=[]) parser.add_argument("-a", "--addrunner", nargs='*', default=[]) - parser.add_argument("-v", "--verbose", action='store_true') + parser.add_argument('--verbose', '-v', action='count', default=0) args = parser.parse_args() + + loglevel = 30 - ((10 * args.verbose) if args.verbose > 0 else 0) + logging.basicConfig(level=loglevel, format='%(message)s') + if not os.path.isfile(args.input): print(f"No such file: {args.input}") exit(1) diff --git a/alice-ci/src/alice/configparser.py b/alice-ci/src/alice/configparser.py index bd4f698..f28c083 100644 --- a/alice-ci/src/alice/configparser.py +++ b/alice-ci/src/alice/configparser.py @@ -1,3 +1,4 @@ +import logging from os import getcwd, path, environ import subprocess import yaml @@ -7,11 +8,10 @@ from alice.runnerfactory import Factory class ConfigParser: - def __init__(self, file_path, cli_env_vars, verbose=False) -> None: - self.verbose = verbose + def __init__(self, file_path, cli_env_vars) -> None: with open(file_path) as f: self.config = yaml.safe_load(f) - self.factory = Factory(verbose, self.__gen_globals(cli_env_vars), self.config.get("runners", {})) + self.factory = Factory(self.__gen_globals(cli_env_vars), self.config.get("runners", {})) self.jobs = self.__get_jobs() self.pipelines = self.config.get("pipelines", {}) @@ -31,8 +31,7 @@ class ConfigParser: if "workdir" in self.config["runners"]["global"]: globals["workdir"] = self.config["runners"]["global"]["workdir"] - if (self.verbose): - print(f"[Alice] Configured globals: {globals}") + logging.debug(f"[Alice] Configured globals: {globals}") return globals def __get_jobs(self): @@ -44,8 +43,7 @@ class ConfigParser: raise ConfigException(f"Job with name {name} already exists!") jobs[name] = job_spec - if (self.verbose): - print(f"[Alice] Parsed jobs: {', '.join(jobs.keys())}") + logging.info(f"[Alice] Parsed jobs: {', '.join(jobs.keys())}") return jobs else: raise ConfigException("No jobs defined in config") @@ -65,9 +63,8 @@ class ConfigParser: for _path in paths: spec_path = path.abspath(_path) if change_path.startswith(spec_path): - if self.verbose: - print(f"[Alice] Modified file: {change_path}") - print(f"[Alice] Path match: {_path}") + logging.info(f"[Alice] Modified file: {change_path}") + logging.info(f"[Alice] Path match: {_path}") return True except KeyError: raise ConfigException(f"Invalid 'changes' config: {changes}") @@ -77,16 +74,16 @@ class ConfigParser: if task_name in self.jobs: self.execute_job(task_name) elif task_name in self.pipelines: - print(f"[Alice][Pipeline] {task_name}: Start") self.execute_pipeline(task_name) - print(f"[Alice][Pipeline] {task_name}: Success") else: raise ConfigException(f"No such job or pipeline: {task_name}") def execute_pipeline(self, pipeline_name): if pipeline_name in self.pipelines: + print(f"[Alice][Pipeline] {pipeline_name}: Start") for job in self.pipelines[pipeline_name]: self.execute_job(job) + print(f"[Alice][Pipeline] {pipeline_name}: Success") def execute_job(self, job_name): if job_name in self.jobs: diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py index 618248c..e587891 100644 --- a/alice-ci/src/alice/runnerfactory.py +++ b/alice-ci/src/alice/runnerfactory.py @@ -1,3 +1,4 @@ +import logging from os.path import join, abspath from alice.runners.pythonrunner import PythonRunner @@ -6,8 +7,7 @@ from alice.exceptions import ConfigException class Factory(): - def __init__(self, verbose, globals, runner_configs) -> None: - self.verbose = verbose + def __init__(self, globals, runner_configs) -> None: self.globals = globals self.runner_configs = {} self.runnertypes = {} @@ -23,14 +23,12 @@ class Factory(): self.runnertypes = {"python": PythonRunner, "pypi": PyPiRunner} - if (self.verbose): - print(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}") + logging.info(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}") def __gen_runner_configs(self, config): for runnertype, runnerconfig in config.items(): if runnertype != "global": - if (self.verbose): - print(f"[Alice] Global config found for runner {runnertype}") + logging.info(f"[Alice] Global config found for runner {runnertype}") config = self.globals.copy() for key, value in runnerconfig.items(): if key == "env": @@ -41,18 +39,15 @@ class Factory(): else: config[key] = value self.runner_configs[runnertype] = config + logging.debug(f"[Alice] Globals for {runnertype}: {runnerconfig}") def get_runner(self, runnertype): if runnertype not in self.runners: if runnertype in self.runnertypes: - if (self.verbose): - print(f"[Alice] Initializing runner: {runnertype}") - params = { - "verbose": self.verbose - } + logging.info(f"[Alice] Initializing runner: {runnertype}") # If there is a runner specific config, use that, else global config = self.runner_configs.get(runnertype, self.globals.copy()) - self.runners[runnertype] = self.runnertypes[runnertype](params, config) + self.runners[runnertype] = self.runnertypes[runnertype](config) else: raise ConfigException(f"Invalid runner type: {runnertype}") return self.runners[runnertype] diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py index fb821c3..5f0879c 100644 --- a/alice-ci/src/alice/runners/pypirunner.py +++ b/alice-ci/src/alice/runners/pypirunner.py @@ -1,4 +1,5 @@ import json +import logging import os import re import subprocess @@ -75,10 +76,8 @@ class PypiConfig: # TODO: consider "--skip-existing" flag for twine class PyPiRunner(): - def __init__(self, params, config) -> None: - self.verbose = params["verbose"] - if self.verbose: - print("[PyPiRunner] Initializing") + def __init__(self, config) -> None: + logging.info("[PyPiRunner] Initializing") self.workdir = config["workdir"] self.config = PypiConfig(config) @@ -126,9 +125,7 @@ class PyPiRunner(): def upload(self, config, package): - command = [sys.executable, "-m", "twine", "upload"] - if self.verbose: - command.append("--verbose") + command = [sys.executable, "-m", "twine", "upload", "--verbose"] if config.repo_uri is not None: command.append("--repository-url") command.append(config.repo_uri) @@ -163,7 +160,7 @@ class PyPiRunner(): PackageManager.getInstance().ensure("build") for package in job_config.packages: print(f"[PyPiRunner] Building {package}") - #self.build(job_config, package) + self.build(job_config, package) print(f"[PyPiRunner] Package {package} built") if job_config.upload: diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index 18011fc..f23bf8d 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -1,3 +1,4 @@ +import logging import subprocess import os import sys @@ -9,10 +10,8 @@ from alice.runners.pyutils import glob_command # TODO: Handle config like PyPiConfig class PythonRunner: - def __init__(self, params, config) -> None: - self.verbose = params["verbose"] - if self.verbose: - print("[PythonRunner] Initializing") + def __init__(self, config) -> None: + logging.info("[PythonRunner] Initializing") self.workdir = config["workdir"] self.virtual_dir = os.path.abspath(os.path.join(self.workdir, "venv")) self.config = config @@ -25,7 +24,7 @@ class PythonRunner: self.vpython = os.path.join(self.virtual_dir, "bin", "python3") if not os.path.exists(self.vpython): - print("[PythonRunner] Initializing venv") + logging.info("[PythonRunner] Initializing venv") with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: p.wait() @@ -33,26 +32,22 @@ class PythonRunner: sys.stdout.buffer.write(p.stderr.read()) raise RunnerError("[PythonRunner] Could not create virtualenv") else: - if self.verbose: - print(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}") + logging.info(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}") else: - if self.verbose: - print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}") + logging.info(f"[PythonRunner] Found virtualenv at {self.virtual_dir}") dependencies = self.config.get("dependencies", []) if len(dependencies) > 0: - if self.verbose: - print(f"[PythonRunner] Ensuring dependencies: {', '.join(dependencies)}") + logging.info(f"[PythonRunner] Ensuring dependencies: {', '.join(dependencies)}") command = [self.vpython, "-m", "pip", "install"] + dependencies with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: p.wait() if p.returncode != 0: sys.stdout.buffer.write(p.stderr.read()) raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})")) - if self.verbose: - print("[PythonRunner] Installation done") + logging.info("[PythonRunner] Installation done") # Executes the given job in the one and only venv - # parameter shall be the raw jobscpec + # parameter is the raw jobscpec def run(self, job_spec): if "workdir" in job_spec: pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"])) @@ -65,16 +60,14 @@ class PythonRunner: if "commands" in job_spec: commands = job_spec["commands"] for command in commands: - if self.verbose: - print(f"[PythonRunner] Raw command: {command}") + logging.debug(f"[PythonRunner] Raw command: {command}") # TODO: only split if command is not an array if "*" in command: - run_command = glob_command(shlex.split(command), pwd, self.verbose) + run_command = glob_command(shlex.split(command), pwd) else: run_command = shlex.split(command) - if self.verbose: - print(f"[PythonRunner] Command to execute: {run_command}") - print(f"[PythonRunner] Workdir: {pwd}") + logging.info(f"[PythonRunner] Command to execute: {run_command}") + logging.debug(f"[PythonRunner] Workdir: {pwd}") if os.path.isdir(pwd): with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p: p.wait() diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py index 62efdf7..6b3f6f1 100644 --- a/alice-ci/src/alice/runners/pyutils.py +++ b/alice-ci/src/alice/runners/pyutils.py @@ -1,3 +1,4 @@ +import logging import os import subprocess import sys @@ -83,11 +84,10 @@ class PackageManager: return False -def glob(item, workdir, verbose=False): +def glob(item, workdir): new_command = [] if "*" in item: - if verbose: - print(f"[Globbing] Found item: [{item}]") + logging.debug(f"[Globbing] Found item: [{item}]") dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item))) base_name = os.path.basename(item) if os.path.isdir(dir): @@ -96,8 +96,7 @@ def glob(item, workdir, verbose=False): # TODO: Fix ordering! A*B = B*A = AB* if item_parts[0] in file and item_parts[1] in file: new_item = os.path.join(dir, file) - if verbose: - print(f"[Globbing] Substitute: {new_item}") + logging.debug(f"[Globbing] Substitute: {new_item}") new_command.append(new_item) else: raise ConfigException(f"[Globbing] Dir not exists: {dir}") @@ -106,10 +105,9 @@ def glob(item, workdir, verbose=False): return [item] -def glob_command(command, workdir, verbose=False): - if verbose: - print(f"[Globbing] Starting command: {' '.join(command)}") +def glob_command(command, workdir): + logging.debug(f"[Globbing] Starting command: {' '.join(command)}") new_command = [] for item in command: - new_command += glob(item, workdir, verbose) + new_command += glob(item, workdir) return new_command