diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index 22f8872..6e79421 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = alice-ci -version = 0.0.4 +version = 0.0.5 author = Daniel Gyulai description = Alice CI framework long_description = file: README.md diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py index 24308b4..4c0140b 100644 --- a/alice-ci/src/alice/cli.py +++ b/alice-ci/src/alice/cli.py @@ -19,10 +19,13 @@ def gen_env(self, param_list): def parse_jobs(args): try: - factory = Factory() + factory = Factory(args.verbose) if len(args.env) > 0: - factory.update_runners({"env": gen_env(args.env)}) - jobParser = ConfigParser(args.input, factory) + envs = gen_env(args.env) + if args.verbose: + print(f"[Alice] Env vars from CLI: {envs}") + factory.update_runners({"env": envs}) + jobParser = ConfigParser(args.input, factory, args.verbose) print("Begin pipeline steps...") for step in args.steps: @@ -48,6 +51,7 @@ def main(): parser.add_argument("-i", "--input", default="alice-ci.yaml") parser.add_argument("-e", "--env", nargs='*', default=[]) parser.add_argument("-a", "--addrunner", nargs='*', default=[]) + parser.add_argument("-v", "--verbose", action='store_true') args = parser.parse_args() if not os.path.isfile(args.input): print(f"No such file: {args.input}") diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py index a9290e2..f9012e5 100644 --- a/alice-ci/src/alice/runnerfactory.py +++ b/alice-ci/src/alice/runnerfactory.py @@ -5,7 +5,8 @@ from alice.exceptions import ConfigException class Factory(): - def __init__(self) -> None: + def __init__(self, verbose) -> None: + self.verbose = verbose self.runnertypes = self.__load_runners() self.runners = {} self.workdir = getcwd() @@ -15,8 +16,11 @@ class Factory(): # TODO: Runners can be imported via cli too # module = __import__("module_file") # my_class = getattr(module, "class_name") + runners = {"python": PythonRunner} - return {"python": PythonRunner} + if (self.verbose): + print(f"[Alice] Available runners: {'|'.join(runners.keys())}") + return runners def set_globals(self, globals): self.globals = globals @@ -28,12 +32,18 @@ class Factory(): def update_runners(self, config): for runnertype, runnerconfig in config.items(): if runnertype != "global": + if (self.verbose): + print(f"[Alice] Configuring runner {runnertype}") self.get_runner(runnertype).update_config(runnerconfig) def get_runner(self, runnertype): if runnertype not in self.runners: if runnertype in self.runnertypes: - self.runners[runnertype] = self.runnertypes[runnertype](self.workdir, self.globals) + if (self.verbose): + print(f"[Alice] Initializing runner: {runnertype}") + self.runners[runnertype] = self.runnertypes[runnertype](self.workdir, + self.globals, + self.verbose) else: raise ConfigException(f"Invalid runner type: {runnertype}") return self.runners[runnertype] diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index 3a58444..30367c5 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -2,19 +2,21 @@ import subprocess import os import sys import shlex +from tabnanny import verbose from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException # same venv across all runs! class PythonRunner(): - def __init__(self, workdir, defaults) -> None: + def __init__(self, workdir, defaults, verbose) -> None: self.workdir = workdir self.virtual_dir = os.path.abspath(os.path.join(workdir, "venv")) self.config = defaults self.env_vars = os.environ.copy() for env_var in defaults["env"]: self.env_vars[env_var["name"]] = env_var["value"] + self.verbose = verbose self.__init_venv() @@ -30,11 +32,13 @@ class PythonRunner(): p.wait() if p.returncode != 0: sys.stdout.buffer.write(p.stderr.read()) - raise RunnerError("PythonRunner: Could not create virtualenv") + raise RunnerError("[PythonRunner] Could not create virtualenv") else: - print(f"PythonRunner: Virtualenv initialized at {self.virtual_dir}") + if self.verbose: + print(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}") else: - print(f"PythonRunner: Found virtualenv at {self.virtual_dir}") + if self.verbose: + print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}") # Stores common defaults for all jobs - all types! # Also - dependency install by config is only allowed in this step @@ -47,25 +51,35 @@ class PythonRunner(): p.wait() if p.returncode != 0: sys.stdout.buffer.write(p.stderr.read()) - raise(RunnerError(f"PythonRunner: Could not install dependency: {dependency} ({p.returncode})")) + raise(RunnerError(f"[PythonRunner] Could not install dependency: {dependency} ({p.returncode})")) if "env" in config: for env_var in config["env"]: self.env_vars[env_var["name"]] = env_var["value"] if "workdir" in config and config["workdir"] is not None: self.workdir = os.path.join(self.workdir, config["workdir"]) - def __ghetto_glob(self, command): + def __ghetto_glob(self, command, workdir): + if self.verbose: + print(f"[PythonRunner][Globbing] Starting command: {' '.join(command)}") new_command = [] for item in command: if "*" in item: - dir = os.path.abspath(os.path.dirname(item)) + if self.verbose: + print(f"[PythonRunner][Globbing] Found item: [{item}]") + dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item))) base_name = os.path.basename(item) if os.path.isdir(dir): item_parts = base_name.split("*") for file in os.listdir(dir): # TODO: Fix ordering! A*B = B*A = AB* if item_parts[0] in file and item_parts[1] in file: - new_command.append(os.path.join(dir, file)) + new_item = os.path.join(dir, file) + if self.verbose: + print(f"[PythonRunner][Globbing] Substitute: {new_item}") + new_command.append(new_item) + else: + if self.verbose: + print(f"[PythonRunner][Globbing] Dir not exists: {dir}") else: new_command.append(item) return new_command @@ -84,14 +98,19 @@ class PythonRunner(): if "commands" in job_spec: commands = job_spec["commands"] for command in commands: + if self.verbose: + print(f"[PythonRunner] Raw command: {command}") # TODO: only split if command is not an array - run_command = self.__ghetto_glob(shlex.split(command)) + run_command = self.__ghetto_glob(shlex.split(command), pwd) + if self.verbose: + print(f"[PythonRunner] Command to execute: {run_command}") + print(f"[PythonRunner] Workdir: {pwd}") if os.path.isdir(pwd): with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p: p.wait() if p.returncode != 0: raise NonZeroRetcode(f"Command {command} returned code {p.returncode}") else: - raise RunnerError(f"PythonRunner: Invalid path for shell command: {pwd}") + raise RunnerError(f"[PythonRunner] Invalid path for shell command: {pwd}") else: - raise ConfigException(f"PythonRunner: No commands specified in step {job_spec['name']}") + raise ConfigException(f"[PythonRunner] No commands specified in step {job_spec['name']}") diff --git a/alice-ci/src/alice/utils.py b/alice-ci/src/alice/utils.py index 49bc933..7794f4a 100644 --- a/alice-ci/src/alice/utils.py +++ b/alice-ci/src/alice/utils.py @@ -4,7 +4,8 @@ from alice.exceptions import ConfigException class ConfigParser: - def __init__(self, file_path, factory) -> None: + def __init__(self, file_path, factory, verbose=False) -> None: + self.verbose = verbose with open(file_path) as f: self.config = yaml.safe_load(f) self.factory = factory @@ -25,6 +26,9 @@ class ConfigParser: globals["env"] = self.config["runners"]["global"]["env"] if "workdir" in self.config["runners"]["global"]: globals["workdir"] = self.config["runners"]["global"]["workdir"] + + if (self.verbose): + print(f"[Alice] Configured globals: {globals}") return globals def __get_jobs(self): @@ -36,9 +40,11 @@ class ConfigParser: raise ConfigException(f"Job with name {name} already exists!") jobs[name] = job_spec + if (self.verbose): + print(f"[Alice] Parsed jobs: {', '.join(jobs.keys())}") return jobs else: - raise ConfigException("No jobs defined in config") + raise ConfigException("[Alice] No jobs defined in config") def execute_job(self, job_name): if job_name in self.jobs: