From cb3781e154c0feee3caf1d98930e95ebfd63da10 Mon Sep 17 00:00:00 2001 From: Daniel Gyulai Date: Sun, 10 Apr 2022 18:37:42 +0000 Subject: [PATCH] Pipelines (#19) Fix #11 Co-authored-by: gyulaid Reviewed-on: https://git.gyulai.cloud/gyulaid/alice/pulls/19 Co-authored-by: Daniel Gyulai Co-committed-by: Daniel Gyulai --- alice-ci/setup.cfg | 2 +- alice-ci/src/alice/cli.py | 10 ++-------- alice-ci/src/alice/configparser.py | 22 ++++++++++++++++++++-- alice-ci/src/alice/runners/pythonrunner.py | 9 +++++++-- ci-examples/full.yaml | 14 ++++++++++---- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index b53d679..3fb1ffb 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = alice-ci -version = 0.0.6 +version = 0.0.9 author = Daniel Gyulai description = Alice CI framework long_description = file: README.md diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py index e03f06a..bfe3efb 100644 --- a/alice-ci/src/alice/cli.py +++ b/alice-ci/src/alice/cli.py @@ -24,14 +24,8 @@ def parse_jobs(args): print(f"[Alice] Env vars from CLI: {envs}") jobParser = ConfigParser(args.input, gen_env(args.env), args.verbose) - print("[Alice] Begin pipeline steps") for step in args.steps: - if step in jobParser.jobs: - print(f"[Alice][Step] {step}: Start") - status = jobParser.execute_job(step) - print(f"[Alice][Step] {step}: {status}") - else: - raise ConfigException(f"Step {step} not found in {args.input}") + jobParser.execute(step) except ConfigException as e: print(f"Configuration error-> {e}") exit(1) @@ -44,7 +38,7 @@ def parse_jobs(args): def main(): parser = argparse.ArgumentParser(prog="alice") - parser.add_argument("steps", nargs='+') + parser.add_argument("steps", nargs='*', default=["default"]) parser.add_argument("-i", "--input", default="alice-ci.yaml") parser.add_argument("-e", "--env", nargs='*', default=[]) parser.add_argument("-a", "--addrunner", nargs='*', default=[]) diff --git a/alice-ci/src/alice/configparser.py b/alice-ci/src/alice/configparser.py index 4ecfb26..bd4f698 100644 --- a/alice-ci/src/alice/configparser.py +++ b/alice-ci/src/alice/configparser.py @@ -13,6 +13,7 @@ class ConfigParser: self.config = yaml.safe_load(f) self.factory = Factory(verbose, self.__gen_globals(cli_env_vars), self.config.get("runners", {})) self.jobs = self.__get_jobs() + self.pipelines = self.config.get("pipelines", {}) # Initialize env and workdir if not present in global def __gen_globals(self, cli_vars): @@ -72,8 +73,24 @@ class ConfigParser: raise ConfigException(f"Invalid 'changes' config: {changes}") return False + def execute(self, task_name): + if task_name in self.jobs: + self.execute_job(task_name) + elif task_name in self.pipelines: + print(f"[Alice][Pipeline] {task_name}: Start") + self.execute_pipeline(task_name) + print(f"[Alice][Pipeline] {task_name}: Success") + else: + raise ConfigException(f"No such job or pipeline: {task_name}") + + def execute_pipeline(self, pipeline_name): + if pipeline_name in self.pipelines: + for job in self.pipelines[pipeline_name]: + self.execute_job(job) + def execute_job(self, job_name): if job_name in self.jobs: + print(f"[Alice][Job] {job_name}: Start") job_spec = self.jobs[job_name] should_run = True if "changes" in job_spec: @@ -81,6 +98,7 @@ class ConfigParser: if should_run: runner = self.factory.get_runner(job_spec["type"]) runner.run(job_spec) - return "SUCCESS" + status = "SUCCESS" else: - return "SKIP, no change detected" + status = "SKIP, no change detected" + print(f"[Alice][Job] {job_name}: {status}") diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index 268cbd8..18011fc 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -4,7 +4,7 @@ import sys import shlex from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException -from alice.runners.pyutils import PackageManager, glob_command +from alice.runners.pyutils import glob_command # TODO: Handle config like PyPiConfig @@ -42,7 +42,12 @@ class PythonRunner: if len(dependencies) > 0: if self.verbose: print(f"[PythonRunner] Ensuring dependencies: {', '.join(dependencies)}") - PackageManager.getInstance().ensure_more(dependencies, executable=self.vpython) + command = [self.vpython, "-m", "pip", "install"] + dependencies + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + sys.stdout.buffer.write(p.stderr.read()) + raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})")) if self.verbose: print("[PythonRunner] Installation done") diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml index a7f8e61..108ba92 100644 --- a/ci-examples/full.yaml +++ b/ci-examples/full.yaml @@ -30,14 +30,14 @@ jobs: - "-c \"from os import environ; assert environ['A'] == 'D'; assert environ['B'] == 'E'; assert environ['C'] == 'C'; print('Assertions passed')\"" - name: lint type: python - workdir: alice-ci + workdir: alice-ci/src commands: - "-m flake8 --ignore E501" - name: pkg type: pypi workdir: . - upload: true - fail_if_exists: false + upload: false + fail_if_exists: false # TODO: currently unused repo: uri: example.com username: @@ -45,4 +45,10 @@ jobs: password: from_env: PYPIPASS packages: - - alice-ci \ No newline at end of file + - alice-ci +pipelines: + default: + - lint + - env + - pkg + \ No newline at end of file