Pipelines (#19)

Fix #11

Co-authored-by: gyulaid <gyulaid@gyulai.cloud>
Reviewed-on: #19
Co-authored-by: Daniel Gyulai <gyulaid@gyulai.cloud>
Co-committed-by: Daniel Gyulai <gyulaid@gyulai.cloud>

alice-ci/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = alice-ci
-version = 0.0.6
+version = 0.0.9
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md

alice-ci/src/alice/cli.py

@@ -24,14 +24,8 @@ def parse_jobs(args):
                 print(f"[Alice] Env vars from CLI: {envs}")
         jobParser = ConfigParser(args.input, gen_env(args.env), args.verbose)
 
-        print("[Alice] Begin pipeline steps")
         for step in args.steps:
-            if step in jobParser.jobs:
-                print(f"[Alice][Step] {step}: Start")
-                status = jobParser.execute_job(step)
-                print(f"[Alice][Step] {step}: {status}")
-            else:
-                raise ConfigException(f"Step {step} not found in {args.input}")
+            jobParser.execute(step)
     except ConfigException as e:
         print(f"Configuration error-> {e}")
         exit(1)
@@ -44,7 +38,7 @@ def parse_jobs(args):
 
 def main():
     parser = argparse.ArgumentParser(prog="alice")
-    parser.add_argument("steps", nargs='+')
+    parser.add_argument("steps", nargs='*', default=["default"])
     parser.add_argument("-i", "--input", default="alice-ci.yaml")
     parser.add_argument("-e", "--env", nargs='*', default=[])
     parser.add_argument("-a", "--addrunner", nargs='*', default=[])

alice-ci/src/alice/configparser.py

@@ -13,6 +13,7 @@ class ConfigParser:
             self.config = yaml.safe_load(f)
         self.factory = Factory(verbose, self.__gen_globals(cli_env_vars), self.config.get("runners", {}))
         self.jobs = self.__get_jobs()
+        self.pipelines = self.config.get("pipelines", {})
 
     # Initialize env and workdir if not present in global
     def __gen_globals(self, cli_vars):
@@ -72,8 +73,24 @@ class ConfigParser:
             raise ConfigException(f"Invalid 'changes' config: {changes}")
         return False
 
+    def execute(self, task_name):
+        if task_name in self.jobs:
+            self.execute_job(task_name)
+        elif task_name in self.pipelines:
+            print(f"[Alice][Pipeline] {task_name}: Start")
+            self.execute_pipeline(task_name)
+            print(f"[Alice][Pipeline] {task_name}: Success")
+        else:
+            raise ConfigException(f"No such job or pipeline: {task_name}")
+
+    def execute_pipeline(self, pipeline_name):
+        if pipeline_name in self.pipelines:
+            for job in self.pipelines[pipeline_name]:
+                self.execute_job(job)
+
     def execute_job(self, job_name):
         if job_name in self.jobs:
+            print(f"[Alice][Job] {job_name}: Start")
             job_spec = self.jobs[job_name]
             should_run = True
             if "changes" in job_spec:
@@ -81,6 +98,7 @@ class ConfigParser:
             if should_run:
                 runner = self.factory.get_runner(job_spec["type"])
                 runner.run(job_spec)
-                return "SUCCESS"
+                status = "SUCCESS"
             else:
-                return "SKIP, no change detected"
+                status = "SKIP, no change detected"
+            print(f"[Alice][Job] {job_name}: {status}")

alice-ci/src/alice/runners/pythonrunner.py

@@ -4,7 +4,7 @@ import sys
 import shlex
 
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
-from alice.runners.pyutils import PackageManager, glob_command
+from alice.runners.pyutils import glob_command
 
 
 # TODO: Handle config like PyPiConfig
@@ -42,7 +42,12 @@ class PythonRunner:
         if len(dependencies) > 0:
             if self.verbose:
                 print(f"[PythonRunner] Ensuring dependencies:  {', '.join(dependencies)}")
-            PackageManager.getInstance().ensure_more(dependencies, executable=self.vpython)
+            command = [self.vpython, "-m", "pip", "install"] + dependencies
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})"))
             if self.verbose:
                 print("[PythonRunner] Installation done")
 

ci-examples/full.yaml

@@ -30,14 +30,14 @@ jobs:
       - "-c \"from os import environ; assert environ['A'] == 'D'; assert environ['B'] == 'E'; assert environ['C'] == 'C'; print('Assertions passed')\""
   - name: lint
     type: python
-    workdir: alice-ci
+    workdir: alice-ci/src
     commands:
       - "-m flake8 --ignore E501"
   - name: pkg
     type: pypi
     workdir: .
-    upload: true
-    fail_if_exists: false
+    upload: false
+    fail_if_exists: false # TODO: currently unused
     repo:
       uri: example.com
       username:
@@ -45,4 +45,10 @@ jobs:
       password:
         from_env: PYPIPASS
     packages:
-     - alice-ci
+     - alice-ci
+pipelines:
+  default:
+    - lint
+    - env
+    - pkg
+