Merge branch '0.0.8' of https://git.gyulai.cloud/gyulaid/alice into 13_manage-python-packages

alice-ci/src/alice/cli.py

@@ -2,7 +2,6 @@ import os
 import argparse
 
 from alice.utils import ConfigParser
-from alice.runnerfactory import Factory
 from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
 
 
@@ -19,16 +18,16 @@ def gen_env(param_list):
 
 def parse_jobs(args):
     try:
-        factory = Factory(args.verbose)
         if len(args.env) > 0:
             envs = gen_env(args.env)
             if args.verbose:
                 print(f"[Alice] Env vars from CLI: {envs}")
-        jobParser = ConfigParser(args.input, factory, gen_env(args.env), args.verbose)
+        jobParser = ConfigParser(args.input, gen_env(args.env), args.verbose)
 
         print("[Alice] Begin pipeline steps")
         for step in args.steps:
             if step in jobParser.jobs:
+                print(f"[Alice][Step] {step}: Start")
                 status = jobParser.execute_job(step)
                 print(f"[Alice][Step] {step}: {status}")
             else:

alice-ci/src/alice/runnerfactory.py

@@ -1,37 +1,46 @@
+from os.path import join, abspath
+
 from alice.runners.pythonrunner import PythonRunner
 from alice.runners.pypirunner import PyPiRunner
 from alice.exceptions import ConfigException
 
 
 class Factory():
-    def __init__(self, verbose) -> None:
+    def __init__(self, verbose, globals, runner_configs) -> None:
         self.verbose = verbose
-        self.runnertypes = self.__load_runners()
+        self.globals = globals
         self.runner_configs = {}
+        self.runnertypes = {}
         self.runners = {}
-        self.globals = {}
+        self.__load_runners()
+        self.__gen_runner_configs(runner_configs)
 
     def __load_runners(self):
         # TODO: Runners can be imported via cli too
         # https://git.gyulai.cloud/gyulaid/alice/issues/4
         # module = __import__("module_file")
         # my_class = getattr(module, "class_name")
-        runners = {"python": PythonRunner,
+        self.runners = {"python": PythonRunner,
                    "pypi": PyPiRunner}
 
         if (self.verbose):
-            print(f"[Alice] Available runners: {'|'.join(runners.keys())}")
-        return runners
-
-    def set_globals(self, globals):
-        self.globals = globals
+            print(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}")
 
-    def update_runners(self, config):
+    def __gen_runner_configs(self, config):
         for runnertype, runnerconfig in config.items():
             if runnertype != "global":
                 if (self.verbose):
-                    print(f"[Alice] Configuring runner: {runnertype}")
-                self.get_runner(runnertype).update_config(runnerconfig)
+                    print(f"[Alice] Global config found for runner {runnertype}")
+                config = self.globals.copy()
+                for key, value in runnerconfig.items():
+                    if key == "env":
+                        for env_var in value:
+                            config["env"][env_var["name"]] = env_var["value"]
+                    elif key == "workdir":
+                        config["workdir"] = abspath(join(config["workdir"], value))
+                    else:
+                        config[key] = value
+                self.runner_configs[runnertype] = config
 
     def get_runner(self, runnertype):
         if runnertype not in self.runners:
@@ -41,7 +50,8 @@ class Factory():
                 params = {
                     "verbose": self.verbose
                 }
-                self.runners[runnertype] = self.runnertypes[runnertype](params, self.globals)
+                config = self.runner_configs[runnertype]
+                self.runners[runnertype] = self.runnertypes[runnertype](params, config)
             else:
                 raise ConfigException(f"Invalid runner type: {runnertype}")
         return self.runners[runnertype]

alice-ci/src/alice/runners/pythonrunner.py

@@ -7,14 +7,13 @@ from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
 
 
 class PythonRunner():
-    def __init__(self, params, user_defaults) -> None:
+    def __init__(self, params, config) -> None:
         self.verbose = params["verbose"]
         if self.verbose:
             print("[PythonRunner] Initializing")
-        self.workdir = user_defaults["workdir"]
+        self.workdir = config["workdir"]
         self.virtual_dir = os.path.abspath(os.path.join(self.workdir, "venv"))
-        self.config = user_defaults
-
+        self.config = config
         self.__init_venv()
 
     def __init_venv(self):
@@ -38,25 +37,13 @@ class PythonRunner():
             if self.verbose:
                 print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}")
 
-    # Stores common defaults for all jobs - all types!
-    # Also - dependency install by config is only allowed in this step
-    def update_config(self, config):
-        if "dependencies" in config:
-            # TODO: Use pip freeze to list installed packages, only install missing
-            # ...or just dump the dependencies in a single list, and let pip deal with it.
-            for dependency in config["dependencies"]:
-                # TODO: Check what happens with fixed version
-                command = [self.vpython, "-m", "pip", "install", dependency, "--upgrade"]
-                with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
-                    p.wait()
-                    if p.returncode != 0:
-                        sys.stdout.buffer.write(p.stderr.read())
-                        raise(RunnerError(f"[PythonRunner] Could not install dependency: {dependency} ({p.returncode})"))
-        if "env" in config:
-            for env_var in config["env"]:
-                self.config["env"][env_var["name"]] = env_var["value"]
-        if "workdir" in config and config["workdir"] is not None:
-            self.workdir = os.path.join(self.workdir, config["workdir"])
+        if "dependencies" in self.config:
+            command = [self.vpython, "-m", "pip", "install"] + self.config["dependencies"] + ["--upgrade"]
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(RunnerError(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
 
     def __ghetto_glob(self, command, workdir):
         if self.verbose:

alice-ci/src/alice/utils.py

@@ -3,17 +3,15 @@ import subprocess
 import yaml
 
 from alice.exceptions import ConfigException
+from alice.runnerfactory import Factory
 
 
 class ConfigParser:
-    def __init__(self, file_path, factory, cli_env_vars, verbose=False) -> None:
+    def __init__(self, file_path, cli_env_vars, verbose=False) -> None:
         self.verbose = verbose
         with open(file_path) as f:
             self.config = yaml.safe_load(f)
-        self.factory = factory
-        self.factory.set_globals(self.__gen_globals(cli_env_vars))
-        if "runners" in self.config:
-            self.factory.update_runners(self.config["runners"])
+        self.factory = Factory(verbose, self.__gen_globals(cli_env_vars), self.config.get("runners", {}))
         self.jobs = self.__get_jobs()
 
     # Initialize env and workdir if not present in global

ci-examples/full.yaml

@@ -22,11 +22,12 @@ jobs:
       branch: origin/master
       paths:
         - "docs"
+        - "alice-ci"
     env:
       - name: B
         value: E
     commands:
-      - "-c \"import os; print(os.environ)\""
+      - "-c \"from os import environ; assert environ['A'] == 'D'; assert environ['B'] == 'E'; assert environ['C'] == 'C'; print('Assertions passed')\""
   - name: lint
     type: python
     workdir: alice-ci

docs/runners.md

@@ -16,17 +16,18 @@ TODO
 
 Each runner has to support the following functions:
 
-### __init__(params, user_defaults)
+### __init__(params, config)
 
 * params: dict of runtime variables for the program itself.
-* user_defaults: raw data from the CI file's global dict, augmented with an "env" dict, which contains environment variables from the host sytem, the CLI params and the pipeline global config, and the "workdir" value, which is the absolute path of the directory that the runner shall recognize as the current working directory.
+* config: Runner config data, aplies to all jobs
 
 #### Params
 
 Currently the only param used is the dict is "verbose", whichis a boolean. The intended purpose is to show debug output if set to True.
 
-#### Workdir
-workdir can be assigned at CI yaml level as global
+#### config
+
+Dict. Has two fix keys, `env` and `workdir`. Env is the environment variables of the host, expanded by CLI parameters, expanded by global config values from yaml. A key defined in the yaml overwrites the value copied from the host. Workdir is similar, can be assigned at CI yaml level as global, but runners may overwrite it.
 Order:
     By default: os.cwd()
     if overwritten in global
@@ -34,11 +35,15 @@ Order:
     if owerwritten in runner config
     if overwritten in job
 
-Runner shall receive the current working directory, unless stated otherwise in global config
+Runner shall receive the current working directory, unless stated otherwise in global config.
 
-### update_config(config)
+The expected behaviour of "overwriting" the workdir is adding the new directory name to the existing path. For example:
+ * cwd = /root
+ * global defines workdir as "projects"
+ * runner config defines workdir as "python"
+ * job defines workdir as "alice"
 
-The function takes the raw data from the parsed yaml under runners.(runnername). Handling its own config is the sole responsibility of the runner. This function may be called at any point of the running lifecycle, so the runner has to support changing its own configuration.
+ In the case above, the actual working directory of the running job shall be `/root/projects/python/alice`.
 
 ### run(job_spec)