7_runner-interface (#12)

Fix #7

Co-authored-by: gyulaid <gyulaid@gyulai.cloud>
Reviewed-on: #12
Co-authored-by: Daniel Gyulai <gyulaid@gyulai.cloud>
Co-committed-by: Daniel Gyulai <gyulaid@gyulai.cloud>

.devcontainer/Dockerfile

@@ -0,0 +1,11 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu/.devcontainer/base.Dockerfile
+
+# [Choice] Ubuntu version (use hirsuite or bionic on local arm64/Apple Silicon): hirsute, focal, bionic
+ARG VARIANT="hirsute"
+FROM mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT}
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+

.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu
+{
+	"name": "Ubuntu",
+	"build": {
+		"dockerfile": "Dockerfile",
+		// Update 'VARIANT' to pick an Ubuntu version: hirsute, focal, bionic
+		// Use hirsute or bionic on local arm64/Apple Silicon.
+		"args": { "VARIANT": "focal" }
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "uname -a",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"features": {
+		"python": "latest"
+	}
+}

alice-ci/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = alice-ci
-version = 0.0.6
+version = 0.0.7
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md
@@ -16,11 +16,15 @@ classifiers =
 [options]
 package_dir =
     = src
-packages = find:
+packages = alice
 python_requires = >=3.6
 install_requires =
     PyYAML==6.0
     virtualenv==20.14.0
 
+[options.entry_points]
+console_scripts =
+    alice = alice.cli:main
+
 [options.packages.find]
 where = src

alice-ci/src/alice/cli.py

@@ -6,7 +6,7 @@ from alice.runnerfactory import Factory
 from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
 
 
-def gen_env(self, param_list):
+def gen_env(param_list):
     env_vars = {}
     for item in param_list:
         item_parts = item.split("=")
@@ -24,22 +24,21 @@ def parse_jobs(args):
             envs = gen_env(args.env)
             if args.verbose:
                 print(f"[Alice] Env vars from CLI: {envs}")
-            factory.update_runners({"env": envs})
-        jobParser = ConfigParser(args.input, factory, args.verbose)
+        jobParser = ConfigParser(args.input, factory, gen_env(args.env), args.verbose)
 
-        print("Begin pipeline steps...")
+        print("[Alice] Begin pipeline steps")
         for step in args.steps:
             if step in jobParser.jobs:
                 status = jobParser.execute_job(step)
-                print(f"[Step] {step}: {status}")
+                print(f"[Alice][Step] {step}: {status}")
             else:
-                print(f"Step {step} not found in {args.input}")
+                raise ConfigException(f"Step {step} not found in {args.input}")
                 exit(1)
     except ConfigException as e:
         print(f"Configuration error-> {e}")
         exit(1)
     except NonZeroRetcode:
-        print("FAILED")
+        print("[Alice] FAILED")
         exit(1)
     except RunnerError as e:
         print(f"RunnerError-> {e}")

alice-ci/src/alice/runnerfactory.py

@@ -1,5 +1,3 @@
-from os import getcwd
-
 from alice.runners.pythonrunner import PythonRunner
 from alice.exceptions import ConfigException
 
@@ -8,12 +6,13 @@ class Factory():
     def __init__(self, verbose) -> None:
         self.verbose = verbose
         self.runnertypes = self.__load_runners()
+        self.runner_configs = {}
         self.runners = {}
-        self.workdir = getcwd()
         self.globals = {}
 
     def __load_runners(self):
         # TODO: Runners can be imported via cli too
+        # https://git.gyulai.cloud/gyulaid/alice/issues/4
         # module = __import__("module_file")
         # my_class = getattr(module, "class_name")
         runners = {"python": PythonRunner}
@@ -25,15 +24,11 @@ class Factory():
     def set_globals(self, globals):
         self.globals = globals
 
-    def update_globals(self, update):
-        if "env" in update:
-            self.globals["env"].update(update["env"])
-
     def update_runners(self, config):
         for runnertype, runnerconfig in config.items():
             if runnertype != "global":
                 if (self.verbose):
-                    print(f"[Alice] Configuring runner {runnertype}")
+                    print(f"[Alice] Configuring runner: {runnertype}")
                 self.get_runner(runnertype).update_config(runnerconfig)
 
     def get_runner(self, runnertype):
@@ -41,9 +36,10 @@ class Factory():
             if runnertype in self.runnertypes:
                 if (self.verbose):
                     print(f"[Alice] Initializing runner: {runnertype}")
-                self.runners[runnertype] = self.runnertypes[runnertype](self.workdir,
-                                                                        self.globals,
-                                                                        self.verbose)
+                params = {
+                    "verbose": self.verbose
+                }
+                self.runners[runnertype] = self.runnertypes[runnertype](params, self.globals)
             else:
                 raise ConfigException(f"Invalid runner type: {runnertype}")
         return self.runners[runnertype]

alice-ci/src/alice/runners/__init__.py

@@ -1,3 +1,2 @@
+# flake8: noqa F401
 from alice.runners.pythonrunner import PythonRunner
-
-__all__ = ["PythonRunner"]

alice-ci/src/alice/runners/pythonrunner.py

@@ -2,21 +2,18 @@ import subprocess
 import os
 import sys
 import shlex
-from tabnanny import verbose
 
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
 
 
-# same venv across all runs!
 class PythonRunner():
-    def __init__(self, workdir, defaults, verbose) -> None:
-        self.workdir = workdir
-        self.virtual_dir = os.path.abspath(os.path.join(workdir, "venv"))
-        self.config = defaults
-        self.env_vars = os.environ.copy()
-        for env_var in defaults["env"]:
-            self.env_vars[env_var["name"]] = env_var["value"]
-        self.verbose = verbose
+    def __init__(self, params, user_defaults) -> None:
+        self.verbose = params["verbose"]
+        if self.verbose:
+            print("[PythonRunner] Initializing")
+        self.workdir = user_defaults["workdir"]
+        self.virtual_dir = os.path.abspath(os.path.join(self.workdir, "venv"))
+        self.config = user_defaults
 
         self.__init_venv()
 
@@ -27,6 +24,7 @@ class PythonRunner():
             self.vpython = os.path.join(self.virtual_dir, "bin", "python3")
 
         if not os.path.exists(self.vpython):
+            print("[PythonRunner] Initializing venv")
             with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir],
                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
                 p.wait()
@@ -54,7 +52,7 @@ class PythonRunner():
                         raise(RunnerError(f"[PythonRunner] Could not install dependency: {dependency} ({p.returncode})"))
         if "env" in config:
             for env_var in config["env"]:
-                self.env_vars[env_var["name"]] = env_var["value"]
+                self.config["env"][env_var["name"]] = env_var["value"]
         if "workdir" in config and config["workdir"] is not None:
             self.workdir = os.path.join(self.workdir, config["workdir"])
 
@@ -91,7 +89,7 @@ class PythonRunner():
             pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"]))
         else:
             pwd = self.workdir
-        run_env = self.env_vars.copy()
+        run_env = self.config["env"].copy()
         if "env" in job_spec:
             for env_var in job_spec["env"]:
                 run_env[env_var["name"]] = env_var["value"]
@@ -101,7 +99,10 @@ class PythonRunner():
                 if self.verbose:
                     print(f"[PythonRunner] Raw command: {command}")
                 # TODO: only split if command is not an array
+                if "*" in command:
                     run_command = self.__ghetto_glob(shlex.split(command), pwd)
+                else:
+                    run_command = shlex.split(command)
                 if self.verbose:
                     print(f"[PythonRunner] Command to execute: {run_command}")
                     print(f"[PythonRunner] Workdir: {pwd}")

alice-ci/src/alice/utils.py

@@ -1,4 +1,4 @@
-import os
+from os import getcwd, path, environ
 import subprocess
 import yaml
 
@@ -6,26 +6,29 @@ from alice.exceptions import ConfigException
 
 
 class ConfigParser:
-    def __init__(self, file_path, factory, verbose=False) -> None:
+    def __init__(self, file_path, factory, cli_env_vars, verbose=False) -> None:
         self.verbose = verbose
         with open(file_path) as f:
             self.config = yaml.safe_load(f)
         self.factory = factory
-        self.factory.set_globals(self.__gen_globals())
+        self.factory.set_globals(self.__gen_globals(cli_env_vars))
         if "runners" in self.config:
             self.factory.update_runners(self.config["runners"])
         self.jobs = self.__get_jobs()
 
-    # Initialize env, workdir if not present
-    def __gen_globals(self):
+    # Initialize env and workdir if not present in global
+    def __gen_globals(self, cli_vars):
+        env_vars = environ.copy()
+        env_vars.update(cli_vars)
         globals = {
-            "env": [],
-            "workdir": None
+            "env": env_vars,
+            "workdir": getcwd()
         }
         if "runners" in self.config:
             if "global" in self.config["runners"]:
                 if "env" in self.config["runners"]["global"]:
-                    globals["env"] = self.config["runners"]["global"]["env"]
+                    for var in self.config["runners"]["global"]["env"]:
+                        globals["env"][var["name"]] = var["value"]
                 if "workdir" in self.config["runners"]["global"]:
                     globals["workdir"] = self.config["runners"]["global"]["workdir"]
 
@@ -52,20 +55,20 @@ class ConfigParser:
         try:
             target = changes["branch"]
             paths = []
-            for path in changes["paths"]:
-                paths.append(os.path.abspath(path))
+            for _path in changes["paths"]:
+                paths.append(path.abspath(_path))
             # TODO: Error handling
             command = ["git", "diff", "--name-only", target]
             with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
                 p.wait()
                 for line in p.stdout:
-                    change_path = os.path.abspath(line.decode("UTF-8").strip())
-                    for path in paths:
-                        spec_path = os.path.abspath(path)
+                    change_path = path.abspath(line.decode("UTF-8").strip())
+                    for _path in paths:
+                        spec_path = path.abspath(_path)
                         if change_path.startswith(spec_path):
                             if self.verbose:
                                 print(f"[Alice] Modified file: {change_path}")
-                                print(f"[Alice] Path match: {path}")
+                                print(f"[Alice] Path match: {_path}")
                             return True
         except KeyError:
             raise ConfigException(f"Invalid 'changes' config: {changes}")
@@ -83,4 +86,3 @@ class ConfigParser:
                 return "SUCCESS"
             else:
                 return "SKIP, no change detected"
-

docs/runners.md

@@ -6,3 +6,40 @@ Runners are responsible to execute a list of commands in a set environment defin
 
 * Python - executes python commands in a virtual environment
 * Docker - executes each job in a separate Docker container - unimplemented
+
+## Import schema
+
+What you need to do to make Alice recognise and import your custom Runners
+TODO
+
+## Runner API
+
+Each runner has to support the following functions:
+
+### __init__(params, user_defaults)
+
+* params: dict of runtime variables for the program itself.
+* user_defaults: raw data from the CI file's global dict, augmented with an "env" dict, which contains environment variables from the host sytem, the CLI params and the pipeline global config, and the "workdir" value, which is the absolute path of the directory that the runner shall recognize as the current working directory.
+
+#### Params
+
+Currently the only param used is the dict is "verbose", whichis a boolean. The intended purpose is to show debug output if set to True.
+
+#### Workdir
+workdir can be assigned at CI yaml level as global
+Order:
+    By default: os.cwd()
+    if overwritten in global
+    ------------------------------- Below this level is the runner's responsibility
+    if owerwritten in runner config
+    if overwritten in job
+
+Runner shall receive the current working directory, unless stated otherwise in global config
+
+### update_config(config)
+
+The function takes the raw data from the parsed yaml under runners.(runnername). Handling its own config is the sole responsibility of the runner. This function may be called at any point of the running lifecycle, so the runner has to support changing its own configuration.
+
+### run(job_spec)
+
+This function executes one job attributed ith the type of the runner called. As the only hard requirement for Alice is the "type" field in a job (or the optional "changes"), everything else is handled by the runner.