refactor - This PR closes #3 (#8)

Co-authored-by: gyulaid <gyulaid@gyulai.cloud>
Reviewed-on: #8
Co-authored-by: Daniel Gyulai <gyulaid@gyulai.cloud>
Co-committed-by: Daniel Gyulai <gyulaid@gyulai.cloud>

alice-ci/README.md

@@ -1,6 +1,6 @@
 # Alice-CI
 
-Continous Integration framework with the goal of using the exact same steps in CI and local env. Steps can be defined in yaml files, syntax seen the docs. Runs on LInux and Windows, Mac should work too, but not yet tested.
+Continous Integration framework with the goal of using the exact same code in CI and local env. Steps can be defined in yaml files, for syntax see the docs. Runs on LInux and Windows, Mac should work too, but not yet tested.
 
 ## Usage
 

alice-ci/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = alice-ci
-version = 0.0.1
+version = 0.0.2
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md
@@ -20,6 +20,7 @@ packages = find:
 python_requires = >=3.6
 install_requires =
     PyYAML==6.0
+    virtualenv==20.14.0
 
 [options.packages.find]
 where = src

alice-ci/src/alice/__main__.py

@@ -1,3 +1,3 @@
-from .cli import main
+from cli import main
 
 main()

alice-ci/src/alice/cli.py

@@ -1,91 +1,58 @@
-# Sourcefor App class:
-# https://stackoverflow.com/questions/57593111/how-to-call-pip-from-a-python-script-and-make-it-install-locally-to-that-script
 import os
-import sys
-import subprocess
 import argparse
 
+from utils import ConfigParser
+from runnerfactory import Factory
+from exceptions import ConfigException, NonZeroRetcode, RunnerError
 
-class App:
-    def __init__(self, virtual_dir):
-        self.virtual_dir = virtual_dir
-        if os.name == "nt":
-            self.virtual_python = os.path.join(self.virtual_dir, "Scripts", "python.exe")
-        else:
-            self.virtual_python = os.path.join(self.virtual_dir, "bin", "python3")
 
-    def install_virtual_env(self):
+def gen_env(self, param_list):
-        self.pip_install("virtualenv")
+    env_vars = {}
-        if not os.path.exists(self.virtual_python):
+    for item in param_list:
-            import subprocess
+        item_parts = item.split("=")
-            subprocess.call([sys.executable, "-m", "virtualenv", self.virtual_dir])
+        if len(item_parts) == 2:
+            env_vars[item_parts[0]] = item_parts[1]
         else:
-            print("found virtual python: " + self.virtual_python)
+            raise ConfigException(f"Invalid parameter: {item}")
-
+    return env_vars
-    def is_venv(self):
+
-        return sys.prefix == self.virtual_dir
+
-
+def parse_jobs(args):
-    def restart_under_venv(self):
+    try:
-        print("Restarting under virtual environment " + self.virtual_dir)
+        factory = Factory()
-        with subprocess.Popen([self.virtual_python, __file__] + sys.argv[1:]) as p:
+        if len(args.env) > 0:
-            p.wait()
+            factory.update_runners({"env": gen_env(args.env)})
-            exit(p.returncode)
+        jobParser = ConfigParser(args.input, factory)
-
+
-    def pip_install(self, package, import_name=None):
+        print("Begin pipeline steps...")
-        try:
+        for step in args.steps:
-            if import_name is None:
+            if step in jobParser.jobs:
-                __import__(package)
+                jobParser.execute_job(step)
+                print(f"[Step] {step}: SUCCESS")
             else:
-                __import__(import_name)
+                print(f"Step {step} not found in {args.input}")
-        except:  # noqa: E722
+                exit(1)
-            subprocess.call([sys.executable, "-m", "pip", "install", package, "--upgrade"])
+    except ConfigException as e:
-
+        print(f"Configuration error-> {e}")
-    def __gen_env(self, param_list):
+        exit(1)
-        env_vars = {}
+    except NonZeroRetcode:
-        for item in param_list:
+        print("FAILED")
-            item_parts = item.split("=")
+        exit(1)
-            if len(item_parts) == 2:
+    except RunnerError as e:
-                env_vars[item_parts[0]] = item_parts[1]
+        print(f"RunnerError-> {e}")
-        return env_vars
-
-    def run(self, args, repoDir):
-        if not self.is_venv():
-            self.install_virtual_env()
-            self.restart_under_venv()
-        else:
-            print("Running under virtual environment")
-            # TODO: yaml is only used in venv, yet installed as dependency in setup.cfg
-            self.pip_install("pyyaml", "yaml")
-
-            from jobparser import JobParser
-            jobParser = JobParser(args.input, repoDir, self.virtual_python)
-            for name, import_name in jobParser.get_modules():
-                self.pip_install(name, import_name)
-
-            print("Begin pipeline steps...")
-            for step in args.steps:
-                if step in jobParser.jobs:
-                    jobParser.jobs[step].run_commands(self.__gen_env(args.env))
-                    print(f"Step {step}: SUCCESS")
-                else:
-                    print(f"Step {step} not found in {args.input}")
-                    exit(1)
 
 
 def main():
-    pathToScriptDir = os.path.dirname(os.path.realpath(__file__))
+    parser = argparse.ArgumentParser(prog="alice")
-    repoDir = os.path.join(pathToScriptDir, "..")
-    app = App(os.path.join(pathToScriptDir, "venv"))
-
-    parser = argparse.ArgumentParser()
     parser.add_argument("steps", nargs='+')
     parser.add_argument("-i", "--input", default="alice-ci.yaml")
     parser.add_argument("-e", "--env", nargs='*', default=[])
+    parser.add_argument("-a", "--addrunner", nargs='*', default=[])
     args = parser.parse_args()
     if not os.path.isfile(args.input):
         print(f"No such file: {args.input}")
         exit(1)
-    app.run(args, repoDir)
+    parse_jobs(args)
 
 
 if __name__ == "__main__":

alice-ci/src/alice/exceptions.py

@@ -1,2 +1,10 @@
 class NonZeroRetcode(Exception):
     pass
+
+
+class RunnerError(Exception):
+    pass
+
+
+class ConfigException(Exception):
+    pass

alice-ci/src/alice/pythonrunner.py

@@ -1,48 +0,0 @@
-import subprocess
-import os
-
-from exceptions import NonZeroRetcode
-
-
-class PythonRunner():
-    def __init__(self, repo, vpython) -> None:
-        self.vpython = vpython
-        self.repopath = repo
-
-    def __get_env(self, overrides):
-        env = os.environ.copy()
-        if overrides is not None:
-            for key, value in overrides.items():
-                env[key] = value
-        return env
-
-    def ghetto_glob(self, command):
-        new_command = []
-        for item in command:
-            if "*" in item:
-                dir = os.path.abspath(os.path.dirname(item))
-                base_name = os.path.basename(item)
-                if os.path.isdir(dir):
-                    item_parts = base_name.split("*")
-                    print(item_parts)
-                    for file in os.listdir(dir):
-                        if item_parts[0] in file and item_parts[1] in file:
-                            new_command.append(os.path.join(dir, file))
-            else:
-                new_command.append(item)
-        return new_command
-
-    def run(self, command, workdir=None, env=None):
-        if workdir is not None:
-            pwd = os.path.abspath(os.path.join(self.repopath, workdir))
-        else:
-            pwd = self.repopath
-        run_env = self.__get_env(env)
-        run_command = self.ghetto_glob(command)
-        if os.path.isdir(pwd):
-            with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p:
-                p.wait()
-                if p.returncode != 0:
-                    raise NonZeroRetcode(f"Command {command} returned code {p.returncode}")
-        else:
-            raise Exception(f"Invalid path for shell command: {pwd}")

alice-ci/src/alice/runnerfactory.py

@@ -0,0 +1,34 @@
+from runners.pythonrunner import PythonRunner
+from os import getcwd
+
+
+class Factory():
+    def __init__(self) -> None:
+        self.runnertypes = self.__load_runners()
+        self.runners = {}
+        self.workdir = getcwd()
+        self.globals = {}
+
+    def __load_runners(self):
+        # TODO: Runners can be imported via cli too
+        # module = __import__("module_file")
+        # my_class = getattr(module, "class_name")
+
+        return {"python": PythonRunner}
+
+    def set_globals(self, globals):
+        self.globals = globals
+
+    def update_globals(self, update):
+        if "env" in update:
+            self.globals["env"].update(update["env"])
+
+    def update_runners(self, config):
+        for runnertype, runnerconfig in config.items():
+            if runnertype != "global":
+                self.get_runner(runnertype).update_config(runnerconfig)
+
+    def get_runner(self, runnertype):
+        if runnertype not in self.runners:
+            self.runners[runnertype] = self.runnertypes[runnertype](self.workdir, self.globals)
+        return self.runners[runnertype]

alice-ci/src/alice/runners/pythonrunner.py

@@ -0,0 +1,96 @@
+import subprocess
+import os
+import sys
+import shlex
+
+from exceptions import NonZeroRetcode, RunnerError, ConfigException
+
+
+# same venv across all runs!
+class PythonRunner():
+    def __init__(self, workdir, defaults) -> None:
+        self.workdir = workdir
+        self.virtual_dir = os.path.abspath(os.path.join(workdir, "venv"))
+        self.config = defaults
+        self.env_vars = os.environ.copy()
+        for env_var in defaults["env"]:
+            self.env_vars[env_var["name"]] = env_var["value"]
+
+        self.__init_venv()
+
+    def __init_venv(self):
+        if os.name == "nt":  # Windows
+            self.vpython = os.path.join(self.virtual_dir, "Scripts", "python.exe")
+        else:  # Linux & Mac
+            self.vpython = os.path.join(self.virtual_dir, "bin", "python3")
+
+        if not os.path.exists(self.vpython):
+            with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir],
+                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise RunnerError("PythonRunner: Could not create virtualenv")
+                else:
+                    print(f"PythonRunner: Virtualenv initialized at {self.virtual_dir}")
+        else:
+            print(f"PythonRunner: Found virtualenv at {self.virtual_dir}")
+
+    # Stores common defaults for all jobs - all types!
+    # Also - dependency install by config is only allowed in this step
+    def update_config(self, config):
+        if "dependencies" in config:
+            for dependency in config["dependencies"]:
+                # TODO: Check what happens with fixed version
+                with subprocess.Popen([self.vpython, "-m", "pip", "install", dependency, "--upgrade"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                    p.wait()
+                    if p.returncode != 0:
+                        sys.stdout.buffer.write(p.stderr.read())
+                        raise(RunnerError(f"PythonRunner: Could not install dependency: {dependency} ({p.returncode})"))
+        for env_var in config["env"]:
+            self.env_vars[env_var["name"]] = env_var["value"]
+        if "workdir" in config and config["workdir"] is not None:
+            self.workdir = os.path.join(self.workdir, config["workdir"])
+
+    def __ghetto_glob(self, command):
+        new_command = []
+        for item in command:
+            if "*" in item:
+                dir = os.path.abspath(os.path.dirname(item))
+                base_name = os.path.basename(item)
+                if os.path.isdir(dir):
+                    item_parts = base_name.split("*")
+                    print(item_parts)
+                    for file in os.listdir(dir):
+                        # TODO: Fix ordering! A*B = B*A = AB*
+                        if item_parts[0] in file and item_parts[1] in file:
+                            new_command.append(os.path.join(dir, file))
+            else:
+                new_command.append(item)
+        return new_command
+
+    # Executes the given job in the one and only venv
+    # parameter shall be the raw jobscpec
+    def run(self, job_spec):
+        if "workdir" in job_spec:
+            pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"]))
+        else:
+            pwd = self.workdir
+        run_env = self.env_vars.copy()
+        if "env" in job_spec:
+            for env_var in job_spec["env"]:
+                run_env[env_var["name"]] = env_var["value"]
+        if "commands" in job_spec:
+            commands = job_spec["commands"]
+            for command in commands:
+                # TODO: only split if command is not an array
+                run_command = self.__ghetto_glob(shlex.split(command))
+                if os.path.isdir(pwd):
+                    with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p:
+                        p.wait()
+                        if p.returncode != 0:
+                            raise NonZeroRetcode(f"Command {command} returned code {p.returncode}")
+                else:
+                    raise RunnerError(f"PythonRunner: Invalid path for shell command: {pwd}")
+        else:
+            raise ConfigException(f"PythonRunner: No commands specified in step {job_spec['name']}")

alice-ci/src/alice/jobparser.py → alice-ci/src/alice/utils.py

@@ -1,7 +1,6 @@
 import yaml
-import shlex
+from runners.pythonrunner import PythonRunner
-from pythonrunner import PythonRunner
+from exceptions import NonZeroRetcode, ConfigException
-from exceptions import NonZeroRetcode
 
 
 class DummyRunner():
@@ -39,40 +38,41 @@ class Job():
             exit(1)
 
 
-class JobParser:
+class ConfigParser:
-    def __init__(self, file_path, repoDir, virtual_python) -> None:
+    def __init__(self, file_path, factory) -> None:
         with open(file_path) as f:
             self.config = yaml.safe_load(f)
-        self.jobs = self.__get_jobs(repoDir, virtual_python)
+        self.factory = factory
+        if "runners" in self.config:
+            if "global" in self.config["runners"]:
+                self.factory.set_globals(self.__gen_globals())
+            self.factory.update_runners(self.config["runners"])
+        self.jobs = self.__get_jobs()
+
+    # Initialize env, workdir if not present
+    def __gen_globals(self):
+        globals = self.config["runners"]["global"]
+        if "env" not in globals:
+            globals["env"] = []
+        if "workdir" not in globals:
+            globals["workdir"] = None
+        return globals
 
-    def __get_jobs(self, repoDir, virtual_python):
+    def __get_jobs(self):
         if "jobs" in self.config:
             jobs = {}
             for job_spec in self.config["jobs"]:
                 name = job_spec["name"]
                 if name in jobs:
-                    raise Exception(f"Job with name {name} already exists!")
+                    raise ConfigException(f"Job with name {name} already exists!")
-
-                job = Job(job_spec["type"],
-                          repoDir,
-                          virtual_python,
-                          job_spec.get("workdir", None),
-                          job_spec.get("env", None))
 
-                for cmd in job_spec["commands"]:
+                jobs[name] = job_spec
-                    job.commands.append(shlex.split(cmd))
-                jobs[name] = job
             return jobs
-
         else:
-            raise Exception("No jobs defined in config")
+            raise ConfigException("No jobs defined in config")
 
-    def get_modules(self):
+    def execute_job(self, job_name):
-        modules = []
+        if job_name in self.jobs:
-        if "runners" in self.config:
+            # Pass the job_spec to a runner
-            if "python" in self.config["runners"]:
+            runner = self.factory.get_runner(self.jobs[job_name]["type"])
-                if "dependencies" in self.config["runners"]["python"]:
+            runner.run(self.jobs[job_name])
-                    for dep in self.config["runners"]["python"]["dependencies"]:
-                        # (name, i_name) if i_name is defined, else (name, name)
-                        modules.append((dep["name"], dep.get("import_name", dep["name"])))
-        return modules

ci-examples/full.yaml

@@ -0,0 +1,29 @@
+runners:
+  global:
+    env:
+      - name: A
+        value: A
+      - name: B
+        value: B
+      - name: C
+        value: C
+    workdir: packages
+  python:
+    env:
+      - name: A
+        value: D
+    dependencies:
+      - flake8
+      - build
+jobs:
+  - name: env
+    type: python
+    env:
+      - name: B
+        value: E
+    commands:
+      - "-c \"import os; print(os.environ)\""
+  - name: lint
+    workdir: alice-ci
+    commands:
+      - "-m flake8 --ignore E501"