alice-ci/README.md

@@ -1,6 +1,6 @@
 # Alice-CI
 
-Continous Integration framework with the goal of using the exact same steps in CI and local env. Steps can be defined in yaml files, syntax seen the docs. Runs on LInux and Windows, Mac should work too, but not yet tested.
+Continous Integration framework with the goal of using the exact same code in CI and local env. Steps can be defined in yaml files, for syntax see the docs. Runs on LInux and Windows, Mac should work too, but not yet tested.
 
 ## Usage
 

alice-ci/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = alice-ci
-version = 0.0.1
+version = 0.0.2
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md
@@ -20,6 +20,7 @@ packages = find:
 python_requires = >=3.6
 install_requires =
     PyYAML==6.0
+    virtualenv==20.14.0
 
 [options.packages.find]
 where = src

alice-ci/src/alice/__main__.py

@@ -1,3 +1,3 @@
-from .cli import main
+from cli import main
 
 main()

alice-ci/src/alice/cli.py

@@ -1,91 +1,58 @@
-# Sourcefor App class:
-# https://stackoverflow.com/questions/57593111/how-to-call-pip-from-a-python-script-and-make-it-install-locally-to-that-script
 import os
-import sys
-import subprocess
 import argparse
 
+from utils import ConfigParser
+from runnerfactory import Factory
+from exceptions import ConfigException, NonZeroRetcode, RunnerError
 
-class App:
-    def __init__(self, virtual_dir):
-        self.virtual_dir = virtual_dir
-        if os.name == "nt":
-            self.virtual_python = os.path.join(self.virtual_dir, "Scripts", "python.exe")
-        else:
-            self.virtual_python = os.path.join(self.virtual_dir, "bin", "python3")
 
-    def install_virtual_env(self):
-        self.pip_install("virtualenv")
-        if not os.path.exists(self.virtual_python):
-            import subprocess
-            subprocess.call([sys.executable, "-m", "virtualenv", self.virtual_dir])
+def gen_env(self, param_list):
+    env_vars = {}
+    for item in param_list:
+        item_parts = item.split("=")
+        if len(item_parts) == 2:
+            env_vars[item_parts[0]] = item_parts[1]
         else:
-            print("found virtual python: " + self.virtual_python)
-
-    def is_venv(self):
-        return sys.prefix == self.virtual_dir
-
-    def restart_under_venv(self):
-        print("Restarting under virtual environment " + self.virtual_dir)
-        with subprocess.Popen([self.virtual_python, __file__] + sys.argv[1:]) as p:
-            p.wait()
-            exit(p.returncode)
-
-    def pip_install(self, package, import_name=None):
-        try:
-            if import_name is None:
-                __import__(package)
+            raise ConfigException(f"Invalid parameter: {item}")
+    return env_vars
+
+
+def parse_jobs(args):
+    try:
+        factory = Factory()
+        if len(args.env) > 0:
+            factory.update_runners({"env": gen_env(args.env)})
+        jobParser = ConfigParser(args.input, factory)
+
+        print("Begin pipeline steps...")
+        for step in args.steps:
+            if step in jobParser.jobs:
+                jobParser.execute_job(step)
+                print(f"[Step] {step}: SUCCESS")
             else:
-                __import__(import_name)
-        except:  # noqa: E722
-            subprocess.call([sys.executable, "-m", "pip", "install", package, "--upgrade"])
-
-    def __gen_env(self, param_list):
-        env_vars = {}
-        for item in param_list:
-            item_parts = item.split("=")
-            if len(item_parts) == 2:
-                env_vars[item_parts[0]] = item_parts[1]
-        return env_vars
-
-    def run(self, args, repoDir):
-        if not self.is_venv():
-            self.install_virtual_env()
-            self.restart_under_venv()
-        else:
-            print("Running under virtual environment")
-            # TODO: yaml is only used in venv, yet installed as dependency in setup.cfg
-            self.pip_install("pyyaml", "yaml")
-
-            from jobparser import JobParser
-            jobParser = JobParser(args.input, repoDir, self.virtual_python)
-            for name, import_name in jobParser.get_modules():
-                self.pip_install(name, import_name)
-
-            print("Begin pipeline steps...")
-            for step in args.steps:
-                if step in jobParser.jobs:
-                    jobParser.jobs[step].run_commands(self.__gen_env(args.env))
-                    print(f"Step {step}: SUCCESS")
-                else:
-                    print(f"Step {step} not found in {args.input}")
-                    exit(1)
+                print(f"Step {step} not found in {args.input}")
+                exit(1)
+    except ConfigException as e:
+        print(f"Configuration error-> {e}")
+        exit(1)
+    except NonZeroRetcode:
+        print("FAILED")
+        exit(1)
+    except RunnerError as e:
+        print(f"RunnerError-> {e}")
 
 
 def main():
-    pathToScriptDir = os.path.dirname(os.path.realpath(__file__))
-    repoDir = os.path.join(pathToScriptDir, "..")
-    app = App(os.path.join(pathToScriptDir, "venv"))
-
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(prog="alice")
     parser.add_argument("steps", nargs='+')
     parser.add_argument("-i", "--input", default="alice-ci.yaml")
     parser.add_argument("-e", "--env", nargs='*', default=[])
+    parser.add_argument("-a", "--addrunner", nargs='*', default=[])
     args = parser.parse_args()
     if not os.path.isfile(args.input):
         print(f"No such file: {args.input}")
         exit(1)
-    app.run(args, repoDir)
+    parse_jobs(args)
 
 
 if __name__ == "__main__":

alice-ci/src/alice/exceptions.py

@@ -1,2 +1,10 @@
 class NonZeroRetcode(Exception):
     pass
+
+
+class RunnerError(Exception):
+    pass
+
+
+class ConfigException(Exception):
+    pass

alice-ci/src/alice/pythonrunner.py

@@ -1,48 +0,0 @@
-import subprocess
-import os
-
-from exceptions import NonZeroRetcode
-
-
-class PythonRunner():
-    def __init__(self, repo, vpython) -> None:
-        self.vpython = vpython
-        self.repopath = repo
-
-    def __get_env(self, overrides):
-        env = os.environ.copy()
-        if overrides is not None:
-            for key, value in overrides.items():
-                env[key] = value
-        return env
-
-    def ghetto_glob(self, command):
-        new_command = []
-        for item in command:
-            if "*" in item:
-                dir = os.path.abspath(os.path.dirname(item))
-                base_name = os.path.basename(item)
-                if os.path.isdir(dir):
-                    item_parts = base_name.split("*")
-                    print(item_parts)
-                    for file in os.listdir(dir):
-                        if item_parts[0] in file and item_parts[1] in file:
-                            new_command.append(os.path.join(dir, file))
-            else:
-                new_command.append(item)
-        return new_command
-
-    def run(self, command, workdir=None, env=None):
-        if workdir is not None:
-            pwd = os.path.abspath(os.path.join(self.repopath, workdir))
-        else:
-            pwd = self.repopath
-        run_env = self.__get_env(env)
-        run_command = self.ghetto_glob(command)
-        if os.path.isdir(pwd):
-            with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p:
-                p.wait()
-                if p.returncode != 0:
-                    raise NonZeroRetcode(f"Command {command} returned code {p.returncode}")
-        else:
-            raise Exception(f"Invalid path for shell command: {pwd}")

alice-ci/src/alice/runnerfactory.py

@@ -0,0 +1,34 @@
+from runners.pythonrunner import PythonRunner
+from os import getcwd
+
+
+class Factory():
+    def __init__(self) -> None:
+        self.runnertypes = self.__load_runners()
+        self.runners = {}
+        self.workdir = getcwd()
+        self.globals = {}
+
+    def __load_runners(self):
+        # TODO: Runners can be imported via cli too
+        # module = __import__("module_file")
+        # my_class = getattr(module, "class_name")
+
+        return {"python": PythonRunner}
+
+    def set_globals(self, globals):
+        self.globals = globals
+
+    def update_globals(self, update):
+        if "env" in update:
+            self.globals["env"].update(update["env"])
+
+    def update_runners(self, config):
+        for runnertype, runnerconfig in config.items():
+            if runnertype != "global":
+                self.get_runner(runnertype).update_config(runnerconfig)
+
+    def get_runner(self, runnertype):
+        if runnertype not in self.runners:
+            self.runners[runnertype] = self.runnertypes[runnertype](self.workdir, self.globals)
+        return self.runners[runnertype]

alice-ci/src/alice/runners/pythonrunner.py

@@ -0,0 +1,96 @@
+import subprocess
+import os
+import sys
+import shlex
+
+from exceptions import NonZeroRetcode, RunnerError, ConfigException
+
+
+# same venv across all runs!
+class PythonRunner():
+    def __init__(self, workdir, defaults) -> None:
+        self.workdir = workdir
+        self.virtual_dir = os.path.abspath(os.path.join(workdir, "venv"))
+        self.config = defaults
+        self.env_vars = os.environ.copy()
+        for env_var in defaults["env"]:
+            self.env_vars[env_var["name"]] = env_var["value"]
+
+        self.__init_venv()
+
+    def __init_venv(self):
+        if os.name == "nt":  # Windows
+            self.vpython = os.path.join(self.virtual_dir, "Scripts", "python.exe")
+        else:  # Linux & Mac
+            self.vpython = os.path.join(self.virtual_dir, "bin", "python3")
+
+        if not os.path.exists(self.vpython):
+            with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir],
+                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise RunnerError("PythonRunner: Could not create virtualenv")
+                else:
+                    print(f"PythonRunner: Virtualenv initialized at {self.virtual_dir}")
+        else:
+            print(f"PythonRunner: Found virtualenv at {self.virtual_dir}")
+
+    # Stores common defaults for all jobs - all types!
+    # Also - dependency install by config is only allowed in this step
+    def update_config(self, config):
+        if "dependencies" in config:
+            for dependency in config["dependencies"]:
+                # TODO: Check what happens with fixed version
+                with subprocess.Popen([self.vpython, "-m", "pip", "install", dependency, "--upgrade"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                    p.wait()
+                    if p.returncode != 0:
+                        sys.stdout.buffer.write(p.stderr.read())
+                        raise(RunnerError(f"PythonRunner: Could not install dependency: {dependency} ({p.returncode})"))
+        for env_var in config["env"]:
+            self.env_vars[env_var["name"]] = env_var["value"]
+        if "workdir" in config and config["workdir"] is not None:
+            self.workdir = os.path.join(self.workdir, config["workdir"])
+
+    def __ghetto_glob(self, command):
+        new_command = []
+        for item in command:
+            if "*" in item:
+                dir = os.path.abspath(os.path.dirname(item))
+                base_name = os.path.basename(item)
+                if os.path.isdir(dir):
+                    item_parts = base_name.split("*")
+                    print(item_parts)
+                    for file in os.listdir(dir):
+                        # TODO: Fix ordering! A*B = B*A = AB*
+                        if item_parts[0] in file and item_parts[1] in file:
+                            new_command.append(os.path.join(dir, file))
+            else:
+                new_command.append(item)
+        return new_command
+
+    # Executes the given job in the one and only venv
+    # parameter shall be the raw jobscpec
+    def run(self, job_spec):
+        if "workdir" in job_spec:
+            pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"]))
+        else:
+            pwd = self.workdir
+        run_env = self.env_vars.copy()
+        if "env" in job_spec:
+            for env_var in job_spec["env"]:
+                run_env[env_var["name"]] = env_var["value"]
+        if "commands" in job_spec:
+            commands = job_spec["commands"]
+            for command in commands:
+                # TODO: only split if command is not an array
+                run_command = self.__ghetto_glob(shlex.split(command))
+                if os.path.isdir(pwd):
+                    with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p:
+                        p.wait()
+                        if p.returncode != 0:
+                            raise NonZeroRetcode(f"Command {command} returned code {p.returncode}")
+                else:
+                    raise RunnerError(f"PythonRunner: Invalid path for shell command: {pwd}")
+        else:
+            raise ConfigException(f"PythonRunner: No commands specified in step {job_spec['name']}")

alice-ci/src/alice/jobparser.py → alice-ci/src/alice/utils.py

@@ -1,7 +1,6 @@
 import yaml
-import shlex
-from pythonrunner import PythonRunner
-from exceptions import NonZeroRetcode
+from runners.pythonrunner import PythonRunner
+from exceptions import NonZeroRetcode, ConfigException
 
 
 class DummyRunner():
@@ -39,40 +38,41 @@ class Job():
             exit(1)
 
 
-class JobParser:
-    def __init__(self, file_path, repoDir, virtual_python) -> None:
+class ConfigParser:
+    def __init__(self, file_path, factory) -> None:
         with open(file_path) as f:
             self.config = yaml.safe_load(f)
-        self.jobs = self.__get_jobs(repoDir, virtual_python)
+        self.factory = factory
+        if "runners" in self.config:
+            if "global" in self.config["runners"]:
+                self.factory.set_globals(self.__gen_globals())
+            self.factory.update_runners(self.config["runners"])
+        self.jobs = self.__get_jobs()
+
+    # Initialize env, workdir if not present
+    def __gen_globals(self):
+        globals = self.config["runners"]["global"]
+        if "env" not in globals:
+            globals["env"] = []
+        if "workdir" not in globals:
+            globals["workdir"] = None
+        return globals
 
-    def __get_jobs(self, repoDir, virtual_python):
+    def __get_jobs(self):
         if "jobs" in self.config:
             jobs = {}
             for job_spec in self.config["jobs"]:
                 name = job_spec["name"]
                 if name in jobs:
-                    raise Exception(f"Job with name {name} already exists!")
-
-                job = Job(job_spec["type"],
-                          repoDir,
-                          virtual_python,
-                          job_spec.get("workdir", None),
-                          job_spec.get("env", None))
+                    raise ConfigException(f"Job with name {name} already exists!")
 
-                for cmd in job_spec["commands"]:
-                    job.commands.append(shlex.split(cmd))
-                jobs[name] = job
+                jobs[name] = job_spec
             return jobs
-
         else:
-            raise Exception("No jobs defined in config")
+            raise ConfigException("No jobs defined in config")
 
-    def get_modules(self):
-        modules = []
-        if "runners" in self.config:
-            if "python" in self.config["runners"]:
-                if "dependencies" in self.config["runners"]["python"]:
-                    for dep in self.config["runners"]["python"]["dependencies"]:
-                        # (name, i_name) if i_name is defined, else (name, name)
-                        modules.append((dep["name"], dep.get("import_name", dep["name"])))
-        return modules
+    def execute_job(self, job_name):
+        if job_name in self.jobs:
+            # Pass the job_spec to a runner
+            runner = self.factory.get_runner(self.jobs[job_name]["type"])
+            runner.run(self.jobs[job_name])

ci-examples/full.yaml

@@ -0,0 +1,29 @@
+runners:
+  global:
+    env:
+      - name: A
+        value: A
+      - name: B
+        value: B
+      - name: C
+        value: C
+    workdir: packages
+  python:
+    env:
+      - name: A
+        value: D
+    dependencies:
+      - flake8
+      - build
+jobs:
+  - name: env
+    type: python
+    env:
+      - name: B
+        value: E
+    commands:
+      - "-c \"import os; print(os.environ)\""
+  - name: lint
+    workdir: alice-ci
+    commands:
+      - "-m flake8 --ignore E501"