Starting py package

alice/pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

alice/setup.cfg

@@ -0,0 +1,23 @@
+[metadata]
+name = alice
+version = 0.0.1
+author = Daniel Gyulai
+description = Alice CI framework
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://git.gyulai.cloud/gyulaid/alice
+project_urls =
+    Bug Tracker = https://git.gyulai.cloud/gyulaid/alice/issues
+classifiers =
+    Programming Language :: Python :: 3
+    License :: OSI Approved :: MIT License
+    Operating System :: OS Independent
+
+[options]
+package_dir =
+    = src
+packages = find:
+python_requires = >=3.6
+
+[options.packages.find]
+where = src

alice/src/alice/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

alice/src/alice/cli.py

@@ -0,0 +1,85 @@
+# Sourcefor App class:
+# https://stackoverflow.com/questions/57593111/how-to-call-pip-from-a-python-script-and-make-it-install-locally-to-that-script
+import os
+import sys
+import subprocess
+import argparse
+
+
+class App:
+    def __init__(self, virtual_dir):
+        self.virtual_dir = virtual_dir
+        if os.name == "nt":
+            self.virtual_python = os.path.join(self.virtual_dir, "Scripts", "python.exe")
+        else:
+            self.virtual_python = os.path.join(self.virtual_dir, "bin", "python3")
+
+    def install_virtual_env(self):
+        self.pip_install("virtualenv")
+        if not os.path.exists(self.virtual_python):
+            import subprocess
+            subprocess.call([sys.executable, "-m", "virtualenv", self.virtual_dir])
+        else:
+            print("found virtual python: " + self.virtual_python)
+
+    def is_venv(self):
+        return sys.prefix == self.virtual_dir
+
+    def restart_under_venv(self):
+        print("Restarting under virtual environment " + self.virtual_dir)
+        with subprocess.Popen([self.virtual_python, __file__] + sys.argv[1:]) as p:
+            p.wait()
+            exit(p.returncode)
+
+    def pip_install(self, package, import_name=None):
+        try:
+            if import_name is None:
+                __import__(package)
+            else:
+                __import__(import_name)
+        except:  # noqa: E722
+            subprocess.call([sys.executable, "-m", "pip", "install", package, "--upgrade"])
+
+    def __gen_env(self, param_list):
+        env_vars = {}
+        for item in param_list:
+            item_parts = item.split("=")
+            if len(item_parts) == 2:
+                env_vars[item_parts[0]] = item_parts[1]
+        return env_vars
+
+    def run(self, args, repoDir):
+        if not self.is_venv():
+            self.install_virtual_env()
+            self.restart_under_venv()
+        else:
+            print("Running under virtual environment")
+            self.pip_install("pyyaml", "yaml")
+
+            from jobparser import JobParser
+            jobParser = JobParser(args.input, repoDir, self.virtual_python)
+            for name, import_name in jobParser.get_modules():
+                self.pip_install(name, import_name)
+
+            print("Begin pipeline steps...")
+            for step in args.steps:
+                if step in jobParser.jobs:
+                    jobParser.jobs[step].run_commands(self.__gen_env(args.env))
+                    print(f"Step {step}: SUCCESS")
+                else:
+                    raise Exception(f"Step {step} not found in {args.input}")
+
+def main():
+    pathToScriptDir = os.path.dirname(os.path.realpath(__file__))
+    repoDir = os.path.join(pathToScriptDir, "..")
+    app = App(os.path.join(pathToScriptDir, "venv"))
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("steps", nargs='+')
+    parser.add_argument("-i", "--input", default="ci.yaml")
+    parser.add_argument("-e", "--env", nargs='*', default=[])
+    args = parser.parse_args()
+    app.run(args, repoDir)
+
+if __name__ == "__main__":
+    main()

alice/src/alice/docker.py

@@ -0,0 +1 @@
+# TODO Implement

alice/src/alice/exceptions.py

@@ -0,0 +1,2 @@
+class NonZeroRetcode(Exception):
+    pass

alice/src/alice/jobparser.py

@@ -0,0 +1,78 @@
+import yaml
+import shlex
+from pythonrunner import PythonRunner
+from exceptions import NonZeroRetcode
+
+
+class DummyRunner():
+    def __init__(self, type) -> None:
+        self.type = type
+
+    def run(self, command, workdir=None, env=None):
+        raise Exception(f"Invalid runner type in config: {self.type}")
+
+
+class Job():
+    def __init__(self, type, repoDir, vpython, workspace, env={}) -> None:
+        self.runner = self.__get_runner(type, repoDir, vpython)
+        self.commands = []
+        self.workspace = workspace
+        self.env = env
+
+    def __get_runner(self, type, repoDir, vpython):
+        if type == "python":
+            return PythonRunner(repoDir, vpython)
+        else:
+            return DummyRunner(type)
+
+    def run_commands(self, _env={}):
+        try:
+            if self.env is None:
+                env = _env.copy()
+            else:
+                env = self.env.copy()
+                env.update(_env)
+            for command in self.commands:
+                self.runner.run(command, self.workspace, env)
+        except NonZeroRetcode as n:
+            print(n)
+            exit(1)
+
+
+class JobParser:
+    def __init__(self, file_path, repoDir, virtual_python) -> None:
+        with open(file_path) as f:
+            self.config = yaml.safe_load(f)
+        self.jobs = self.__get_jobs(repoDir, virtual_python)
+
+    def __get_jobs(self, repoDir, virtual_python):
+        if "jobs" in self.config:
+            jobs = {}
+            for job_spec in self.config["jobs"]:
+                name = job_spec["name"]
+                if name in jobs:
+                    raise Exception(f"Job with name {name} already exists!")
+
+                job = Job(job_spec["type"],
+                          repoDir,
+                          virtual_python,
+                          job_spec.get("workdir", None),
+                          job_spec.get("env", None))
+
+                for cmd in job_spec["commands"]:
+                    job.commands.append(shlex.split(cmd))
+                jobs[name] = job
+            return jobs
+
+        else:
+            raise Exception("No jobs defined in config")
+
+    def get_modules(self):
+        modules = []
+        if "runners" in self.config:
+            if "python" in self.config["runners"]:
+                if "dependencies" in self.config["runners"]["python"]:
+                    for dep in self.config["runners"]["python"]["dependencies"]:
+                        # (name, i_name) if i_name is defined, else (name, name)
+                        modules.append((dep["name"], dep.get("import_name", dep["name"])))
+        return modules

alice/src/alice/pythonrunner.py

@@ -0,0 +1,48 @@
+import subprocess
+import os
+
+from exceptions import NonZeroRetcode
+
+
+class PythonRunner():
+    def __init__(self, repo, vpython) -> None:
+        self.vpython = vpython
+        self.repopath = repo
+
+    def __get_env(self, overrides):
+        env = os.environ.copy()
+        if overrides is not None:
+            for key, value in overrides.items():
+                env[key] = value
+        return env
+
+    def ghetto_glob(self, command):
+        new_command = []
+        for item in command:
+            if "*" in item:
+                dir = os.path.abspath(os.path.dirname(item))
+                base_name = os.path.basename(item)
+                if os.path.isdir(dir):
+                    item_parts = base_name.split("*")
+                    print(item_parts)
+                    for file in os.listdir(dir):
+                        if item_parts[0] in file and item_parts[1] in file:
+                            new_command.append(os.path.join(dir, file))
+            else:
+                new_command.append(item)
+        return new_command
+
+    def run(self, command, workdir=None, env=None):
+        if workdir is not None:
+            pwd = os.path.abspath(os.path.join(self.repopath, workdir))
+        else:
+            pwd = self.repopath
+        run_env = self.__get_env(env)
+        run_command = self.ghetto_glob(command)
+        if os.path.isdir(pwd):
+            with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p:
+                p.wait()
+                if p.returncode != 0:
+                    raise NonZeroRetcode(f"Command {command} returned code {p.returncode}")
+        else:
+            raise Exception(f"Invalid path for shell command: {pwd}")