From ac6d38492a3c829a5b745e277652a48173875709 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 27 Mar 2022 13:16:07 +0200 Subject: [PATCH] Starting py package --- alice/README.md | 0 alice/pyproject.toml | 6 +++ alice/setup.cfg | 23 +++++++++ alice/src/alice/__init__.py | 0 alice/src/alice/__main__.py | 3 ++ alice/src/alice/cli.py | 85 +++++++++++++++++++++++++++++++++ alice/src/alice/docker.py | 1 + alice/src/alice/exceptions.py | 2 + alice/src/alice/jobparser.py | 78 ++++++++++++++++++++++++++++++ alice/src/alice/pythonrunner.py | 48 +++++++++++++++++++ 10 files changed, 246 insertions(+) create mode 100644 alice/README.md create mode 100644 alice/pyproject.toml create mode 100644 alice/setup.cfg create mode 100644 alice/src/alice/__init__.py create mode 100644 alice/src/alice/__main__.py create mode 100644 alice/src/alice/cli.py create mode 100644 alice/src/alice/docker.py create mode 100644 alice/src/alice/exceptions.py create mode 100644 alice/src/alice/jobparser.py create mode 100644 alice/src/alice/pythonrunner.py diff --git a/alice/README.md b/alice/README.md new file mode 100644 index 0000000..e69de29 diff --git a/alice/pyproject.toml b/alice/pyproject.toml new file mode 100644 index 0000000..b5a3c46 --- /dev/null +++ b/alice/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/alice/setup.cfg b/alice/setup.cfg new file mode 100644 index 0000000..a5ae477 --- /dev/null +++ b/alice/setup.cfg @@ -0,0 +1,23 @@ +[metadata] +name = alice +version = 0.0.1 +author = Daniel Gyulai +description = Alice CI framework +long_description = file: README.md +long_description_content_type = text/markdown +url = https://git.gyulai.cloud/gyulaid/alice +project_urls = + Bug Tracker = https://git.gyulai.cloud/gyulaid/alice/issues +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Operating System :: OS Independent + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.6 + +[options.packages.find] +where = src \ No newline at end of file diff --git a/alice/src/alice/__init__.py b/alice/src/alice/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/alice/src/alice/__main__.py b/alice/src/alice/__main__.py new file mode 100644 index 0000000..c49f9df --- /dev/null +++ b/alice/src/alice/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() \ No newline at end of file diff --git a/alice/src/alice/cli.py b/alice/src/alice/cli.py new file mode 100644 index 0000000..ff35b0d --- /dev/null +++ b/alice/src/alice/cli.py @@ -0,0 +1,85 @@ +# Sourcefor App class: +# https://stackoverflow.com/questions/57593111/how-to-call-pip-from-a-python-script-and-make-it-install-locally-to-that-script +import os +import sys +import subprocess +import argparse + + +class App: + def __init__(self, virtual_dir): + self.virtual_dir = virtual_dir + if os.name == "nt": + self.virtual_python = os.path.join(self.virtual_dir, "Scripts", "python.exe") + else: + self.virtual_python = os.path.join(self.virtual_dir, "bin", "python3") + + def install_virtual_env(self): + self.pip_install("virtualenv") + if not os.path.exists(self.virtual_python): + import subprocess + subprocess.call([sys.executable, "-m", "virtualenv", self.virtual_dir]) + else: + print("found virtual python: " + self.virtual_python) + + def is_venv(self): + return sys.prefix == self.virtual_dir + + def restart_under_venv(self): + print("Restarting under virtual environment " + self.virtual_dir) + with subprocess.Popen([self.virtual_python, __file__] + sys.argv[1:]) as p: + p.wait() + exit(p.returncode) + + def pip_install(self, package, import_name=None): + try: + if import_name is None: + __import__(package) + else: + __import__(import_name) + except: # noqa: E722 + subprocess.call([sys.executable, "-m", "pip", "install", package, "--upgrade"]) + + def __gen_env(self, param_list): + env_vars = {} + for item in param_list: + item_parts = item.split("=") + if len(item_parts) == 2: + env_vars[item_parts[0]] = item_parts[1] + return env_vars + + def run(self, args, repoDir): + if not self.is_venv(): + self.install_virtual_env() + self.restart_under_venv() + else: + print("Running under virtual environment") + self.pip_install("pyyaml", "yaml") + + from jobparser import JobParser + jobParser = JobParser(args.input, repoDir, self.virtual_python) + for name, import_name in jobParser.get_modules(): + self.pip_install(name, import_name) + + print("Begin pipeline steps...") + for step in args.steps: + if step in jobParser.jobs: + jobParser.jobs[step].run_commands(self.__gen_env(args.env)) + print(f"Step {step}: SUCCESS") + else: + raise Exception(f"Step {step} not found in {args.input}") + +def main(): + pathToScriptDir = os.path.dirname(os.path.realpath(__file__)) + repoDir = os.path.join(pathToScriptDir, "..") + app = App(os.path.join(pathToScriptDir, "venv")) + + parser = argparse.ArgumentParser() + parser.add_argument("steps", nargs='+') + parser.add_argument("-i", "--input", default="ci.yaml") + parser.add_argument("-e", "--env", nargs='*', default=[]) + args = parser.parse_args() + app.run(args, repoDir) + +if __name__ == "__main__": + main() diff --git a/alice/src/alice/docker.py b/alice/src/alice/docker.py new file mode 100644 index 0000000..b1dd741 --- /dev/null +++ b/alice/src/alice/docker.py @@ -0,0 +1 @@ +# TODO Implement diff --git a/alice/src/alice/exceptions.py b/alice/src/alice/exceptions.py new file mode 100644 index 0000000..475f20a --- /dev/null +++ b/alice/src/alice/exceptions.py @@ -0,0 +1,2 @@ +class NonZeroRetcode(Exception): + pass diff --git a/alice/src/alice/jobparser.py b/alice/src/alice/jobparser.py new file mode 100644 index 0000000..0f088b2 --- /dev/null +++ b/alice/src/alice/jobparser.py @@ -0,0 +1,78 @@ +import yaml +import shlex +from pythonrunner import PythonRunner +from exceptions import NonZeroRetcode + + +class DummyRunner(): + def __init__(self, type) -> None: + self.type = type + + def run(self, command, workdir=None, env=None): + raise Exception(f"Invalid runner type in config: {self.type}") + + +class Job(): + def __init__(self, type, repoDir, vpython, workspace, env={}) -> None: + self.runner = self.__get_runner(type, repoDir, vpython) + self.commands = [] + self.workspace = workspace + self.env = env + + def __get_runner(self, type, repoDir, vpython): + if type == "python": + return PythonRunner(repoDir, vpython) + else: + return DummyRunner(type) + + def run_commands(self, _env={}): + try: + if self.env is None: + env = _env.copy() + else: + env = self.env.copy() + env.update(_env) + for command in self.commands: + self.runner.run(command, self.workspace, env) + except NonZeroRetcode as n: + print(n) + exit(1) + + +class JobParser: + def __init__(self, file_path, repoDir, virtual_python) -> None: + with open(file_path) as f: + self.config = yaml.safe_load(f) + self.jobs = self.__get_jobs(repoDir, virtual_python) + + def __get_jobs(self, repoDir, virtual_python): + if "jobs" in self.config: + jobs = {} + for job_spec in self.config["jobs"]: + name = job_spec["name"] + if name in jobs: + raise Exception(f"Job with name {name} already exists!") + + job = Job(job_spec["type"], + repoDir, + virtual_python, + job_spec.get("workdir", None), + job_spec.get("env", None)) + + for cmd in job_spec["commands"]: + job.commands.append(shlex.split(cmd)) + jobs[name] = job + return jobs + + else: + raise Exception("No jobs defined in config") + + def get_modules(self): + modules = [] + if "runners" in self.config: + if "python" in self.config["runners"]: + if "dependencies" in self.config["runners"]["python"]: + for dep in self.config["runners"]["python"]["dependencies"]: + # (name, i_name) if i_name is defined, else (name, name) + modules.append((dep["name"], dep.get("import_name", dep["name"]))) + return modules diff --git a/alice/src/alice/pythonrunner.py b/alice/src/alice/pythonrunner.py new file mode 100644 index 0000000..9a4de56 --- /dev/null +++ b/alice/src/alice/pythonrunner.py @@ -0,0 +1,48 @@ +import subprocess +import os + +from exceptions import NonZeroRetcode + + +class PythonRunner(): + def __init__(self, repo, vpython) -> None: + self.vpython = vpython + self.repopath = repo + + def __get_env(self, overrides): + env = os.environ.copy() + if overrides is not None: + for key, value in overrides.items(): + env[key] = value + return env + + def ghetto_glob(self, command): + new_command = [] + for item in command: + if "*" in item: + dir = os.path.abspath(os.path.dirname(item)) + base_name = os.path.basename(item) + if os.path.isdir(dir): + item_parts = base_name.split("*") + print(item_parts) + for file in os.listdir(dir): + if item_parts[0] in file and item_parts[1] in file: + new_command.append(os.path.join(dir, file)) + else: + new_command.append(item) + return new_command + + def run(self, command, workdir=None, env=None): + if workdir is not None: + pwd = os.path.abspath(os.path.join(self.repopath, workdir)) + else: + pwd = self.repopath + run_env = self.__get_env(env) + run_command = self.ghetto_glob(command) + if os.path.isdir(pwd): + with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p: + p.wait() + if p.returncode != 0: + raise NonZeroRetcode(f"Command {command} returned code {p.returncode}") + else: + raise Exception(f"Invalid path for shell command: {pwd}")