From 51c78028f1f982c891d9c009c3583ae185615127 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 10 Apr 2022 21:43:20 +0000 Subject: [PATCH 1/9] Use python logging --- alice-ci/src/alice/cli.py | 12 +++++--- alice-ci/src/alice/configparser.py | 21 ++++++-------- alice-ci/src/alice/runnerfactory.py | 19 +++++-------- alice-ci/src/alice/runners/pypirunner.py | 13 ++++----- alice-ci/src/alice/runners/pythonrunner.py | 33 +++++++++------------- alice-ci/src/alice/runners/pyutils.py | 16 +++++------ 6 files changed, 49 insertions(+), 65 deletions(-) diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py index bfe3efb..5cfc193 100644 --- a/alice-ci/src/alice/cli.py +++ b/alice-ci/src/alice/cli.py @@ -1,3 +1,4 @@ +import logging import os import argparse @@ -20,9 +21,8 @@ def parse_jobs(args): try: if len(args.env) > 0: envs = gen_env(args.env) - if args.verbose: - print(f"[Alice] Env vars from CLI: {envs}") - jobParser = ConfigParser(args.input, gen_env(args.env), args.verbose) + logging.debug(f"[Alice] Env vars from CLI: {envs}") + jobParser = ConfigParser(args.input, gen_env(args.env)) for step in args.steps: jobParser.execute(step) @@ -42,8 +42,12 @@ def main(): parser.add_argument("-i", "--input", default="alice-ci.yaml") parser.add_argument("-e", "--env", nargs='*', default=[]) parser.add_argument("-a", "--addrunner", nargs='*', default=[]) - parser.add_argument("-v", "--verbose", action='store_true') + parser.add_argument('--verbose', '-v', action='count', default=0) args = parser.parse_args() + + loglevel = 30 - ((10 * args.verbose) if args.verbose > 0 else 0) + logging.basicConfig(level=loglevel, format='%(message)s') + if not os.path.isfile(args.input): print(f"No such file: {args.input}") exit(1) diff --git a/alice-ci/src/alice/configparser.py b/alice-ci/src/alice/configparser.py index bd4f698..f28c083 100644 --- a/alice-ci/src/alice/configparser.py +++ b/alice-ci/src/alice/configparser.py @@ -1,3 +1,4 @@ +import logging from os import getcwd, path, environ import subprocess import yaml @@ -7,11 +8,10 @@ from alice.runnerfactory import Factory class ConfigParser: - def __init__(self, file_path, cli_env_vars, verbose=False) -> None: - self.verbose = verbose + def __init__(self, file_path, cli_env_vars) -> None: with open(file_path) as f: self.config = yaml.safe_load(f) - self.factory = Factory(verbose, self.__gen_globals(cli_env_vars), self.config.get("runners", {})) + self.factory = Factory(self.__gen_globals(cli_env_vars), self.config.get("runners", {})) self.jobs = self.__get_jobs() self.pipelines = self.config.get("pipelines", {}) @@ -31,8 +31,7 @@ class ConfigParser: if "workdir" in self.config["runners"]["global"]: globals["workdir"] = self.config["runners"]["global"]["workdir"] - if (self.verbose): - print(f"[Alice] Configured globals: {globals}") + logging.debug(f"[Alice] Configured globals: {globals}") return globals def __get_jobs(self): @@ -44,8 +43,7 @@ class ConfigParser: raise ConfigException(f"Job with name {name} already exists!") jobs[name] = job_spec - if (self.verbose): - print(f"[Alice] Parsed jobs: {', '.join(jobs.keys())}") + logging.info(f"[Alice] Parsed jobs: {', '.join(jobs.keys())}") return jobs else: raise ConfigException("No jobs defined in config") @@ -65,9 +63,8 @@ class ConfigParser: for _path in paths: spec_path = path.abspath(_path) if change_path.startswith(spec_path): - if self.verbose: - print(f"[Alice] Modified file: {change_path}") - print(f"[Alice] Path match: {_path}") + logging.info(f"[Alice] Modified file: {change_path}") + logging.info(f"[Alice] Path match: {_path}") return True except KeyError: raise ConfigException(f"Invalid 'changes' config: {changes}") @@ -77,16 +74,16 @@ class ConfigParser: if task_name in self.jobs: self.execute_job(task_name) elif task_name in self.pipelines: - print(f"[Alice][Pipeline] {task_name}: Start") self.execute_pipeline(task_name) - print(f"[Alice][Pipeline] {task_name}: Success") else: raise ConfigException(f"No such job or pipeline: {task_name}") def execute_pipeline(self, pipeline_name): if pipeline_name in self.pipelines: + print(f"[Alice][Pipeline] {pipeline_name}: Start") for job in self.pipelines[pipeline_name]: self.execute_job(job) + print(f"[Alice][Pipeline] {pipeline_name}: Success") def execute_job(self, job_name): if job_name in self.jobs: diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py index 618248c..e587891 100644 --- a/alice-ci/src/alice/runnerfactory.py +++ b/alice-ci/src/alice/runnerfactory.py @@ -1,3 +1,4 @@ +import logging from os.path import join, abspath from alice.runners.pythonrunner import PythonRunner @@ -6,8 +7,7 @@ from alice.exceptions import ConfigException class Factory(): - def __init__(self, verbose, globals, runner_configs) -> None: - self.verbose = verbose + def __init__(self, globals, runner_configs) -> None: self.globals = globals self.runner_configs = {} self.runnertypes = {} @@ -23,14 +23,12 @@ class Factory(): self.runnertypes = {"python": PythonRunner, "pypi": PyPiRunner} - if (self.verbose): - print(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}") + logging.info(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}") def __gen_runner_configs(self, config): for runnertype, runnerconfig in config.items(): if runnertype != "global": - if (self.verbose): - print(f"[Alice] Global config found for runner {runnertype}") + logging.info(f"[Alice] Global config found for runner {runnertype}") config = self.globals.copy() for key, value in runnerconfig.items(): if key == "env": @@ -41,18 +39,15 @@ class Factory(): else: config[key] = value self.runner_configs[runnertype] = config + logging.debug(f"[Alice] Globals for {runnertype}: {runnerconfig}") def get_runner(self, runnertype): if runnertype not in self.runners: if runnertype in self.runnertypes: - if (self.verbose): - print(f"[Alice] Initializing runner: {runnertype}") - params = { - "verbose": self.verbose - } + logging.info(f"[Alice] Initializing runner: {runnertype}") # If there is a runner specific config, use that, else global config = self.runner_configs.get(runnertype, self.globals.copy()) - self.runners[runnertype] = self.runnertypes[runnertype](params, config) + self.runners[runnertype] = self.runnertypes[runnertype](config) else: raise ConfigException(f"Invalid runner type: {runnertype}") return self.runners[runnertype] diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py index fb821c3..5f0879c 100644 --- a/alice-ci/src/alice/runners/pypirunner.py +++ b/alice-ci/src/alice/runners/pypirunner.py @@ -1,4 +1,5 @@ import json +import logging import os import re import subprocess @@ -75,10 +76,8 @@ class PypiConfig: # TODO: consider "--skip-existing" flag for twine class PyPiRunner(): - def __init__(self, params, config) -> None: - self.verbose = params["verbose"] - if self.verbose: - print("[PyPiRunner] Initializing") + def __init__(self, config) -> None: + logging.info("[PyPiRunner] Initializing") self.workdir = config["workdir"] self.config = PypiConfig(config) @@ -126,9 +125,7 @@ class PyPiRunner(): def upload(self, config, package): - command = [sys.executable, "-m", "twine", "upload"] - if self.verbose: - command.append("--verbose") + command = [sys.executable, "-m", "twine", "upload", "--verbose"] if config.repo_uri is not None: command.append("--repository-url") command.append(config.repo_uri) @@ -163,7 +160,7 @@ class PyPiRunner(): PackageManager.getInstance().ensure("build") for package in job_config.packages: print(f"[PyPiRunner] Building {package}") - #self.build(job_config, package) + self.build(job_config, package) print(f"[PyPiRunner] Package {package} built") if job_config.upload: diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index 18011fc..f23bf8d 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -1,3 +1,4 @@ +import logging import subprocess import os import sys @@ -9,10 +10,8 @@ from alice.runners.pyutils import glob_command # TODO: Handle config like PyPiConfig class PythonRunner: - def __init__(self, params, config) -> None: - self.verbose = params["verbose"] - if self.verbose: - print("[PythonRunner] Initializing") + def __init__(self, config) -> None: + logging.info("[PythonRunner] Initializing") self.workdir = config["workdir"] self.virtual_dir = os.path.abspath(os.path.join(self.workdir, "venv")) self.config = config @@ -25,7 +24,7 @@ class PythonRunner: self.vpython = os.path.join(self.virtual_dir, "bin", "python3") if not os.path.exists(self.vpython): - print("[PythonRunner] Initializing venv") + logging.info("[PythonRunner] Initializing venv") with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: p.wait() @@ -33,26 +32,22 @@ class PythonRunner: sys.stdout.buffer.write(p.stderr.read()) raise RunnerError("[PythonRunner] Could not create virtualenv") else: - if self.verbose: - print(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}") + logging.info(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}") else: - if self.verbose: - print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}") + logging.info(f"[PythonRunner] Found virtualenv at {self.virtual_dir}") dependencies = self.config.get("dependencies", []) if len(dependencies) > 0: - if self.verbose: - print(f"[PythonRunner] Ensuring dependencies: {', '.join(dependencies)}") + logging.info(f"[PythonRunner] Ensuring dependencies: {', '.join(dependencies)}") command = [self.vpython, "-m", "pip", "install"] + dependencies with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: p.wait() if p.returncode != 0: sys.stdout.buffer.write(p.stderr.read()) raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})")) - if self.verbose: - print("[PythonRunner] Installation done") + logging.info("[PythonRunner] Installation done") # Executes the given job in the one and only venv - # parameter shall be the raw jobscpec + # parameter is the raw jobscpec def run(self, job_spec): if "workdir" in job_spec: pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"])) @@ -65,16 +60,14 @@ class PythonRunner: if "commands" in job_spec: commands = job_spec["commands"] for command in commands: - if self.verbose: - print(f"[PythonRunner] Raw command: {command}") + logging.debug(f"[PythonRunner] Raw command: {command}") # TODO: only split if command is not an array if "*" in command: - run_command = glob_command(shlex.split(command), pwd, self.verbose) + run_command = glob_command(shlex.split(command), pwd) else: run_command = shlex.split(command) - if self.verbose: - print(f"[PythonRunner] Command to execute: {run_command}") - print(f"[PythonRunner] Workdir: {pwd}") + logging.info(f"[PythonRunner] Command to execute: {run_command}") + logging.debug(f"[PythonRunner] Workdir: {pwd}") if os.path.isdir(pwd): with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p: p.wait() diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py index 62efdf7..6b3f6f1 100644 --- a/alice-ci/src/alice/runners/pyutils.py +++ b/alice-ci/src/alice/runners/pyutils.py @@ -1,3 +1,4 @@ +import logging import os import subprocess import sys @@ -83,11 +84,10 @@ class PackageManager: return False -def glob(item, workdir, verbose=False): +def glob(item, workdir): new_command = [] if "*" in item: - if verbose: - print(f"[Globbing] Found item: [{item}]") + logging.debug(f"[Globbing] Found item: [{item}]") dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item))) base_name = os.path.basename(item) if os.path.isdir(dir): @@ -96,8 +96,7 @@ def glob(item, workdir, verbose=False): # TODO: Fix ordering! A*B = B*A = AB* if item_parts[0] in file and item_parts[1] in file: new_item = os.path.join(dir, file) - if verbose: - print(f"[Globbing] Substitute: {new_item}") + logging.debug(f"[Globbing] Substitute: {new_item}") new_command.append(new_item) else: raise ConfigException(f"[Globbing] Dir not exists: {dir}") @@ -106,10 +105,9 @@ def glob(item, workdir, verbose=False): return [item] -def glob_command(command, workdir, verbose=False): - if verbose: - print(f"[Globbing] Starting command: {' '.join(command)}") +def glob_command(command, workdir): + logging.debug(f"[Globbing] Starting command: {' '.join(command)}") new_command = [] for item in command: - new_command += glob(item, workdir, verbose) + new_command += glob(item, workdir) return new_command -- 2.25.1 From d90c5b7659b97b6a76520852cd8f3775852e47a4 Mon Sep 17 00:00:00 2001 From: Daniel Gyulai Date: Sun, 17 Apr 2022 10:58:05 +0000 Subject: [PATCH 2/9] 5_DockerRunner (#26) Fix #5 Co-authored-by: gyulaid Reviewed-on: https://git.gyulai.cloud/gyulaid/alice/pulls/26 Co-authored-by: Daniel Gyulai Co-committed-by: Daniel Gyulai --- .devcontainer/Dockerfile | 2 +- .devcontainer/devcontainer.json | 6 +- alice-ci/src/alice/config.py | 38 ++++ alice-ci/src/alice/runnerfactory.py | 4 +- alice-ci/src/alice/runners/dockerrunner.py | 224 ++++++++++++++++++++- alice-ci/src/alice/runners/pypirunner.py | 9 +- alice-ci/src/alice/runners/pyutils.py | 33 ++- ci-examples/full.yaml | 35 ++++ ci-examples/images/hello/Dockerfile | 9 + ci-examples/images/hello/hello.py | 2 + docs/runners/docker.md | 26 +++ 11 files changed, 374 insertions(+), 14 deletions(-) create mode 100644 alice-ci/src/alice/config.py create mode 100644 ci-examples/images/hello/Dockerfile create mode 100644 ci-examples/images/hello/hello.py create mode 100644 docs/runners/docker.md diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7ab2f96..9e1c79c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu/.devcontainer/base.Dockerfile +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/ubuntu/.devcontainer/base.Dockerfile # [Choice] Ubuntu version (use hirsuite or bionic on local arm64/Apple Silicon): hirsute, focal, bionic ARG VARIANT="hirsute" diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6a078e8..816edeb 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu +// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/ubuntu { "name": "Ubuntu", "build": { @@ -25,6 +25,8 @@ // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode", "features": { - "python": "latest" + "docker-from-docker": "20.10", + "git": "latest", + "python": "3.10" } } diff --git a/alice-ci/src/alice/config.py b/alice-ci/src/alice/config.py new file mode 100644 index 0000000..3598fd3 --- /dev/null +++ b/alice-ci/src/alice/config.py @@ -0,0 +1,38 @@ +import logging +import os + +from alice.exceptions import ConfigException + + +class ConfigHolder: + __instance = None + file_name = ".alice" + + @staticmethod + def getInstance(): + """ Static access method. """ + if ConfigHolder.__instance is None: + ConfigHolder() + return ConfigHolder.__instance + + def __init__(self): + """ Virtually private constructor. """ + if ConfigHolder.__instance is not None: + raise Exception("This class is a singleton!") + else: + ConfigHolder.__instance = self + config = os.path.abspath(os.path.join(os.getcwd(), self.file_name)) + self.vars = {} + if os.path.isfile(config): + with open(config) as f: + for line in f: + items = line.split("=") + if len(items) > 1: + self.vars[items[0]] = line.replace(f"{items[0]}=", "") + logging.debug(f"Loaded from {self.file_name}: {self.vars}") + + def get(self, key): + try: + self.vars[key] + except KeyError: + raise ConfigException(f"{key} not defined in .conf!") diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py index e587891..11b7b41 100644 --- a/alice-ci/src/alice/runnerfactory.py +++ b/alice-ci/src/alice/runnerfactory.py @@ -3,6 +3,7 @@ from os.path import join, abspath from alice.runners.pythonrunner import PythonRunner from alice.runners.pypirunner import PyPiRunner +from alice.runners.dockerrunner import DockerRunner from alice.exceptions import ConfigException @@ -21,7 +22,8 @@ class Factory(): # module = __import__("module_file") # my_class = getattr(module, "class_name") self.runnertypes = {"python": PythonRunner, - "pypi": PyPiRunner} + "pypi": PyPiRunner, + "docker": DockerRunner} logging.info(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}") diff --git a/alice-ci/src/alice/runners/dockerrunner.py b/alice-ci/src/alice/runners/dockerrunner.py index 50a3260..3a84fed 100644 --- a/alice-ci/src/alice/runners/dockerrunner.py +++ b/alice-ci/src/alice/runners/dockerrunner.py @@ -1 +1,223 @@ -# TODO Implement +from enum import Enum +import json +import logging +from os import path, getcwd +import docker + +from alice.runners.pyutils import grab_from, gen_dict +from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError + + +class ImageSource(Enum): + NONE = 1 + BUILD = 2 + PULL = 3 + + +def get_user(config, default): + if "credentials" in config: + if "username" in config["credentials"]: + data = config["credentials"]["username"] + if isinstance(data, str): + return data + else: + return grab_from(data) + return default + + +def get_pass(config, default): + if "credentials" in config: + if "password" in config["credentials"]: + data = config["credentials"]["password"] + if isinstance(data, str): + return data + else: + return grab_from(data) + return default + + +def get_provider(config, default, default_type): + if "image" in config: + build = False + pull = False + candidate_type = default_type + if "build" in config["image"]: + build = True + if default_type == ImageSource.BUILD: + candidate = default.copy(config["image"]["build"]) + else: + candidate = Builder(config["image"]["build"]) + candidate_type = ImageSource.BUILD + elif "pull" in config["image"]: + pull = True + if default_type == ImageSource.PULL: + candidate = default.copy(config["image"]["pull"]) + else: + candidate = Puller(config["image"]["pull"]) + candidate_type = ImageSource.PULL + + if build and pull: + raise ConfigException("[DockerRunner] Can't build and pull the same image!") + + return candidate, candidate_type + return default, default_type + + +class Tagger: + def __init__(self, config={}) -> None: + self.name = config.get("name", None) + self.username = get_user(config, None) + self.password = get_pass(config, None) + self.publish = config.get("publish", False) + + def copy(self, job_config): + t = Tagger() + t.name = job_config.get("name", self.name) + t.username = get_user(job_config, self.username) + t.password = get_pass(job_config, self.password) + t.publish = job_config.get("publish", self.publish) + return t + + def __str__(self) -> str: + data = { + "name": self.name, + "publish": self.publish, + "credentials": { + "username": self.username, + "password": self.password + } + } + return f"{data}" + + +class Builder(): + def __init__(self, config) -> None: + self.dir = path.abspath(config.get("dir", getcwd())) + self.dockerfile = config.get("dockerfile", None) + self.name = config.get("name", None) + self.args = gen_dict(config.get("args", [])) + + def copy(self, job_config): + b = Builder({}) + b.dir = path.abspath(path.join(self.dir, job_config.get("dir", "."))) + b.dockerfile = job_config.get("dockerfile", self.dockerfile) + b.name = job_config.get("name", self.name) + b.args = self.args.copy().update(gen_dict(job_config.get("args", []))) + return b + + def __str__(self) -> str: + data = { + "type": "builder", + "dir": self.dir, + "dockerfile": self.dockerfile, + "name": self.name, + "args": self.args + } + return json.dumps(data) + + def prepare(self, client): + print(f"[DockerRunner] Building image {self.name}") + if self.dockerfile is None: + self.dockerfile = "Dockerfile" + try: + image, log = client.images.build(path=self.dir, + dockerfile=self.dockerfile, + tag=self.name, + buildargs=self.args, + labels={"builder": "alice-ci"}) + for i in log: + logging.debug(i) + return image + except docker.errors.BuildError as e: + raise RunnerError(f"[DockerRunner] Build failed: {e}") + except docker.errors.APIError as e: + raise RunnerError(f"[DockerRunner] Error: {e}") + + +class Puller(): + def __init__(self, config={}) -> None: + self.name = config.get("name", None) + self.username = get_user(config, None) + self.password = get_pass(config, None) + + def copy(self, job_config={}): + p = Puller() + p.name = job_config.get("name", self.name) + p.username = get_user(job_config, self.username) + p.password = get_pass(job_config, self.password) + + def __str__(self) -> str: + data = { + "name": self.name, + "credentials": { + "username": self.username, + "password": self.password + } + } + return f"{data}" + + def prepare(self, client): + print(f"[DockerRunner] Pulling image {self.name}") + return client.images.pull(self.name) + + +class DockerConfig: + def __init__(self, config={}) -> None: + self.username = get_user(config, None) + self.password = get_pass(config, None) + self.image_provider, self.provider_type = get_provider(config, None, ImageSource.NONE) + self.tagger = Tagger(config.get("tag", {})) + self.commands = config.get("commands", []) + + def copy(self, job_config={}): + d = DockerConfig() + d.username = get_user(job_config, self.username) + d.password = get_pass(job_config, self.password) + d.image_provider, d.provider_type = get_provider(job_config, self.image_provider, self.provider_type) + d.tagger = self.tagger.copy(job_config.get("tag", {})) + d.commands = self.commands.copy() + job_config.get("commands", []) + return d + + def __str__(self) -> str: + data = { + "credentials": { + "username": {self.username}, + "password": {self.password} + }, + "image": self.image_provider.__str__(), + "commands": self.commands, + "tag": self.tagger.__str__() + } + return f"{data}" + + +class DockerRunner(): + def __init__(self, config) -> None: + logging.info("[DockerRunner] Initializing") + self.config = DockerConfig(config) + self.client = docker.from_env() + + def run(self, job_spec): + job_config = self.config.copy(job_spec) + if job_config.image_provider is None: + raise RunnerError("[DockerRunner] No image provider configured!") + image = job_config.image_provider.prepare(self.client) + logging.info(f"[DockerRunner] Image: {image.tags} ({image.id})") + + if len(job_config.commands) > 0: + container = self.client.containers.run(image=image.id, + entrypoint=["sleep", "infinity"], + detach=True, + auto_remove=True) + try: + for i in job_config.commands: + command = ["/bin/sh", "-c", i] + logging.debug(f"[DockerRunner] Command array: {command}") + code, output = container.exec_run(cmd=command) + for line in output.decode("UTF-8").splitlines(): + print(f"[{job_spec['name']}] {line}") + if code != 0: + raise NonZeroRetcode(f"Command {i} returned code {code}") + finally: + if container is not None: + container.stop() diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py index 5f0879c..10918bd 100644 --- a/alice-ci/src/alice/runners/pypirunner.py +++ b/alice-ci/src/alice/runners/pypirunner.py @@ -7,17 +7,10 @@ import sys from urllib import request, error from pkg_resources import parse_version from os import environ, path -from alice.runners.pyutils import PackageManager, glob +from alice.runners.pyutils import PackageManager, glob, grab_from from alice.exceptions import ConfigException, RunnerError -def grab_from(target): - if "from_env" in target: - return environ[target["from_env"]] - else: - raise ConfigException(f"Unsupported grabber: {target.keys()}") - - def get_uri(config, default): url = config.get("repo", {}).get("uri", default) if url is not None: diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py index 6b3f6f1..0446cb4 100644 --- a/alice-ci/src/alice/runners/pyutils.py +++ b/alice-ci/src/alice/runners/pyutils.py @@ -6,6 +6,7 @@ from pkg_resources import parse_version import re from alice.exceptions import RunnerError, ConfigException +from alice.config import ConfigHolder class PackageManager: @@ -24,7 +25,7 @@ class PackageManager: raise Exception("This class is a singleton!") else: PackageManager.__instance = self - self.package_list = self.__get_packages() + self.package_list = self.__get_packages() def __get_packages(self): packages = {} @@ -111,3 +112,33 @@ def glob_command(command, workdir): for item in command: new_command += glob(item, workdir) return new_command + + +def grab_from(target): + if "from_env" in target: + try: + return os.environ[target["from_env"]] + except KeyError: + raise ConfigException(f"Env var unset: {target['from_env']}") + elif "from_cfg" in target: + ConfigHolder.getInstance().get(target["from_cfg"]) + else: + raise ConfigException(f"Unsupported grabber: {target.keys()}") + + +def gen_dict(list_of_dicts): + """ + Generates a dictionary from a list of dictionaries composed of + 'name' and 'value' keys. + + [{'name': 'a', 'value': 'b'}] => {'a': 'b'} + """ + return_dict = {} + + for _dict in list_of_dicts: + try: + return_dict[_dict["name"]] = _dict["value"] + except KeyError: + raise ConfigException(f"Invalid dict item: {_dict}") + + return return_dict diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml index 108ba92..b5f086c 100644 --- a/ci-examples/full.yaml +++ b/ci-examples/full.yaml @@ -15,6 +15,11 @@ runners: dependencies: - flake8 - build + docker: + credentials: + username: D + password: D + jobs: - name: env type: python @@ -46,6 +51,36 @@ jobs: from_env: PYPIPASS packages: - alice-ci + - name: "image" + type: docker + credentials: + username: A + #password: B + image: + build: + dir: ci-examples/images/hello + #dockerfile: ci-examples/images/hello/Dockerfile + dockerfile: Dockerfile + name: "sssss" + args: + - name: CIPASS + value: NONE + #pull: + #name: python:latest + #credentials: + #username: PASS + #password: WORD + commands: + - which python3 + - /usr/bin/python3 --version + - date + tag: + publish: true + name: published name with repo and everything + credentials: + username: B + password: B + pipelines: default: - lint diff --git a/ci-examples/images/hello/Dockerfile b/ci-examples/images/hello/Dockerfile new file mode 100644 index 0000000..17cd4aa --- /dev/null +++ b/ci-examples/images/hello/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:latest + +RUN apt update && apt install -y python3 + +ADD hello.py /opt/hello.py + +#ENTRYPOINT [ "/bin/sh", "-c" ] + +#CMD ["/usr/local/python/bin/python3", "/opt/hello.py"] \ No newline at end of file diff --git a/ci-examples/images/hello/hello.py b/ci-examples/images/hello/hello.py new file mode 100644 index 0000000..617e813 --- /dev/null +++ b/ci-examples/images/hello/hello.py @@ -0,0 +1,2 @@ +if __name__ == "__main__": + print("Hi Mom!") diff --git a/docs/runners/docker.md b/docs/runners/docker.md new file mode 100644 index 0000000..0ae0853 --- /dev/null +++ b/docs/runners/docker.md @@ -0,0 +1,26 @@ +# Schema + +``` +name: "" +type: docker +credentials: - global ...ish + username + password +image: - to use, pull, run + build: + dir: + dockerfile: + name: - defaults to step name + args: + - name: + - value: + pull: - pulls, current working image - mutually exclusive with build + name: + credentials: - optional +command: - overwrite, not append + - ... +tag: + publish: true + name: - published name with repo and everything + credentials: +``` \ No newline at end of file -- 2.25.1 From 20d0a45a34fd481163b75f63110026ee1b243099 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 17 Apr 2022 12:58:48 +0200 Subject: [PATCH 3/9] Version bump --- alice-ci/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index 3fb1ffb..02171ed 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = alice-ci -version = 0.0.9 +version = 0.0.10 author = Daniel Gyulai description = Alice CI framework long_description = file: README.md -- 2.25.1 From b75f2358b8d3070165ee608bf7a96c64a38474b6 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 17 Apr 2022 13:38:32 +0200 Subject: [PATCH 4/9] Fix #25 get rid of dependency versions --- alice-ci/setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index 02171ed..2c4cdee 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -19,8 +19,8 @@ package_dir = packages = alice python_requires = >=3.6 install_requires = - PyYAML==6.0 - virtualenv==20.14.0 + PyYAML + virtualenv [options.entry_points] console_scripts = -- 2.25.1 From 24cdacfcb050279ca61dccba541458e7fa196659 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 17 Apr 2022 14:17:12 +0200 Subject: [PATCH 5/9] Close #28 --- alice-ci/setup.cfg | 8 +++++--- alice-ci/src/alice/__init__.py | 13 ++++++------- alice-ci/src/alice/__main__.py | 5 +++-- alice-ci/src/alice/cli.py | 4 ++-- alice-ci/src/alice/config.py | 2 +- alice-ci/src/alice/configparser.py | 4 ++-- alice-ci/src/alice/runnerfactory.py | 8 ++++---- alice-ci/src/alice/runners/__init__.py | 5 +++-- alice-ci/src/alice/runners/dockerrunner.py | 4 ++-- alice-ci/src/alice/runners/pypirunner.py | 1 - alice-ci/src/alice/runners/pythonrunner.py | 4 ++-- alice-ci/src/alice/runners/pyutils.py | 4 ++-- 12 files changed, 32 insertions(+), 30 deletions(-) diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index 2c4cdee..20b563d 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -16,7 +16,9 @@ classifiers = [options] package_dir = = src -packages = alice +packages = + alice + alice.runners python_requires = >=3.6 install_requires = PyYAML @@ -26,5 +28,5 @@ install_requires = console_scripts = alice = alice.cli:main -[options.packages.find] -where = src \ No newline at end of file +#[options.packages.find] +#where = src \ No newline at end of file diff --git a/alice-ci/src/alice/__init__.py b/alice-ci/src/alice/__init__.py index 3c513fe..9a0f501 100644 --- a/alice-ci/src/alice/__init__.py +++ b/alice-ci/src/alice/__init__.py @@ -1,10 +1,9 @@ # flake8: noqa F401 -from alice.configparser import ConfigParser -from alice.exceptions import NonZeroRetcode -from alice.runnerfactory import Factory -from alice.runners.pythonrunner import PythonRunner -from alice.exceptions import NonZeroRetcode -from alice.exceptions import RunnerError -from alice.exceptions import ConfigException +from .configparser import ConfigParser +from .exceptions import NonZeroRetcode +from .runnerfactory import Factory +from .exceptions import NonZeroRetcode +from .exceptions import RunnerError +from .exceptions import ConfigException name = "alice" \ No newline at end of file diff --git a/alice-ci/src/alice/__main__.py b/alice-ci/src/alice/__main__.py index a4d3a54..d5cb563 100644 --- a/alice-ci/src/alice/__main__.py +++ b/alice-ci/src/alice/__main__.py @@ -1,3 +1,4 @@ -from alice.cli import main +import alice -main() +if __name__ == '__main__': + alice.cli.main() diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py index 5cfc193..5630055 100644 --- a/alice-ci/src/alice/cli.py +++ b/alice-ci/src/alice/cli.py @@ -2,8 +2,8 @@ import logging import os import argparse -from alice.configparser import ConfigParser -from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError +from .configparser import ConfigParser +from .exceptions import ConfigException, NonZeroRetcode, RunnerError def gen_env(param_list): diff --git a/alice-ci/src/alice/config.py b/alice-ci/src/alice/config.py index 3598fd3..d0d473b 100644 --- a/alice-ci/src/alice/config.py +++ b/alice-ci/src/alice/config.py @@ -1,7 +1,7 @@ import logging import os -from alice.exceptions import ConfigException +from .exceptions import ConfigException class ConfigHolder: diff --git a/alice-ci/src/alice/configparser.py b/alice-ci/src/alice/configparser.py index f28c083..d65b092 100644 --- a/alice-ci/src/alice/configparser.py +++ b/alice-ci/src/alice/configparser.py @@ -3,8 +3,8 @@ from os import getcwd, path, environ import subprocess import yaml -from alice.exceptions import ConfigException -from alice.runnerfactory import Factory +from .exceptions import ConfigException +from .runnerfactory import Factory class ConfigParser: diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py index 11b7b41..424d213 100644 --- a/alice-ci/src/alice/runnerfactory.py +++ b/alice-ci/src/alice/runnerfactory.py @@ -1,10 +1,10 @@ import logging from os.path import join, abspath -from alice.runners.pythonrunner import PythonRunner -from alice.runners.pypirunner import PyPiRunner -from alice.runners.dockerrunner import DockerRunner -from alice.exceptions import ConfigException +from .runners.pythonrunner import PythonRunner +from .runners.pypirunner import PyPiRunner +from .runners.dockerrunner import DockerRunner +from .exceptions import ConfigException class Factory(): diff --git a/alice-ci/src/alice/runners/__init__.py b/alice-ci/src/alice/runners/__init__.py index b67238e..8c0f819 100644 --- a/alice-ci/src/alice/runners/__init__.py +++ b/alice-ci/src/alice/runners/__init__.py @@ -1,3 +1,4 @@ # flake8: noqa F401 -from alice.runners.pythonrunner import PythonRunner -from alice.runners.pypirunner import PyPiRunner +from .pythonrunner import PythonRunner +from .pypirunner import PyPiRunner +from .dockerrunner import DockerRunner diff --git a/alice-ci/src/alice/runners/dockerrunner.py b/alice-ci/src/alice/runners/dockerrunner.py index 3a84fed..45a500c 100644 --- a/alice-ci/src/alice/runners/dockerrunner.py +++ b/alice-ci/src/alice/runners/dockerrunner.py @@ -4,8 +4,8 @@ import logging from os import path, getcwd import docker -from alice.runners.pyutils import grab_from, gen_dict -from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError +from .pyutils import grab_from, gen_dict +from ..exceptions import ConfigException, NonZeroRetcode, RunnerError class ImageSource(Enum): diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py index 10918bd..63d0d04 100644 --- a/alice-ci/src/alice/runners/pypirunner.py +++ b/alice-ci/src/alice/runners/pypirunner.py @@ -87,7 +87,6 @@ class PyPiRunner(): return sorted(releases, key=parse_version, reverse=True) def build(self, config, package): - # TODO: Actual build - silent, unless failure! pkg_path = path.join(config.workdir, package) if not path.isdir(pkg_path): raise ConfigException(f"Path does not exists: {pkg_path}") diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index f23bf8d..07f8a8c 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -4,8 +4,8 @@ import os import sys import shlex -from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException -from alice.runners.pyutils import glob_command +from ..exceptions import NonZeroRetcode, RunnerError, ConfigException +from .pyutils import glob_command # TODO: Handle config like PyPiConfig diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py index 0446cb4..83b155c 100644 --- a/alice-ci/src/alice/runners/pyutils.py +++ b/alice-ci/src/alice/runners/pyutils.py @@ -5,8 +5,8 @@ import sys from pkg_resources import parse_version import re -from alice.exceptions import RunnerError, ConfigException -from alice.config import ConfigHolder +from ..exceptions import RunnerError, ConfigException +from ..config import ConfigHolder class PackageManager: -- 2.25.1 From 472ecc89bcb275975bd03cafac24d761c5b3ab58 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 17 Apr 2022 21:50:47 +0200 Subject: [PATCH 6/9] Closes #27 Env vars in docker exec --- alice-ci/setup.cfg | 1 + alice-ci/src/alice/runners/dockerrunner.py | 8 +++++++- ci-examples/full.yaml | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index 20b563d..cf915bc 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -23,6 +23,7 @@ python_requires = >=3.6 install_requires = PyYAML virtualenv + docker [options.entry_points] console_scripts = diff --git a/alice-ci/src/alice/runners/dockerrunner.py b/alice-ci/src/alice/runners/dockerrunner.py index 45a500c..faf80a0 100644 --- a/alice-ci/src/alice/runners/dockerrunner.py +++ b/alice-ci/src/alice/runners/dockerrunner.py @@ -168,6 +168,7 @@ class DockerConfig: self.image_provider, self.provider_type = get_provider(config, None, ImageSource.NONE) self.tagger = Tagger(config.get("tag", {})) self.commands = config.get("commands", []) + self.env = config.get("env", {}) def copy(self, job_config={}): d = DockerConfig() @@ -176,6 +177,8 @@ class DockerConfig: d.image_provider, d.provider_type = get_provider(job_config, self.image_provider, self.provider_type) d.tagger = self.tagger.copy(job_config.get("tag", {})) d.commands = self.commands.copy() + job_config.get("commands", []) + d.env = self.env.copy() + d.env.update(gen_dict(job_config.get("env", []))) return d def __str__(self) -> str: @@ -205,6 +208,8 @@ class DockerRunner(): logging.info(f"[DockerRunner] Image: {image.tags} ({image.id})") if len(job_config.commands) > 0: + if "PATH" in job_config.env: + del job_config.env["PATH"] container = self.client.containers.run(image=image.id, entrypoint=["sleep", "infinity"], detach=True, @@ -213,7 +218,8 @@ class DockerRunner(): for i in job_config.commands: command = ["/bin/sh", "-c", i] logging.debug(f"[DockerRunner] Command array: {command}") - code, output = container.exec_run(cmd=command) + code, output = container.exec_run(cmd=command, + environment=job_config.env) for line in output.decode("UTF-8").splitlines(): print(f"[{job_spec['name']}] {line}") if code != 0: diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml index b5f086c..a4f24f9 100644 --- a/ci-examples/full.yaml +++ b/ci-examples/full.yaml @@ -70,10 +70,14 @@ jobs: #credentials: #username: PASS #password: WORD + env: + - name: VAR + value: CHAR commands: - which python3 - /usr/bin/python3 --version - date + - env tag: publish: true name: published name with repo and everything -- 2.25.1 From bfb997ae956efbbcbc2777e6308a668bf81cd780 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 17 Apr 2022 22:22:06 +0200 Subject: [PATCH 7/9] Grabber support in gen_dict --- alice-ci/src/alice/runners/pyutils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py index 83b155c..e37fb60 100644 --- a/alice-ci/src/alice/runners/pyutils.py +++ b/alice-ci/src/alice/runners/pyutils.py @@ -137,7 +137,10 @@ def gen_dict(list_of_dicts): for _dict in list_of_dicts: try: - return_dict[_dict["name"]] = _dict["value"] + if isinstance(_dict["value"], str): + return_dict[_dict["name"]] = _dict["value"] + else: + return_dict[_dict["name"]] = grab_from(_dict["value"]) except KeyError: raise ConfigException(f"Invalid dict item: {_dict}") -- 2.25.1 From 26ccb6985eb2c40eb651f5ac6262c7a6866a1413 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 17 Apr 2022 22:59:37 +0200 Subject: [PATCH 8/9] Close #30 Added upload + tagging --- alice-ci/src/alice/runners/dockerrunner.py | 12 ++++++++++++ ci-examples/full.yaml | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/alice-ci/src/alice/runners/dockerrunner.py b/alice-ci/src/alice/runners/dockerrunner.py index faf80a0..7383f7f 100644 --- a/alice-ci/src/alice/runners/dockerrunner.py +++ b/alice-ci/src/alice/runners/dockerrunner.py @@ -89,6 +89,15 @@ class Tagger: } return f"{data}" + def handle(self, client, image): + if self.name is not None: + if self.name not in image.tags and f"{self.name}:latest" not in image.tags: + print(f"[DockerRunner] Tagging {image.tags[0]} as {self.name}") + image.tag(self.name) + if self.publish: + print(f"[DockerRunner] Pushing {self.name}") + client.push(self.name) + class Builder(): def __init__(self, config) -> None: @@ -202,6 +211,7 @@ class DockerRunner(): def run(self, job_spec): job_config = self.config.copy(job_spec) + logging.debug(f"[DockerRunner] Job config: {job_config.__str__()}") if job_config.image_provider is None: raise RunnerError("[DockerRunner] No image provider configured!") image = job_config.image_provider.prepare(self.client) @@ -227,3 +237,5 @@ class DockerRunner(): finally: if container is not None: container.stop() + + job_config.tagger.handle(self.client, image) diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml index a4f24f9..0e5714f 100644 --- a/ci-examples/full.yaml +++ b/ci-examples/full.yaml @@ -79,8 +79,8 @@ jobs: - date - env tag: - publish: true - name: published name with repo and everything + publish: false + name: repo.example.com/test/na credentials: username: B password: B -- 2.25.1 From 83dfa63035f86cff13f68b8ed3e979b0cb372b1a Mon Sep 17 00:00:00 2001 From: gyulaid Date: Mon, 18 Apr 2022 15:37:59 +0200 Subject: [PATCH 9/9] #23 PackageMananger for virtualenv --- alice-ci/setup.cfg | 4 ---- alice-ci/src/alice/runners/pythonrunner.py | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index cf915bc..8c1f9c8 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -22,12 +22,8 @@ packages = python_requires = >=3.6 install_requires = PyYAML - virtualenv docker [options.entry_points] console_scripts = alice = alice.cli:main - -#[options.packages.find] -#where = src \ No newline at end of file diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index 07f8a8c..ea1351a 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -5,7 +5,7 @@ import sys import shlex from ..exceptions import NonZeroRetcode, RunnerError, ConfigException -from .pyutils import glob_command +from .pyutils import PackageManager, glob_command # TODO: Handle config like PyPiConfig @@ -15,6 +15,7 @@ class PythonRunner: self.workdir = config["workdir"] self.virtual_dir = os.path.abspath(os.path.join(self.workdir, "venv")) self.config = config + PackageManager.getInstance().ensure("build") self.__init_venv() def __init_venv(self): -- 2.25.1