5_DockerRunner (#26)

Fix #5

Co-authored-by: gyulaid <gyulaid@gyulai.cloud>
Reviewed-on: #26
Co-authored-by: Daniel Gyulai <gyulaid@gyulai.cloud>
Co-committed-by: Daniel Gyulai <gyulaid@gyulai.cloud>

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu/.devcontainer/base.Dockerfile
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/ubuntu/.devcontainer/base.Dockerfile
 
 # [Choice] Ubuntu version (use hirsuite or bionic on local arm64/Apple Silicon): hirsute, focal, bionic
 ARG VARIANT="hirsute"

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
-// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/ubuntu
 {
 	"name": "Ubuntu",
 	"build": {
@@ -25,6 +25,8 @@
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",
 	"features": {
-		"python": "latest"
+		"docker-from-docker": "20.10",
+		"git": "latest",
+		"python": "3.10"
 	}
 }

alice-ci/src/alice/config.py

@@ -0,0 +1,38 @@
+import logging
+import os
+
+from alice.exceptions import ConfigException
+
+
+class ConfigHolder:
+    __instance = None
+    file_name = ".alice"
+
+    @staticmethod
+    def getInstance():
+        """ Static access method. """
+        if ConfigHolder.__instance is None:
+            ConfigHolder()
+        return ConfigHolder.__instance
+
+    def __init__(self):
+        """ Virtually private constructor. """
+        if ConfigHolder.__instance is not None:
+            raise Exception("This class is a singleton!")
+        else:
+            ConfigHolder.__instance = self
+            config = os.path.abspath(os.path.join(os.getcwd(), self.file_name))
+            self.vars = {}
+            if os.path.isfile(config):
+                with open(config) as f:
+                    for line in f:
+                        items = line.split("=")
+                        if len(items) > 1:
+                            self.vars[items[0]] = line.replace(f"{items[0]}=", "")
+            logging.debug(f"Loaded from {self.file_name}: {self.vars}")
+
+    def get(self, key):
+        try:
+            self.vars[key]
+        except KeyError:
+            raise ConfigException(f"{key} not defined in .conf!")

alice-ci/src/alice/runnerfactory.py

@@ -3,6 +3,7 @@ from os.path import join, abspath
 
 from alice.runners.pythonrunner import PythonRunner
 from alice.runners.pypirunner import PyPiRunner
+from alice.runners.dockerrunner import DockerRunner
 from alice.exceptions import ConfigException
 
 
@@ -21,7 +22,8 @@ class Factory():
         # module = __import__("module_file")
         # my_class = getattr(module, "class_name")
         self.runnertypes = {"python": PythonRunner,
-                            "pypi": PyPiRunner}
+                            "pypi": PyPiRunner,
+                            "docker": DockerRunner}
 
         logging.info(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}")
 

alice-ci/src/alice/runners/dockerrunner.py

@@ -1 +1,223 @@
-# TODO Implement
+from enum import Enum
+import json
+import logging
+from os import path, getcwd
+import docker
+
+from alice.runners.pyutils import grab_from, gen_dict
+from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
+
+
+class ImageSource(Enum):
+    NONE = 1
+    BUILD = 2
+    PULL = 3
+
+
+def get_user(config, default):
+    if "credentials" in config:
+        if "username" in config["credentials"]:
+            data = config["credentials"]["username"]
+            if isinstance(data, str):
+                return data
+            else:
+                return grab_from(data)
+    return default
+
+
+def get_pass(config, default):
+    if "credentials" in config:
+        if "password" in config["credentials"]:
+            data = config["credentials"]["password"]
+            if isinstance(data, str):
+                return data
+            else:
+                return grab_from(data)
+    return default
+
+
+def get_provider(config, default, default_type):
+    if "image" in config:
+        build = False
+        pull = False
+        candidate_type = default_type
+        if "build" in config["image"]:
+            build = True
+            if default_type == ImageSource.BUILD:
+                candidate = default.copy(config["image"]["build"])
+            else:
+                candidate = Builder(config["image"]["build"])
+                candidate_type = ImageSource.BUILD
+        elif "pull" in config["image"]:
+            pull = True
+            if default_type == ImageSource.PULL:
+                candidate = default.copy(config["image"]["pull"])
+            else:
+                candidate = Puller(config["image"]["pull"])
+                candidate_type = ImageSource.PULL
+
+        if build and pull:
+            raise ConfigException("[DockerRunner] Can't build and pull the same image!")
+
+        return candidate, candidate_type
+    return default, default_type
+
+
+class Tagger:
+    def __init__(self, config={}) -> None:
+        self.name = config.get("name", None)
+        self.username = get_user(config, None)
+        self.password = get_pass(config, None)
+        self.publish = config.get("publish", False)
+
+    def copy(self, job_config):
+        t = Tagger()
+        t.name = job_config.get("name", self.name)
+        t.username = get_user(job_config, self.username)
+        t.password = get_pass(job_config, self.password)
+        t.publish = job_config.get("publish", self.publish)
+        return t
+
+    def __str__(self) -> str:
+        data = {
+            "name": self.name,
+            "publish": self.publish,
+            "credentials": {
+                "username": self.username,
+                "password": self.password
+            }
+        }
+        return f"{data}"
+
+
+class Builder():
+    def __init__(self, config) -> None:
+        self.dir = path.abspath(config.get("dir", getcwd()))
+        self.dockerfile = config.get("dockerfile", None)
+        self.name = config.get("name", None)
+        self.args = gen_dict(config.get("args", []))
+
+    def copy(self, job_config):
+        b = Builder({})
+        b.dir = path.abspath(path.join(self.dir, job_config.get("dir", ".")))
+        b.dockerfile = job_config.get("dockerfile", self.dockerfile)
+        b.name = job_config.get("name", self.name)
+        b.args = self.args.copy().update(gen_dict(job_config.get("args", [])))
+        return b
+
+    def __str__(self) -> str:
+        data = {
+            "type": "builder",
+            "dir": self.dir,
+            "dockerfile": self.dockerfile,
+            "name": self.name,
+            "args": self.args
+        }
+        return json.dumps(data)
+
+    def prepare(self, client):
+        print(f"[DockerRunner] Building image {self.name}")
+        if self.dockerfile is None:
+            self.dockerfile = "Dockerfile"
+        try:
+            image, log = client.images.build(path=self.dir,
+                                             dockerfile=self.dockerfile,
+                                             tag=self.name,
+                                             buildargs=self.args,
+                                             labels={"builder": "alice-ci"})
+            for i in log:
+                logging.debug(i)
+            return image
+        except docker.errors.BuildError as e:
+            raise RunnerError(f"[DockerRunner] Build failed: {e}")
+        except docker.errors.APIError as e:
+            raise RunnerError(f"[DockerRunner] Error: {e}")
+
+
+class Puller():
+    def __init__(self, config={}) -> None:
+        self.name = config.get("name", None)
+        self.username = get_user(config, None)
+        self.password = get_pass(config, None)
+
+    def copy(self, job_config={}):
+        p = Puller()
+        p.name = job_config.get("name", self.name)
+        p.username = get_user(job_config, self.username)
+        p.password = get_pass(job_config, self.password)
+
+    def __str__(self) -> str:
+        data = {
+            "name": self.name,
+            "credentials": {
+                "username": self.username,
+                "password": self.password
+            }
+        }
+        return f"{data}"
+
+    def prepare(self, client):
+        print(f"[DockerRunner] Pulling image {self.name}")
+        return client.images.pull(self.name)
+
+
+class DockerConfig:
+    def __init__(self, config={}) -> None:
+        self.username = get_user(config, None)
+        self.password = get_pass(config, None)
+        self.image_provider, self.provider_type = get_provider(config, None, ImageSource.NONE)
+        self.tagger = Tagger(config.get("tag", {}))
+        self.commands = config.get("commands", [])
+
+    def copy(self, job_config={}):
+        d = DockerConfig()
+        d.username = get_user(job_config, self.username)
+        d.password = get_pass(job_config, self.password)
+        d.image_provider, d.provider_type = get_provider(job_config, self.image_provider, self.provider_type)
+        d.tagger = self.tagger.copy(job_config.get("tag", {}))
+        d.commands = self.commands.copy() + job_config.get("commands", [])
+        return d
+
+    def __str__(self) -> str:
+        data = {
+            "credentials": {
+                "username": {self.username},
+                "password": {self.password}
+            },
+            "image": self.image_provider.__str__(),
+            "commands": self.commands,
+            "tag": self.tagger.__str__()
+        }
+        return f"{data}"
+
+
+class DockerRunner():
+    def __init__(self, config) -> None:
+        logging.info("[DockerRunner] Initializing")
+        self.config = DockerConfig(config)
+        self.client = docker.from_env()
+
+    def run(self, job_spec):
+        job_config = self.config.copy(job_spec)
+        if job_config.image_provider is None:
+            raise RunnerError("[DockerRunner] No image provider configured!")
+        image = job_config.image_provider.prepare(self.client)
+        logging.info(f"[DockerRunner] Image: {image.tags} ({image.id})")
+
+        if len(job_config.commands) > 0:
+            container = self.client.containers.run(image=image.id,
+                                                   entrypoint=["sleep", "infinity"],
+                                                   detach=True,
+                                                   auto_remove=True)
+            try:
+                for i in job_config.commands:
+                    command = ["/bin/sh", "-c", i]
+                    logging.debug(f"[DockerRunner] Command array: {command}")
+                    code, output = container.exec_run(cmd=command)
+                    for line in output.decode("UTF-8").splitlines():
+                        print(f"[{job_spec['name']}] {line}")
+                    if code != 0:
+                        raise NonZeroRetcode(f"Command {i} returned code {code}")
+            finally:
+                if container is not None:
+                    container.stop()

alice-ci/src/alice/runners/pypirunner.py

@@ -7,17 +7,10 @@ import sys
 from urllib import request, error
 from pkg_resources import parse_version
 from os import environ, path
-from alice.runners.pyutils import PackageManager, glob
+from alice.runners.pyutils import PackageManager, glob, grab_from
 from alice.exceptions import ConfigException, RunnerError
 
 
-def grab_from(target):
-    if "from_env" in target:
-        return environ[target["from_env"]]
-    else:
-        raise ConfigException(f"Unsupported grabber: {target.keys()}")
-
-
 def get_uri(config, default):
     url = config.get("repo", {}).get("uri", default)
     if url is not None:

alice-ci/src/alice/runners/pyutils.py

@@ -6,6 +6,7 @@ from pkg_resources import parse_version
 import re
 
 from alice.exceptions import RunnerError, ConfigException
+from alice.config import ConfigHolder
 
 
 class PackageManager:
@@ -24,7 +25,7 @@ class PackageManager:
             raise Exception("This class is a singleton!")
         else:
             PackageManager.__instance = self
-        self.package_list = self.__get_packages()
+            self.package_list = self.__get_packages()
 
     def __get_packages(self):
         packages = {}
@@ -111,3 +112,33 @@ def glob_command(command, workdir):
     for item in command:
         new_command += glob(item, workdir)
     return new_command
+
+
+def grab_from(target):
+    if "from_env" in target:
+        try:
+            return os.environ[target["from_env"]]
+        except KeyError:
+            raise ConfigException(f"Env var unset: {target['from_env']}")
+    elif "from_cfg" in target:
+        ConfigHolder.getInstance().get(target["from_cfg"])
+    else:
+        raise ConfigException(f"Unsupported grabber: {target.keys()}")
+
+
+def gen_dict(list_of_dicts):
+    """
+    Generates a dictionary from a list of dictionaries composed of
+    'name' and 'value' keys.
+
+    [{'name': 'a', 'value': 'b'}] => {'a': 'b'}
+    """
+    return_dict = {}
+
+    for _dict in list_of_dicts:
+        try:
+            return_dict[_dict["name"]] = _dict["value"]
+        except KeyError:
+            raise ConfigException(f"Invalid dict item: {_dict}")
+
+    return return_dict

ci-examples/full.yaml

@@ -15,6 +15,11 @@ runners:
     dependencies:
       - flake8
       - build
+  docker:
+    credentials:
+      username: D
+      password: D
+
 jobs:
   - name: env
     type: python
@@ -46,6 +51,36 @@ jobs:
         from_env: PYPIPASS
     packages:
      - alice-ci
+  - name: "image"
+    type: docker
+    credentials:
+      username: A
+      #password: B
+    image:
+      build:
+        dir: ci-examples/images/hello
+        #dockerfile: ci-examples/images/hello/Dockerfile
+        dockerfile: Dockerfile
+        name: "sssss"
+        args:
+          - name: CIPASS
+            value: NONE
+      #pull:
+        #name: python:latest
+        #credentials:
+          #username: PASS
+          #password: WORD
+    commands:
+      - which python3
+      - /usr/bin/python3 --version
+      - date
+    tag:
+      publish: true
+      name: published name with repo and everything
+      credentials:
+        username: B
+        password: B
+    
 pipelines:
   default:
     - lint

ci-examples/images/hello/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:latest
+
+RUN apt update && apt install -y python3
+
+ADD hello.py /opt/hello.py
+
+#ENTRYPOINT [ "/bin/sh", "-c" ]
+
+#CMD ["/usr/local/python/bin/python3", "/opt/hello.py"]

ci-examples/images/hello/hello.py

@@ -0,0 +1,2 @@
+if __name__ == "__main__":
+    print("Hi Mom!")

docs/runners/docker.md

@@ -0,0 +1,26 @@
+# Schema
+
+```
+name: ""
+type: docker
+credentials: - global ...ish
+  username
+  password
+image: - to use, pull, run
+  build:
+    dir:
+    dockerfile:
+    name: - defaults to step name
+    args:
+      - name:
+      - value:
+  pull:  - pulls, current working image - mutually exclusive with build
+    name:
+    credentials: - optional
+command: - overwrite, not append
+  - ...
+tag:
+  publish: true
+  name: - published name with repo and everything
+  credentials:
+```