Docker runner

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu/.devcontainer/base.Dockerfile
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/ubuntu/.devcontainer/base.Dockerfile
 
 # [Choice] Ubuntu version (use hirsuite or bionic on local arm64/Apple Silicon): hirsute, focal, bionic
 ARG VARIANT="hirsute"

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
-// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/ubuntu
 {
 	"name": "Ubuntu",
 	"build": {
@@ -25,6 +25,8 @@
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",
 	"features": {
-		"python": "latest"
+		"docker-from-docker": "20.10",
+		"git": "latest",
+		"python": "3.10"
 	}
 }

alice-ci/src/alice/cli.py

@@ -6,40 +6,6 @@ from alice.configparser import ConfigParser
 from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
 
 
-class ConfigHolder:
-    __instance = None
-    file_name = ".alice"
-
-    @staticmethod
-    def getInstance():
-        """ Static access method. """
-        if ConfigHolder.__instance is None:
-            ConfigHolder()
-        return ConfigHolder.__instance
-
-    def __init__(self):
-        """ Virtually private constructor. """
-        if ConfigHolder.__instance is not None:
-            raise Exception("This class is a singleton!")
-        else:
-            ConfigHolder.__instance = self
-            config = os.path.abspath(os.path.join(os.getcwd(), self.file_name))
-            self.vars = {}
-            if os.path.isfile(config):
-                with open(config) as f:
-                    for line in f:
-                        items = line.split("=")
-                        if len(items) > 1:
-                            self.vars[items[0]] = line.replace(f"{items[0]}=", "")
-            logging.debug(f"Loaded from {self.file_name}: {self.vars}")
-
-    def get(self, key):
-        try:
-            self.vars[key]
-        except KeyError:
-            raise ConfigException(f"{key} not defined in .conf!")
-
-
 def gen_env(param_list):
     env_vars = {}
     for item in param_list:

alice-ci/src/alice/config.py

@@ -0,0 +1,38 @@
+import logging
+import os
+
+from alice.exceptions import ConfigException
+
+
+class ConfigHolder:
+    __instance = None
+    file_name = ".alice"
+
+    @staticmethod
+    def getInstance():
+        """ Static access method. """
+        if ConfigHolder.__instance is None:
+            ConfigHolder()
+        return ConfigHolder.__instance
+
+    def __init__(self):
+        """ Virtually private constructor. """
+        if ConfigHolder.__instance is not None:
+            raise Exception("This class is a singleton!")
+        else:
+            ConfigHolder.__instance = self
+            config = os.path.abspath(os.path.join(os.getcwd(), self.file_name))
+            self.vars = {}
+            if os.path.isfile(config):
+                with open(config) as f:
+                    for line in f:
+                        items = line.split("=")
+                        if len(items) > 1:
+                            self.vars[items[0]] = line.replace(f"{items[0]}=", "")
+            logging.debug(f"Loaded from {self.file_name}: {self.vars}")
+
+    def get(self, key):
+        try:
+            self.vars[key]
+        except KeyError:
+            raise ConfigException(f"{key} not defined in .conf!")

alice-ci/src/alice/runners/dockerrunner.py

@@ -1,19 +1,223 @@
-# TODO Implement
-class DockerConfig:
+from enum import Enum
+import json
+import logging
+from os import path, getcwd
+import docker
+
+from alice.runners.pyutils import grab_from, gen_dict
+from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
+
+
+class ImageSource(Enum):
+    NONE = 1
+    BUILD = 2
+    PULL = 3
+
+
+def get_user(config, default):
+    if "credentials" in config:
+        if "username" in config["credentials"]:
+            data = config["credentials"]["username"]
+            if isinstance(data, str):
+                return data
+            else:
+                return grab_from(data)
+    return default
+
+
+def get_pass(config, default):
+    if "credentials" in config:
+        if "password" in config["credentials"]:
+            data = config["credentials"]["password"]
+            if isinstance(data, str):
+                return data
+            else:
+                return grab_from(data)
+    return default
+
+
+def get_provider(config, default, default_type):
+    if "image" in config:
+        build = False
+        pull = False
+        candidate_type = default_type
+        if "build" in config["image"]:
+            build = True
+            if default_type == ImageSource.BUILD:
+                candidate = default.copy(config["image"]["build"])
+            else:
+                candidate = Builder(config["image"]["build"])
+                candidate_type = ImageSource.BUILD
+        elif "pull" in config["image"]:
+            pull = True
+            if default_type == ImageSource.PULL:
+                candidate = default.copy(config["image"]["pull"])
+            else:
+                candidate = Puller(config["image"]["pull"])
+                candidate_type = ImageSource.PULL
+
+        if build and pull:
+            raise ConfigException("[DockerRunner] Can't build and pull the same image!")
+
+        return candidate, candidate_type
+    return default, default_type
+
+
+class Tagger:
     def __init__(self, config={}) -> None:
-        pass
+        self.name = config.get("name", None)
+        self.username = get_user(config, None)
+        self.password = get_pass(config, None)
+        self.publish = config.get("publish", False)
+
+    def copy(self, job_config):
+        t = Tagger()
+        t.name = job_config.get("name", self.name)
+        t.username = get_user(job_config, self.username)
+        t.password = get_pass(job_config, self.password)
+        t.publish = job_config.get("publish", self.publish)
+        return t
+
+    def __str__(self) -> str:
+        data = {
+            "name": self.name,
+            "publish": self.publish,
+            "credentials": {
+                "username": self.username,
+                "password": self.password
+            }
+        }
+        return f"{data}"
+
+
+class Builder():
+    def __init__(self, config) -> None:
+        self.dir = path.abspath(config.get("dir", getcwd()))
+        self.dockerfile = config.get("dockerfile", None)
+        self.name = config.get("name", None)
+        self.args = gen_dict(config.get("args", []))
 
     def copy(self, job_config):
-        pass
+        b = Builder({})
+        b.dir = path.abspath(path.join(self.dir, job_config.get("dir", ".")))
+        b.dockerfile = job_config.get("dockerfile", self.dockerfile)
+        b.name = job_config.get("name", self.name)
+        b.args = self.args.copy().update(gen_dict(job_config.get("args", [])))
+        return b
+
+    def __str__(self) -> str:
+        data = {
+            "type": "builder",
+            "dir": self.dir,
+            "dockerfile": self.dockerfile,
+            "name": self.name,
+            "args": self.args
+        }
+        return json.dumps(data)
+
+    def prepare(self, client):
+        print(f"[DockerRunner] Building image {self.name}")
+        if self.dockerfile is None:
+            self.dockerfile = "Dockerfile"
+        try:
+            image, log = client.images.build(path=self.dir,
+                                             dockerfile=self.dockerfile,
+                                             tag=self.name,
+                                             buildargs=self.args,
+                                             labels={"builder": "alice-ci"})
+            for i in log:
+                logging.debug(i)
+            return image
+        except docker.errors.BuildError as e:
+            raise RunnerError(f"[DockerRunner] Build failed: {e}")
+        except docker.errors.APIError as e:
+            raise RunnerError(f"[DockerRunner] Error: {e}")
+
+
+class Puller():
+    def __init__(self, config={}) -> None:
+        self.name = config.get("name", None)
+        self.username = get_user(config, None)
+        self.password = get_pass(config, None)
+
+    def copy(self, job_config={}):
+        p = Puller()
+        p.name = job_config.get("name", self.name)
+        p.username = get_user(job_config, self.username)
+        p.password = get_pass(job_config, self.password)
+
+    def __str__(self) -> str:
+        data = {
+            "name": self.name,
+            "credentials": {
+                "username": self.username,
+                "password": self.password
+            }
+        }
+        return f"{data}"
+
+    def prepare(self, client):
+        print(f"[DockerRunner] Pulling image {self.name}")
+        return client.images.pull(self.name)
+
+
+class DockerConfig:
+    def __init__(self, config={}) -> None:
+        self.username = get_user(config, None)
+        self.password = get_pass(config, None)
+        self.image_provider, self.provider_type = get_provider(config, None, ImageSource.NONE)
+        self.tagger = Tagger(config.get("tag", {}))
+        self.commands = config.get("commands", [])
+
+    def copy(self, job_config={}):
+        d = DockerConfig()
+        d.username = get_user(job_config, self.username)
+        d.password = get_pass(job_config, self.password)
+        d.image_provider, d.provider_type = get_provider(job_config, self.image_provider, self.provider_type)
+        d.tagger = self.tagger.copy(job_config.get("tag", {}))
+        d.commands = self.commands.copy() + job_config.get("commands", [])
+        return d
+
+    def __str__(self) -> str:
+        data = {
+            "credentials": {
+                "username": {self.username},
+                "password": {self.password}
+            },
+            "image": self.image_provider.__str__(),
+            "commands": self.commands,
+            "tag": self.tagger.__str__()
+        }
+        return f"{data}"
 
 
-# Supported tasks:
-#  - Build image
-#  - Push image to repo
-#  - Run arbitrary code in image
 class DockerRunner():
-    def __init__(self) -> None:
-        pass
+    def __init__(self, config) -> None:
+        logging.info("[DockerRunner] Initializing")
+        self.config = DockerConfig(config)
+        self.client = docker.from_env()
 
     def run(self, job_spec):
-        pass
+        job_config = self.config.copy(job_spec)
+        if job_config.image_provider is None:
+            raise RunnerError("[DockerRunner] No image provider configured!")
+        image = job_config.image_provider.prepare(self.client)
+        logging.info(f"[DockerRunner] Image: {image.tags} ({image.id})")
+
+        if len(job_config.commands) > 0:
+            container = self.client.containers.run(image=image.id,
+                                                   entrypoint=["sleep", "infinity"],
+                                                   detach=True,
+                                                   auto_remove=True)
+            try:
+                for i in job_config.commands:
+                    command = ["/bin/sh", "-c", i]
+                    logging.debug(f"[DockerRunner] Command array: {command}")
+                    code, output = container.exec_run(cmd=command)
+                    for line in output.decode("UTF-8").splitlines():
+                        print(f"[{job_spec['name']}] {line}")
+                    if code != 0:
+                        raise NonZeroRetcode(f"Command {i} returned code {code}")
+            finally:
+                if container is not None:
+                    container.stop()

alice-ci/src/alice/runners/pyutils.py

@@ -6,7 +6,7 @@ from pkg_resources import parse_version
 import re
 
 from alice.exceptions import RunnerError, ConfigException
-from alice.cli import ConfigHolder
+from alice.config import ConfigHolder
 
 
 class PackageManager:
@@ -124,3 +124,21 @@ def grab_from(target):
         ConfigHolder.getInstance().get(target["from_cfg"])
     else:
         raise ConfigException(f"Unsupported grabber: {target.keys()}")
+
+
+def gen_dict(list_of_dicts):
+    """
+    Generates a dictionary from a list of dictionaries composed of
+    'name' and 'value' keys.
+
+    [{'name': 'a', 'value': 'b'}] => {'a': 'b'}
+    """
+    return_dict = {}
+
+    for _dict in list_of_dicts:
+        try:
+            return_dict[_dict["name"]] = _dict["value"]
+        except KeyError:
+            raise ConfigException(f"Invalid dict item: {_dict}")
+
+    return return_dict

ci-examples/full.yaml

@@ -17,6 +17,9 @@ runners:
       - build
   docker:
     credentials:
+      username: D
+      password: D
+
 jobs:
   - name: env
     type: python
@@ -48,18 +51,35 @@ jobs:
         from_env: PYPIPASS
     packages:
      - alice-ci
-  - name: image
+  - name: "image"
     type: docker
+    credentials:
+      username: A
+      #password: B
     image:
-      credentials:
-        user:
-          from_env: DOCKERUSER
-        pass:
-          from_env: DOCKERPASS
-      name: python:latest
-      build_dir: ci-examples/docker
+      build:
+        dir: ci-examples/images/hello
+        #dockerfile: ci-examples/images/hello/Dockerfile
+        dockerfile: Dockerfile
+        name: "sssss"
+        args:
+          - name: CIPASS
+            value: NONE
+      #pull:
+        #name: python:latest
+        #credentials:
+          #username: PASS
+          #password: WORD
     commands:
-      - python3 --version
+      - which python3
+      - /usr/bin/python3 --version
+      - date
+    tag:
+      publish: true
+      name: published name with repo and everything
+      credentials:
+        username: B
+        password: B
     
 pipelines:
   default:

ci-examples/images/hello/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:latest
+
+RUN apt update && apt install -y python3
+
+ADD hello.py /opt/hello.py
+
+#ENTRYPOINT [ "/bin/sh", "-c" ]
+
+#CMD ["/usr/local/python/bin/python3", "/opt/hello.py"]

ci-examples/images/hello/hello.py

@@ -0,0 +1,2 @@
+if __name__ == "__main__":
+    print("Hi Mom!")

docs/runners/docker.md

@@ -4,18 +4,19 @@
 name: ""
 type: docker
 credentials: - global ...ish
-  user
-  pass
+  username
+  password
 image: - to use, pull, run
-  credentials:
   build:
     dir:
     dockerfile:
-    name:
+    name: - defaults to step name
     args:
       - name:
       - value:
-  name: - pulls, current working image - mutually exclusive with build
+  pull:  - pulls, current working image - mutually exclusive with build
+    name:
+    credentials: - optional
 command: - overwrite, not append
   - ...
 tag: