repo image build + bump version

Reviewed-on: #32

.drone.yml

@@ -6,7 +6,7 @@ steps:
 - name: static-test
   image: alpine/flake8
   commands:
-  - python3 -m flake8 --ignore E501,W503
+  - python3 -m flake8 --ignore E501,W503 alice-ci/src
 
 - name: build
   image: python
@@ -14,6 +14,15 @@ steps:
   - python3 -m pip install build
   - python3 -m build alice-ci
 
+- name: test
+  image: python
+  environment:
+    PYPIUSER: USER
+    PYPIPASS: PASS
+  commands:
+  - python3 -m pip install alice-ci/dist/alice_ci*.whl
+  - alice -i ci-examples/full.yaml -vv
+
 - name: publish
   image: python
   environment:

.gitignore

@@ -138,3 +138,5 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
+# persistency dir
+.alice

alice-ci/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = alice-ci
-version = 0.0.10
+version = 0.0.14
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md

alice-ci/src/alice/__main__.py

@@ -1,4 +1,4 @@
-import alice
+from .cli import main
 
 if __name__ == '__main__':
-    alice.cli.main()
+    main()

alice-ci/src/alice/cli.py

@@ -34,12 +34,13 @@ def parse_jobs(args):
         exit(1)
     except RunnerError as e:
         print(f"RunnerError-> {e}")
+        exit(1)
 
 
 def main():
     parser = argparse.ArgumentParser(prog="alice")
     parser.add_argument("steps", nargs='*', default=["default"])
-    parser.add_argument("-i", "--input", default="alice-ci.yaml")
+    parser.add_argument("-i", "--input", default=".alice-ci.yaml")
     parser.add_argument("-e", "--env", nargs='*', default=[])
     parser.add_argument("-a", "--addrunner", nargs='*', default=[])
     parser.add_argument('--verbose', '-v', action='count', default=0)
@@ -51,6 +52,9 @@ def main():
     if not os.path.isfile(args.input):
         print(f"No such file: {args.input}")
         exit(1)
+    persistency_path = os.path.join(os.getcwd(), ".alice")
+    if not os.path.isdir(persistency_path):
+        os.mkdir(persistency_path)
     parse_jobs(args)
 
 

alice-ci/src/alice/config.py

@@ -6,7 +6,7 @@ from .exceptions import ConfigException
 
 class ConfigHolder:
     __instance = None
-    file_name = ".alice"
+    file_name = os.path.join(os.getcwd(), ".alice", "vars")
 
     @staticmethod
     def getInstance():
@@ -25,7 +25,8 @@ class ConfigHolder:
             self.vars = {}
             if os.path.isfile(config):
                 with open(config) as f:
-                    for line in f:
+                    for _line in f:
+                        line = _line.strip()
                         items = line.split("=")
                         if len(items) > 1:
                             self.vars[items[0]] = line.replace(f"{items[0]}=", "")
@@ -33,6 +34,18 @@ class ConfigHolder:
 
     def get(self, key):
         try:
-            self.vars[key]
+            return self.vars[key]
         except KeyError:
             raise ConfigException(f"{key} not defined in .conf!")
+
+    def set(self, key, value):
+        self.vars[key] = value
+        self.commit()
+
+    def soft_set(self, key, value):
+        self.vars[key] = value
+
+    def commit(self):
+        with open(self.file_name, 'w') as f:
+            for k, v in self.vars.items():
+                f.write(f"{k}={v if v is not None else ''}\n")

alice-ci/src/alice/configparser.py

@@ -81,8 +81,8 @@ class ConfigParser:
     def execute_pipeline(self, pipeline_name):
         if pipeline_name in self.pipelines:
             print(f"[Alice][Pipeline] {pipeline_name}: Start")
-            for job in self.pipelines[pipeline_name]:
-                self.execute_job(job)
+            for task in self.pipelines[pipeline_name]:
+                self.execute(task)
             print(f"[Alice][Pipeline] {pipeline_name}: Success")
 
     def execute_job(self, job_name):

alice-ci/src/alice/runnerfactory.py

@@ -4,6 +4,7 @@ from os.path import join, abspath
 from .runners.pythonrunner import PythonRunner
 from .runners.pypirunner import PyPiRunner
 from .runners.dockerrunner import DockerRunner
+from .runners.pypirepo import PypiRepoRunner
 from .exceptions import ConfigException
 
 
@@ -23,7 +24,8 @@ class Factory():
         # my_class = getattr(module, "class_name")
         self.runnertypes = {"python": PythonRunner,
                             "pypi": PyPiRunner,
-                            "docker": DockerRunner}
+                            "docker": DockerRunner,
+                            "pypirepo": PypiRepoRunner}
 
         logging.info(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}")
 

alice-ci/src/alice/runners/pypirepo.py

@@ -0,0 +1,168 @@
+import logging
+import subprocess
+import docker
+from os.path import join, isdir
+from os import getcwd, mkdir
+import os
+import requests
+import platform
+import time
+
+from ..exceptions import RunnerError
+from ..config import ConfigHolder
+
+
+pipconf = """[global]
+index-url = URL
+trusted-host = BASE
+               pypi.org
+extra-index-url= http://pypi.org/simple"""
+
+
+class RepoConfig:
+    def __init__(self, config={}) -> None:
+        self.port = config.get("port", 8888)
+        self.enabled = config.get("enabled", True)
+        self.container_name = config.get("container_name", "alice-pypiserver")
+
+    def copy(self, job_config):
+        r = RepoConfig()
+        r.container_name = job_config.get("container_name", self.container_name)
+        r.enabled = job_config.get("enabled", self.enabled)
+        r.port = job_config.get("port", self.port)
+        return r
+
+
+class PypiRepoRunner:
+    def __init__(self, config) -> None:
+        logging.info("[PyPiRepo] Initializing")
+        self.config = RepoConfig(config)
+        self.client = docker.from_env()
+        self.user = "alice"
+        self.passwd = "alice"
+        self.htpasswd = 'alice:{SHA}UisnajVr3zkBPfq+os1D4UHsyeg='
+
+    def get_image(self):
+        # TODO: remove when resolved:
+        # Official Docker image support for ARM?
+        # https://github.com/pypiserver/pypiserver/issues/364
+        pypiserver = "https://github.com/pypiserver/pypiserver.git"
+        if platform.machine() == "aarch64":
+            tag = "alice.localhost/pypiserver:arm"
+            try:
+                self.client.images.get(tag)
+                return tag
+            except docker.errors.ImageNotFound:
+                print("[PyPiRepo] Building PyPiServer ARM image, this could take a while")
+                workdir = join(getcwd(), ".alice", "pypirepo", "source")
+                if not os.path.isdir(workdir):
+                    os.mkdir(workdir)
+                git_command = ["git", "clone", pypiserver, "--branch=v1.3.2"]
+                output = []
+                with subprocess.Popen(git_command, cwd=workdir, stdout=subprocess.PIPE) as p:
+                    for line in p.stdout:
+                        output.append(line.decode('utf8').strip())
+                    p.wait()
+                    if p.returncode != 0:
+                        print("\n".join(output))
+                        raise(RunnerError("[PyPiRepo] Could not fetch pypiserver source"))
+                source_path = os.path.join(workdir, "pypiserver")
+                self.client.images.build(path=source_path, tag=tag)
+                return tag
+        else:
+            return "pypiserver/pypiserver:latest"
+
+    def run(self, job_spec):
+        job_config = self.config.copy(job_spec)
+        docker_host_ip = None
+        for network in self.client.networks.list():
+            if network.name == "bridge":
+                try:
+                    docker_host_ip = network.attrs["IPAM"]["Config"][0]["Gateway"]
+                except KeyError:
+                    docker_host_ip = network.attrs["IPAM"]["Config"][0]["Subnet"].replace(".0/16", ".1")
+        if docker_host_ip is None:
+            raise RunnerError("Unable to determine Docker host IP")
+
+        if job_config.enabled:
+            try:
+                c = self.client.containers.get(job_config.container_name)
+                print(f"[PyPiRepo] {job_config.container_name} already running")
+            except docker.errors.NotFound:
+                persistency_dir = join(getcwd(), ".alice", "pypirepo")
+                if not isdir(persistency_dir):
+                    mkdir(persistency_dir)
+
+                package_dir = join(persistency_dir, "packages")
+                if not isdir(package_dir):
+                    mkdir(package_dir)
+
+                htpasswd_file = join(persistency_dir, ".htpasswd")
+                with open(htpasswd_file, 'w') as f:
+                    f.write(self.htpasswd)
+
+                c = self.client.containers.run(
+                    name=job_config.container_name,
+                    image=self.get_image(),
+                    detach=True,
+                    labels={"app": "alice"},
+                    command=["--overwrite", "-P", ".htpasswd", "packages"],
+                    ports={"8080/tcp": job_config.port},
+                    volumes={
+                        package_dir: {
+                            "bind": "/data/packages",
+                            "mode": "rw"
+                        },
+                        htpasswd_file: {
+                            "bind": "/data/.htpasswd",
+                            "mode": "ro"
+                        }
+                    },
+                    restart_policy={
+                        "Name": "unless-stopped"
+                    }
+                )
+                print(f"[PyPiRepo] Started {job_config.container_name}")
+
+            c.reload()
+            logging.info(f"[PyPiRepo] {job_config.container_name} : {c.status}")
+            if c.status != "running":
+                raise RunnerError(f"[PyPiRepo] Repo container unstable: {c.status}")
+
+            uri = f"http://localhost:{job_config.port}"
+            unreachable = True
+            attempts = 0
+            while unreachable and attempts < 5:
+                attempts += 1                
+                try:
+                    requests.get(uri)
+                    unreachable = False
+                except Exception as e:
+                    logging.info(f"[PyPiRepo] {attempts} - Repo at {uri} is unavailable: {e}")
+                    time.sleep(2)
+            if unreachable:
+                raise RunnerError(f"[PyPiRepo] Repo unreachable")
+
+
+            cfgh = ConfigHolder.getInstance()
+            cfgh.soft_set("PYPI_USER", self.user)
+            cfgh.soft_set("PYPI_PASS", self.passwd)
+            cfgh.soft_set("PYPI_REPO", uri)
+            cfgh.soft_set("DOCKER_PYPI_USER", self.user)
+            cfgh.soft_set("DOCKER_PYPI_PASS", self.passwd)
+            cfgh.soft_set("DOCKER_PYPI_REPO", f"http://{docker_host_ip}:{job_config.port}")
+            cfgh.commit()
+
+            venv = join(os.getcwd(), "venv")
+            if os.path.isdir(venv):
+                netloc = f"localhost:{job_config.port}"
+                url = f"http://{self.user}:{self.passwd}@{netloc}"
+                conf = pipconf.replace("URL", url).replace("BASE", netloc)
+
+                if os.name == "nt":  # Windows
+                    filename = join(venv, "pip.ini")
+                else:  # Linux & Mac
+                    filename = join(venv, "pip.conf")
+                with open(filename, 'w') as f:
+                    f.write(conf)
+                print(f"[PyPiRepo] pip conf written to {filename}")

alice-ci/src/alice/runners/pypirunner.py

@@ -1,19 +1,62 @@
+from distutils.command.config import config
+from distutils.log import debug
 import json
 import logging
+from ntpath import join
 import os
 import re
 import subprocess
 import sys
-from urllib import request, error
 from pkg_resources import parse_version
+from requests import get
+from requests.auth import HTTPBasicAuth
 from os import environ, path
+from html.parser import HTMLParser
 from alice.runners.pyutils import PackageManager, glob, grab_from
 from alice.exceptions import ConfigException, RunnerError
+import hashlib
+from pathlib import Path
+
+
+def md5_update_from_file(filename, hash):
+    assert Path(filename).is_file()
+    with open(str(filename), "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash.update(chunk)
+    return hash
+
+
+def md5_file(filename):
+    return md5_update_from_file(filename, hashlib.md5()).hexdigest()
+
+
+def md5_update_from_dir(directory, hash, exclude_dirs, exclude_extensions, exclude_dirs_wildcard):
+    assert Path(directory).is_dir()
+    for _path in os.listdir(directory):
+        path = os.path.join(directory, _path)        
+        if os.path.isfile(path) :
+            hash.update(_path.encode())
+            logging.debug(f"[PyPiRunner][Hash] File: {path}")
+            hash = md5_update_from_file(path, hash)
+        elif os.path.isdir(path):
+            skip = False
+            for name in exclude_dirs:
+                if name in os.path.basename(_path):
+                    skip = True
+            if not skip:
+                hash = md5_update_from_dir(path, hash, exclude_dirs, exclude_extensions, exclude_dirs_wildcard)
+    return hash
+
+
+def md5_dir(directory, exclude_dirs=[], exclude_extensions=[], exclude_dirs_wildcard=[]):
+    return md5_update_from_dir(directory, hashlib.sha1(), exclude_dirs, exclude_extensions, exclude_dirs_wildcard).hexdigest()
 
 
 def get_uri(config, default):
     url = config.get("repo", {}).get("uri", default)
     if url is not None:
+        if not isinstance(url, str):
+            url = grab_from(url)
         if not re.match('(?:http|ftp|https)://', url):
             url = f"https://{url}"
     return url
@@ -41,6 +84,19 @@ def get_pass(config, default):
     return default
 
 
+class SimpleRepoParser(HTMLParser):
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.packages = []
+
+    def handle_data(self, data):
+        re_groups = re.findall("(\d*\.\d*\.\d*)", data)
+        if len(re_groups) == 1:
+            file_version = re_groups[0]
+            if file_version not in self.packages:
+                self.packages.append(file_version)
+
+
 # Parses and stores the config from yaml
 class PypiConfig:
     def __init__(self, config={}) -> None:
@@ -67,41 +123,106 @@ class PypiConfig:
         return p
 
 
+# TODO: Refactor to something sensible, more flexible
+class PackageMeta:
+    def __init__(self):
+        self.conf_dir = path.join(os.getcwd(), ".alice", "pypirunner")
+        self.metafile = path.join(self.conf_dir, "packagemeta.json")
+        if not path.isdir(self.conf_dir):
+            os.mkdir(self.conf_dir)
+        if path.isfile(self.metafile):
+            with open(self.metafile) as f:
+                self.metadata = json.load(f)
+        else:
+            self.metadata = {}
+            self.__save()
+
+    def __save(self):
+        with open(self.metafile, 'w') as f:
+            json.dump(self.metadata, f)
+
+    def get(self, package, key):
+        return self.metadata.get(package, {}).get(key, "")
+
+    def set(self, package, key, value):
+        if package not in self.metadata:
+            self.metadata[package] = {}
+        self.metadata[package][key] = value
+        self.__save()
+
+
 # TODO: consider "--skip-existing" flag for twine
 class PyPiRunner():
     def __init__(self, config) -> None:
         logging.info("[PyPiRunner] Initializing")
         self.workdir = config["workdir"]
         self.config = PypiConfig(config)
+        self.metadata = PackageMeta()
 
-    def __versions(self, repo, pkg_name):
-        if repo is not None:
-            url = f'{repo}/{pkg_name}/json'
+    def __versions(self, config, pkg_name):
+        repo = config.repo_uri
+        if repo is None:
+            repo = "https://pypi.python.org/pypi"
+
+        if config.repo_pass is not None and config.repo_user is not None:
+            logging.info(f"[PyPiRunner][Versions] Set auth headers from config")
+            logging.debug(f"[PyPiRunner][Versions] Auth: {config.repo_user}:{config.repo_pass}")
+            auth = HTTPBasicAuth(config.repo_user, config.repo_pass)
         else:
-            url = f"https://pypi.python.org/pypi/{pkg_name}/json"
-        try:
-            releases = json.loads(request.urlopen(url).read())['releases']
-        except error.URLError as e:
-            raise RunnerError(f"{url}: {e}")
+            logging.info(f"[PyPiRunner][Versions] No auth headers in config, skip")
+            logging.debug(f"[PyPiRunner][Versions] Auth: {config.repo_user}:{config.repo_pass}")
+            auth = None
 
-        return sorted(releases, key=parse_version, reverse=True)
+        try:
+            if repo.endswith("pypi"):
+                url = f'{repo}/{pkg_name}/json'
+                logging.info(f"[PyPiRunner][Versions] Trying JSON API at {url}")
+                response = get(url, auth=auth)
+                if response.status_code == 200:
+                    releases = json.loads(response.text)["releases"]
+                    return sorted(releases, key=parse_version, reverse=True)
+                else:
+                    logging.info(f"[PyPiRunner][Versions] JSON failed: [{response.status_code}]")
+                    logging.debug(response.text)
+                    repo = f"{repo}/simple"
+            url = f"{repo}/{pkg_name}"
+            logging.info(f"[PyPiRunner][Versions] Trying Simple API at {url}")
+            response = get(url, auth=auth)
+            if response.status_code == 200:
+                parser = SimpleRepoParser()
+                parser.feed(response.text)
+                return sorted(parser.packages, key=parse_version, reverse=True)
+            if response.status_code == 404:
+                return []
+            else:
+                logging.info(f"[PyPiRunner][Versions] Simple failed: [{response.status_code}]")
+                logging.debug(response.text)
+                raise Exception("Failed to fetch available versions")
+            
+        except Exception as e:
+            raise RunnerError(f"{url}: {e}")        
 
     def build(self, config, package):
+        print(f"[PyPiRunner] Building {package}")
         pkg_path = path.join(config.workdir, package)
         if not path.isdir(pkg_path):
             raise ConfigException(f"Path does not exists: {pkg_path}")
+        PackageManager.getInstance().ensure("build")
         command = [sys.executable, "-m", "build", package]
-        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
-            p.wait()
-            if p.returncode != 0:
-                print("STDOUT:")
-                sys.stdout.buffer.write(p.stdout.read())
-                print("STDERR:")
-                sys.stdout.buffer.write(p.stderr.read())
-                raise RunnerError(f"[PyPiRunner] Failed to build {package}")
-
-    def find_unuploaded(self, repo, file_list, pkg_name):
-        versions = self.__versions(repo, pkg_name)
+        if logging.root.isEnabledFor(logging.DEBUG):
+            with subprocess.Popen(command, cwd=config.workdir) as p:
+                p.wait()
+                if p.returncode != 0:
+                    raise RunnerError(f"[PyPiRunner] Failed to build {package}")
+        else:
+            with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    raise RunnerError(f"[PyPiRunner] Failed to build {package}")
+        print(f"[PyPiRunner] Package {package} built")
+
+    def find_unuploaded(self, config, file_list, pkg_name):
+        versions = self.__versions(config, pkg_name)
         unuploaded = []
         for file in file_list:
             # flake8: noqa W605
@@ -113,52 +234,112 @@ class PyPiRunner():
                 unuploaded.append(file)
             else:
                 print(f"[PyPiRunner] File already uploaded: {os.path.basename(file)}")
+        print(f"[PyPiRunner] Packages to publish: {', '.join(unuploaded) if len(unuploaded) > 1 else 'None'}")
         return unuploaded
 
+    def upload_command(self, config, package, _command, to_upload):
+        unregistered = False
+        command = _command + to_upload
+        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            if p.returncode != 0:
+                for line in map(lambda x: x.decode('utf8').strip(), p.stderr):
+                    if "405 Method Not Allowed" in line:
+                        unregistered = True
+                if not unregistered:
+                    print("STDOUT:")
+                    sys.stdout.buffer.write(p.stdout.read())
+                    print("STDERR:")
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})")
+        if unregistered:
+            print("[PyPiRunner] Registering package")
+            register_command = [sys.executable, "-m", "twine", "register", "--verbose", "--non-interactive"]
+            if config.repo_uri is not None:
+                register_command.append("--repository-url")
+                register_command.append(config.repo_uri)
+            if config.repo_user is not None and config.repo_pass is not None:
+                register_command.append("-u")
+                register_command.append(config.repo_user)
+                register_command.append("-p")
+                register_command.append(config.repo_pass)
+            register_command.append(to_upload[0])
+            with subprocess.Popen(register_command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    print("STDOUT:")
+                    sys.stdout.buffer.write(p.stdout.read())
+                    print("STDERR:")
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise RunnerError(f"[PyPiRunner] Failed to register {package} ({p.returncode})")
+            self.upload_command(config, package, _command, to_upload)
 
-    def upload(self, config, package):
-        command = [sys.executable, "-m", "twine", "upload", "--verbose"]
+    def upload(self, config, package, current_version):
+        print(f"[PyPiRunner] Uploading {package}")
+        PackageManager.getInstance().ensure("twine")
+        command = [sys.executable, "-m", "twine", "upload", "--verbose", "--non-interactive"]
         if config.repo_uri is not None:
             command.append("--repository-url")
             command.append(config.repo_uri)
-        if config.repo_user is not None:
+        if config.repo_user is not None and config.repo_pass is not None:
             command.append("-u")
             command.append(config.repo_user)
-        if config.repo_pass is not None:
             command.append("-p")
             command.append(config.repo_pass)
+        else:
+            raise RunnerError("[PyPiRunner] Can't upload without credentials!")
         
         dist_path = os.path.abspath(os.path.join(config.workdir, package, "dist"))
-        files = glob(os.path.join(dist_path, "*"), config.workdir)
-        for file in files:
-            print(f"[PyPiRunner] Found: {file}")
+        _files = glob(os.path.join(dist_path, "*"), config.workdir)
+        files = []
+        for file in _files:
+            if current_version in os.path.basename(file):
+                files.append(file)
+                print(f"[PyPiRunner] Found: {file}")
+            else:
+                logging.info(f"[PyPiRunner] Dropped: {file} doesn't match current version: {current_version}")
 
-        to_upload = self.find_unuploaded(config.repo_uri, files, package)
+        to_upload = self.find_unuploaded(config, files, package)
         if len(to_upload) == 0:
             return
-        command += to_upload
-        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
-            p.wait()
-            if p.returncode != 0:
-                print("STDOUT:")
-                sys.stdout.buffer.write(p.stdout.read())
-                print("STDERR:")
-                sys.stdout.buffer.write(p.stderr.read())
-                raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})")
+        #command += to_upload
+        self.upload_command(config, package, command, to_upload)
+        print(f"[PyPiRunner] Uploaded {package}")
+
+    def package_version(self, config, package):
+        cfg_path = path.join(config.workdir, package, "setup.cfg")
+        with open(cfg_path) as f:
+            for line in f:
+                if line.startswith("version"):
+                    re_groups = re.findall("(\d*\.\d*\.\d*)", line)
+                    if len(re_groups) < 1:
+                        raise RunnerError(f"Unable to determine version of package:  |{line}|")
+                    return re_groups[0]
 
     def run(self, job_spec):
         job_config = self.config.copy(job_spec)
-
-        PackageManager.getInstance().ensure("build")
+        
         for package in job_config.packages:
-            print(f"[PyPiRunner] Building {package}")
-            self.build(job_config, package)
-            print(f"[PyPiRunner] Package {package} built")
-
-        if job_config.upload:
-            PackageManager.getInstance().ensure("twine")
-            for package in job_config.packages:
-                print(f"[PyPiRunner] Uploading {package}")
-                self.upload(job_config, package)
-        else:
-            print(f"[PyPiRunner] Upload disabled, skiping")
+            pkg_dir = path.join(job_config.workdir, package)
+            pkg_hash = md5_dir(pkg_dir, exclude_dirs=["pycache", "pytest_cache", "build", "dist", "egg-info"])
+            logging.debug(f"[PyPiRunner] {package} hash: {pkg_hash}")
+            pkg_version = self.package_version(job_config, package)
+            logging.debug(f"[PyPiRunner] {package} local version: {pkg_version}")
+            repo_versions = self.__versions(job_config, package)
+            logging.debug(f"[PyPiRunner] {package} remote version: {repo_versions}")
+
+            if pkg_version not in repo_versions:
+                print(f"[PyPiRunner] {package} not found in repo")
+                self.build(job_config, package)
+                self.metadata.set(package, pkg_version, pkg_hash)
+            else:
+                if pkg_hash != self.metadata.get(package, pkg_version):
+                    self.build(job_config, package)
+                    self.metadata.set(package, pkg_version, pkg_hash)
+                else:
+                    print(f"[PyPiRunner] {package} Unchanged since last build")
+
+            if job_config.upload:
+                self.upload(job_config, package, pkg_version)
+            else:
+                print(f"[PyPiRunner] Upload disabled, skipping")

alice-ci/src/alice/runners/pythonrunner.py

@@ -5,7 +5,7 @@ import sys
 import shlex
 
 from ..exceptions import NonZeroRetcode, RunnerError, ConfigException
-from .pyutils import PackageManager, glob_command
+from .pyutils import PackageManager, glob_command, grab_from
 
 
 # TODO: Handle config like PyPiConfig
@@ -15,22 +15,27 @@ class PythonRunner:
         self.workdir = config["workdir"]
         self.virtual_dir = os.path.abspath(os.path.join(self.workdir, "venv"))
         self.config = config
-        PackageManager.getInstance().ensure("build")
+        PackageManager.getInstance().ensure("virtualenv")
         self.__init_venv()
 
+    # TODO: Detect if the prev venv is the same OS type
     def __init_venv(self):
         if os.name == "nt":  # Windows
             self.vpython = os.path.join(self.virtual_dir, "Scripts", "python.exe")
         else:  # Linux & Mac
-            self.vpython = os.path.join(self.virtual_dir, "bin", "python3")
+            self.vpython = os.path.join(self.virtual_dir, "bin", "python")
 
         if not os.path.exists(self.vpython):
+            logging.debug(f"[PythonRunner] Venv not found at {self.vpython}")
             logging.info("[PythonRunner] Initializing venv")
+            output = []
             with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                                  stdout=subprocess.PIPE) as p:
                 p.wait()
+                for line in p.stdout:
+                    output.append(line.decode('utf8').strip())
                 if p.returncode != 0:
-                    sys.stdout.buffer.write(p.stderr.read())
+                    print("\n".join(output))
                     raise RunnerError("[PythonRunner] Could not create virtualenv")
                 else:
                     logging.info(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}")
@@ -40,11 +45,20 @@ class PythonRunner:
         if len(dependencies) > 0:
             logging.info(f"[PythonRunner] Ensuring dependencies:  {', '.join(dependencies)}")
             command = [self.vpython, "-m", "pip", "install"] + dependencies
-            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
-                p.wait()
-                if p.returncode != 0:
-                    sys.stdout.buffer.write(p.stderr.read())
-                    raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})"))
+            if logging.root.isEnabledFor(logging.DEBUG):
+                with subprocess.Popen(command) as p:
+                    p.wait()
+                    if p.returncode != 0:
+                        raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})"))
+            else:
+                output = []
+                with subprocess.Popen(command, stdout=subprocess.PIPE) as p:
+                    for line in p.stdout:
+                        output.append(line.decode('utf8').strip())
+                    p.wait()
+                    if p.returncode != 0:
+                        print("\n".join(output))
+                        raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})"))
             logging.info("[PythonRunner] Installation done")
 
     # Executes the given job in the one and only venv
@@ -54,10 +68,18 @@ class PythonRunner:
             pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"]))
         else:
             pwd = self.workdir
-        run_env = self.config["env"].copy()
+        run_env = {}
+        for k, v in self.config["env"].items():
+            if isinstance(v, str):
+                run_env[k] = v
+            else:
+                run_env[k] = grab_from(v)
         if "env" in job_spec:
             for env_var in job_spec["env"]:
-                run_env[env_var["name"]] = env_var["value"]
+                if isinstance(env_var["value"], str):
+                    run_env[env_var["name"]] = env_var["value"]
+                else:
+                    run_env[env_var["name"]] = grab_from(env_var["value"])
         if "commands" in job_spec:
             commands = job_spec["commands"]
             for command in commands:

alice-ci/src/alice/runners/pyutils.py

@@ -35,6 +35,7 @@ class PackageManager:
             installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines())))
         for name, version in installed:
             packages[name] = parse_version(version)
+        logging.debug(f"[PackageManager] Picked up packages: {packages}")
         return packages
 
     def ensure_more(self, package_list, executable=sys.executable):
@@ -51,6 +52,7 @@ class PackageManager:
     # Assumption: there are more hits in the long run, than misses
     def ensure(self, package_string, executable=sys.executable):
         if not self.__has_package(package_string):
+            logging.info(f"[PackageManager] Installing {package_string}")
             command = [executable, "-m", "pip", "install", package_string]
             with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
                 p.wait()
@@ -58,6 +60,8 @@ class PackageManager:
                     sys.stdout.buffer.write(p.stderr.read())
                     raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})"))
             self.package_list = self.__get_packages()
+        else:
+            logging.info(f"[PackageManager] {package_string} already installed")
 
     def __has_package(self, package_string):
         package_data = re.split("==|>|>=|<|<=", package_string)
@@ -121,9 +125,12 @@ def grab_from(target):
         except KeyError:
             raise ConfigException(f"Env var unset: {target['from_env']}")
     elif "from_cfg" in target:
-        ConfigHolder.getInstance().get(target["from_cfg"])
+        value = ConfigHolder.getInstance().get(target["from_cfg"])
+        if len(value) == 0:
+            value = None
+        return value
     else:
-        raise ConfigException(f"Unsupported grabber: {target.keys()}")
+        raise ConfigException(f"Unsupported grabber: {target}")
 
 
 def gen_dict(list_of_dicts):

ci-examples/full.yaml

@@ -84,6 +84,11 @@ jobs:
       credentials:
         username: B
         password: B
+  - name: pypi_init
+    type: pypirepo
+    enabled: true
+    port: 8888
+    container_name: pypiserver
     
 pipelines:
   default: