alice-ci/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = alice-ci
-version = 0.0.8
+version = 0.0.6
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md

alice-ci/src/alice/__init__.py

@@ -1,5 +1,5 @@
 # flake8: noqa F401
-from alice.utils import ConfigParser
+from alice.configparser import ConfigParser
 from alice.exceptions import NonZeroRetcode
 from alice.runnerfactory import Factory
 from alice.runners.pythonrunner import PythonRunner

alice-ci/src/alice/cli.py

@@ -1,7 +1,7 @@
 import os
 import argparse
 
-from alice.utils import ConfigParser
+from alice.configparser import ConfigParser
 from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
 
 

alice-ci/src/alice/runnerfactory.py

@@ -1,6 +1,7 @@
 from os.path import join, abspath
 
 from alice.runners.pythonrunner import PythonRunner
+from alice.runners.pypirunner import PyPiRunner
 from alice.exceptions import ConfigException
 
 
@@ -19,7 +20,8 @@ class Factory():
         # https://git.gyulai.cloud/gyulaid/alice/issues/4
         # module = __import__("module_file")
         # my_class = getattr(module, "class_name")
-        self.runnertypes = {"python": PythonRunner}
+        self.runnertypes = {"python": PythonRunner,
+                            "pypi": PyPiRunner}
 
         if (self.verbose):
             print(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}")
@@ -48,7 +50,8 @@ class Factory():
                 params = {
                     "verbose": self.verbose
                 }
-                config = self.runner_configs[runnertype]
+                # If there is a runner specific config, use that, else global
+                config = self.runner_configs.get(runnertype, self.globals.copy())
                 self.runners[runnertype] = self.runnertypes[runnertype](params, config)
             else:
                 raise ConfigException(f"Invalid runner type: {runnertype}")

alice-ci/src/alice/runners/__init__.py

@@ -1,2 +1,3 @@
 # flake8: noqa F401
 from alice.runners.pythonrunner import PythonRunner
+from alice.runners.pypirunner import PyPiRunner

alice-ci/src/alice/runners/pypirunner.py

@@ -0,0 +1,177 @@
+import json
+import os
+import re
+import subprocess
+import sys
+from urllib import request, error
+from pkg_resources import parse_version
+from os import environ, path
+from alice.runners.pyutils import PackageManager, glob
+from alice.exceptions import ConfigException, RunnerError
+
+
+def grab_from(target):
+    if "from_env" in target:
+        return environ[target["from_env"]]
+    else:
+        raise ConfigException(f"Unsupported grabber: {target.keys()}")
+
+
+def get_uri(config, default):
+    url = config.get("repo", {}).get("uri", default)
+    if url is not None:
+        if not re.match('(?:http|ftp|https)://', url):
+            url = f"https://{url}"
+    return url
+
+
+def get_user(config, default):
+    if "repo" in config:
+        if "username" in config["repo"]:
+            data = config["repo"]["username"]
+            if isinstance(data, str):
+                return data
+            else:
+                return grab_from(data)
+    return default
+
+
+def get_pass(config, default):
+    if "repo" in config:
+        if "password" in config["repo"]:
+            data = config["repo"]["password"]
+            if isinstance(data, str):
+                return data
+            else:
+                return grab_from(data)
+    return default
+
+
+# Parses and stores the config from yaml
+class PypiConfig:
+    def __init__(self, config={}) -> None:
+        self.workdir = path.abspath(config.get("workdir", "."))
+        self.repo_uri = get_uri(config, None)
+        self.repo_user = get_user(config, None)
+        self.repo_pass = get_pass(config, None)
+        self.packages = set(config.get("packages", []))
+        self.upload = config.get("upload", False)
+        self.fail_if_exists = config.get("fail_if_exists", False)
+
+    # returns a PyPiConfig with merged values
+    def copy(self, job_config={}):
+        p = PypiConfig()
+        p.workdir = path.abspath(path.join(self.workdir, job_config.get("workdir", ".")))
+        p.repo_uri = get_uri(job_config, self.repo_uri)
+        p.repo_user = get_user(job_config, self.repo_user)
+        p.repo_pass = get_pass(job_config, self.repo_pass)
+        job_pkg_set = set(job_config["packages"])
+        job_pkg_set.update(self.packages)
+        p.packages = job_pkg_set
+        p.upload = job_config.get("upload", self.upload)
+        p.fail_if_exists = job_config.get("fail_if_exists", self.fail_if_exists)
+        return p
+
+
+# TODO: consider "--skip-existing" flag for twine
+class PyPiRunner():
+    def __init__(self, params, config) -> None:
+        self.verbose = params["verbose"]
+        if self.verbose:
+            print("[PyPiRunner] Initializing")
+        self.workdir = config["workdir"]
+        self.config = PypiConfig(config)
+
+    def __versions(self, repo, pkg_name):
+        if repo is not None:
+            url = f'{repo}/{pkg_name}/json'
+        else:
+            url = f"https://pypi.python.org/pypi/{pkg_name}/json"
+        try:
+            releases = json.loads(request.urlopen(url).read())['releases']
+        except error.URLError as e:
+            raise RunnerError(f"{url}: {e}")
+
+        return sorted(releases, key=parse_version, reverse=True)
+
+    def build(self, config, package):
+        # TODO: Actual build - silent, unless failure!
+        pkg_path = path.join(config.workdir, package)
+        if not path.isdir(pkg_path):
+            raise ConfigException(f"Path does not exists: {pkg_path}")
+        command = [sys.executable, "-m", "build", package]
+        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            if p.returncode != 0:
+                print("STDOUT:")
+                sys.stdout.buffer.write(p.stdout.read())
+                print("STDERR:")
+                sys.stdout.buffer.write(p.stderr.read())
+                raise RunnerError(f"[PyPiRunner] Failed to build {package}")
+
+    def find_unuploaded(self, repo, file_list, pkg_name):
+        versions = self.__versions(repo, pkg_name)
+        unuploaded = []
+        for file in file_list:
+            # flake8: noqa W605
+            re_groups = re.findall("(\d*\.\d*\.\d*)", file)
+            if len(re_groups) < 1:
+                raise RunnerError(f"Unable to determine version of file {file}")
+            file_version = re_groups[0]
+            if file_version not in versions:
+                unuploaded.append(file)
+            else:
+                print(f"[PyPiRunner] File already uploaded: {os.path.basename(file)}")
+        return unuploaded
+
+
+    def upload(self, config, package):
+        command = [sys.executable, "-m", "twine", "upload"]
+        if self.verbose:
+            command.append("--verbose")
+        if config.repo_uri is not None:
+            command.append("--repository-url")
+            command.append(config.repo_uri)
+        if config.repo_user is not None:
+            command.append("-u")
+            command.append(config.repo_user)
+        if config.repo_pass is not None:
+            command.append("-p")
+            command.append(config.repo_pass)
+        
+        dist_path = os.path.abspath(os.path.join(config.workdir, package, "dist"))
+        files = glob(os.path.join(dist_path, "*"), config.workdir)
+        for file in files:
+            print(f"[PyPiRunner] Found: {file}")
+
+        to_upload = self.find_unuploaded(config.repo_uri, files, package)
+        if len(to_upload) == 0:
+            return
+        command += to_upload
+        print(command)
+        print(" ".join(command))
+        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            if p.returncode != 0:
+                print("STDOUT:")
+                sys.stdout.buffer.write(p.stdout.read())
+                print("STDERR:")
+                sys.stdout.buffer.write(p.stderr.read())
+                raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})")
+
+    def run(self, job_spec):
+        job_config = self.config.copy(job_spec)
+
+        PackageManager.getInstance().ensure("build")
+        for package in job_config.packages:
+            print(f"[PyPiRunner] Building {package}")
+            #self.build(job_config, package)
+            print(f"[PyPiRunner] Package {package} built")
+
+        if job_config.upload:
+            PackageManager.getInstance().ensure("twine")
+            for package in job_config.packages:
+                print(f"[PyPiRunner] Uploading {package}")
+                self.upload(job_config, package)
+        else:
+            print(f"[PyPiRunner] Upload disabled, skiping")

alice-ci/src/alice/runners/pythonrunner.py

@@ -4,9 +4,11 @@ import sys
 import shlex
 
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
+from alice.runners.pyutils import PackageManager, glob_command
 
 
-class PythonRunner():
+# TODO: Handle config like PyPiConfig
+class PythonRunner:
     def __init__(self, params, config) -> None:
         self.verbose = params["verbose"]
         if self.verbose:
@@ -36,40 +38,13 @@ class PythonRunner():
         else:
             if self.verbose:
                 print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}")
-
+        dependencies = self.config.get("dependencies", [])
-        if "dependencies" in self.config:
+        if len(dependencies) > 0:
-            command = [self.vpython, "-m", "pip", "install"] + self.config["dependencies"] + ["--upgrade"]
+            if self.verbose:
-            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                print(f"[PythonRunner] Ensuring dependencies:  {', '.join(dependencies)}")
-                p.wait()
+            PackageManager.getInstance().ensure_more(dependencies, executable=self.vpython)
-                if p.returncode != 0:
+            if self.verbose:
-                    sys.stdout.buffer.write(p.stderr.read())
+                print("[PythonRunner] Installation done")
-                    raise(RunnerError(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
-
-    def __ghetto_glob(self, command, workdir):
-        if self.verbose:
-            print(f"[PythonRunner][Globbing] Starting command: {' '.join(command)}")
-        new_command = []
-        for item in command:
-            if "*" in item:
-                if self.verbose:
-                    print(f"[PythonRunner][Globbing] Found item: [{item}]")
-                dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item)))
-                base_name = os.path.basename(item)
-                if os.path.isdir(dir):
-                    item_parts = base_name.split("*")
-                    for file in os.listdir(dir):
-                        # TODO: Fix ordering! A*B = B*A = AB*
-                        if item_parts[0] in file and item_parts[1] in file:
-                            new_item = os.path.join(dir, file)
-                            if self.verbose:
-                                print(f"[PythonRunner][Globbing] Substitute: {new_item}")
-                            new_command.append(new_item)
-                else:
-                    if self.verbose:
-                        print(f"[PythonRunner][Globbing] Dir not exists: {dir}")
-            else:
-                new_command.append(item)
-        return new_command
 
     # Executes the given job in the one and only venv
     # parameter shall be the raw jobscpec
@@ -89,7 +64,7 @@ class PythonRunner():
                     print(f"[PythonRunner] Raw command: {command}")
                 # TODO: only split if command is not an array
                 if "*" in command:
-                    run_command = self.__ghetto_glob(shlex.split(command), pwd)
+                    run_command = glob_command(shlex.split(command), pwd, self.verbose)
                 else:
                     run_command = shlex.split(command)
                 if self.verbose:

alice-ci/src/alice/runners/pyutils.py

@@ -0,0 +1,115 @@
+import os
+import subprocess
+import sys
+from pkg_resources import parse_version
+import re
+
+from alice.exceptions import RunnerError, ConfigException
+
+
+class PackageManager:
+    __instance = None
+
+    @staticmethod
+    def getInstance():
+        """ Static access method. """
+        if PackageManager.__instance is None:
+            PackageManager()
+        return PackageManager.__instance
+
+    def __init__(self):
+        """ Virtually private constructor. """
+        if PackageManager.__instance is not None:
+            raise Exception("This class is a singleton!")
+        else:
+            PackageManager.__instance = self
+        self.package_list = self.__get_packages()
+
+    def __get_packages(self):
+        packages = {}
+        with subprocess.Popen([sys.executable, "-m", "pip", "freeze"],
+                              stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines())))
+        for name, version in installed:
+            packages[name] = parse_version(version)
+        return packages
+
+    def ensure_more(self, package_list, executable=sys.executable):
+        to_install = list(filter(lambda x: not self.__has_package(x), package_list))
+        if len(to_install) > 0:
+            command = [executable, "-m", "pip", "install"] + to_install
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    # Assumption: there are more hits in the long run, than misses
+    def ensure(self, package_string, executable=sys.executable):
+        if not self.__has_package(package_string):
+            command = [executable, "-m", "pip", "install", package_string]
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    def __has_package(self, package_string):
+        package_data = re.split("==|>|>=|<|<=", package_string)
+        # check in cache
+        if package_data[0] in self.package_list:
+            # check if version is needed
+            if len(package_data) == 2:
+                required_version = parse_version(package_data[1])
+                installed_version = self.package_list[package_data[0]]
+                comparator = package_string.replace(package_data[0], "").replace(package_data[1], "")
+                if comparator == "==":
+                    return required_version == installed_version
+                elif comparator == ">":
+                    return installed_version > required_version
+                elif comparator == ">=":
+                    return installed_version >= required_version
+                elif comparator == "<":
+                    return installed_version < required_version
+                elif comparator == "<=":
+                    return installed_version <= required_version
+                else:
+                    raise ConfigException(f"Illegal comparator found: {comparator}")
+            else:
+                return True
+        return False
+
+
+def glob(item, workdir, verbose=False):
+    new_command = []
+    if "*" in item:
+        if verbose:
+            print(f"[Globbing] Found item: [{item}]")
+        dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item)))
+        base_name = os.path.basename(item)
+        if os.path.isdir(dir):
+            item_parts = base_name.split("*")
+            for file in os.listdir(dir):
+                # TODO: Fix ordering! A*B = B*A = AB*
+                if item_parts[0] in file and item_parts[1] in file:
+                    new_item = os.path.join(dir, file)
+                    if verbose:
+                        print(f"[Globbing] Substitute: {new_item}")
+                    new_command.append(new_item)
+        else:
+            raise ConfigException(f"[Globbing] Dir not exists: {dir}")
+        return new_command
+    else:
+        return [item]
+
+
+def glob_command(command, workdir, verbose=False):
+    if verbose:
+        print(f"[Globbing] Starting command: {' '.join(command)}")
+    new_command = []
+    for item in command:
+        new_command += glob(item, workdir, verbose)
+    return new_command

ci-examples/full.yaml

@@ -7,7 +7,7 @@ runners:
         value: B
       - name: C
         value: C
-    workdir: packages
+    workdir: .
   python:
     env:
       - name: A
@@ -29,6 +29,20 @@ jobs:
     commands:
       - "-c \"from os import environ; assert environ['A'] == 'D'; assert environ['B'] == 'E'; assert environ['C'] == 'C'; print('Assertions passed')\""
   - name: lint
+    type: python
     workdir: alice-ci
     commands:
-      - "-m flake8 --ignore E501"
+      - "-m flake8 --ignore E501"
+  - name: pkg
+    type: pypi
+    workdir: .
+    upload: true
+    fail_if_exists: false
+    repo:
+      uri: example.com
+      username:
+        from_env: PYPIUSER
+      password:
+        from_env: PYPIPASS
+    packages:
+     - alice-ci