diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py index f92d6b3..e03f06a 100644 --- a/alice-ci/src/alice/cli.py +++ b/alice-ci/src/alice/cli.py @@ -1,7 +1,7 @@ import os import argparse -from alice.utils import ConfigParser +from alice.configparser import ConfigParser from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError diff --git a/alice-ci/src/alice/utils.py b/alice-ci/src/alice/configparser.py similarity index 97% rename from alice-ci/src/alice/utils.py rename to alice-ci/src/alice/configparser.py index 4ecfb26..4af4848 100644 --- a/alice-ci/src/alice/utils.py +++ b/alice-ci/src/alice/configparser.py @@ -1,3 +1,4 @@ +import inspect from os import getcwd, path, environ import subprocess import yaml diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index 3a7b4dd..bd3dc1b 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -1,12 +1,14 @@ +from http.server import executable import subprocess import os import sys import shlex from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException +from alice.runners.pyutils import PackageManager - -class PythonRunner(): +# TODO: Handle config like PyPiConfig +class PythonRunner: def __init__(self, params, config) -> None: self.verbose = params["verbose"] if self.verbose: @@ -36,14 +38,13 @@ class PythonRunner(): else: if self.verbose: print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}") - - if "dependencies" in self.config: - command = [self.vpython, "-m", "pip", "install"] + self.config["dependencies"] + ["--upgrade"] - with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - p.wait() - if p.returncode != 0: - sys.stdout.buffer.write(p.stderr.read()) - raise(RunnerError(f"[PythonRunner] Could not install dependencies ({p.returncode})")) + dependencies = self.config.get("dependencies", []) + if len(dependencies) >0: + if self.verbose: + print(f"[PythonRunner] Ensuring dependencies: {', '.join(dependencies)}") + PackageManager.getInstance().ensure_more(dependencies, executable=self.vpython) + if self.verbose: + print(f"[PythonRunner] Installation done") def __ghetto_glob(self, command, workdir): if self.verbose: diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py new file mode 100644 index 0000000..6631f53 --- /dev/null +++ b/alice-ci/src/alice/runners/pyutils.py @@ -0,0 +1,79 @@ +import subprocess +import sys +from pkg_resources import parse_version +import re + +from alice.exceptions import RunnerError, ConfigException + +class PackageManager: + __instance = None + @staticmethod + def getInstance(): + """ Static access method. """ + if PackageManager.__instance == None: + PackageManager() + return PackageManager.__instance + def __init__(self): + """ Virtually private constructor. """ + if PackageManager.__instance != None: + raise Exception("This class is a singleton!") + else: + PackageManager.__instance = self + self.package_list = self.__get_packages() + + def __get_packages(self): + packages = {} + with subprocess.Popen([sys.executable, "-m", "pip", "freeze"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines()))) + for name, version in installed: + packages[name] = parse_version(version) + return packages + + def ensure_more(self, package_list, executable=sys.executable): + to_install = list(filter(lambda x: not self.__has_package(x), package_list)) + if len(to_install) > 0: + command = [executable, "-m", "pip", "install"] + to_install + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + sys.stdout.buffer.write(p.stderr.read()) + raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})")) + self.package_list = self.__get_packages() + + # Assumption: there are more hits in the long run, than misses + def ensure(self, package_string, executable=sys.executable): + if not self.__has_package(package_string): + command = [executable, "-m", "pip", "install", package_string] + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + sys.stdout.buffer.write(p.stderr.read()) + raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})")) + self.package_list = self.__get_packages() + + def __has_package(self, package_string): + package_data = re.split("==|>|>=|<|<=", package_string) + # check in cache + if package_data[0] in self.package_list: + # check if version is needed + if len(package_data) == 2: + required_version = parse_version(package_data[1]) + installed_version = self.package_list[package_data[0]] + comparator = package_string.replace(package_data[0], "").replace(package_data[1], "") + if comparator == "==": + return required_version == installed_version + elif comparator == ">": + return installed_version > required_version + elif comparator == ">=": + return installed_version >= required_version + elif comparator == "<": + return installed_version < required_version + elif comparator == "<=": + return installed_version <= required_version + else: + raise ConfigException(f"Illegal comparator found: {comparator}") + else: + return True + return False \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..ef1a001 --- /dev/null +++ b/test.py @@ -0,0 +1,85 @@ +import subprocess +import sys +from pkg_resources import parse_version +import re + +class PackageManager: + __instance = None + @staticmethod + def getInstance(): + """ Static access method. """ + if PackageManager.__instance == None: + PackageManager() + return PackageManager.__instance + def __init__(self): + """ Virtually private constructor. """ + if PackageManager.__instance != None: + raise Exception("This class is a singleton!") + else: + PackageManager.__instance = self + self.package_list = self.__get_packages() + + def __get_packages(self): + packages = {} + with subprocess.Popen([sys.executable, "-m", "pip", "freeze"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines()))) + for name, version in installed: + packages[name] = parse_version(version) + return packages + + def ensure_more(self, package_list, executable=sys.executable): + to_install = list(filter(lambda x: not self.__has_package(x), package_list)) + if len(to_install) > 0: + command = [executable, "-m", "pip", "install"] + to_install + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + sys.stdout.buffer.write(p.stderr.read()) + raise(Exception(f"[PythonRunner] Could not install dependencies ({p.returncode})")) + self.package_list = self.__get_packages() + + # Assumption: there are more hits in the long run, than misses + def ensure(self, package_string, executable=sys.executable): + if not self.__has_package(package_string): + command = [executable, "-m", "pip", "install", package_string] + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + sys.stdout.buffer.write(p.stderr.read()) + raise(Exception(f"[PythonRunner] Could not install dependencies ({p.returncode})")) + self.package_list = self.__get_packages() + + def __has_package(self, package_string): + package_data = re.split("==|>|>=|<|<=", package_string) + # check in cache + if package_data[0] in self.package_list: + # check if version is needed + if len(package_data) == 2: + required_version = parse_version(package_data[1]) + installed_version = self.package_list[package_data[0]] + comparator = package_string.replace(package_data[0], "").replace(package_data[1], "") + if comparator == "==": + return required_version == installed_version + elif comparator == ">": + return installed_version > required_version + elif comparator == ">=": + return installed_version >= required_version + elif comparator == "<": + return installed_version < required_version + elif comparator == "<=": + return installed_version <= required_version + else: + raise Exception(f"Illegal comparator found: {comparator}") + else: + return True + return False + + + +if __name__ == "__main__": + p = PackageManager().getInstance() + + print(p.package_list) + p.ensure_more(["kubernetes", "minio"]) \ No newline at end of file