PackageManager class

alice-ci/src/alice/cli.py

@@ -1,7 +1,7 @@
 import os
 import argparse
 
-from alice.utils import ConfigParser
+from alice.configparser import ConfigParser
 from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
 
 

alice-ci/src/alice/utils.py → alice-ci/src/alice/configparser.py

@@ -1,3 +1,4 @@
+import inspect
 from os import getcwd, path, environ
 import subprocess
 import yaml

alice-ci/src/alice/runners/pythonrunner.py

@@ -1,12 +1,14 @@
+from http.server import executable
 import subprocess
 import os
 import sys
 import shlex
 
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
+from alice.runners.pyutils import PackageManager
 
-
+# TODO: Handle config like PyPiConfig
-class PythonRunner():
+class PythonRunner:
     def __init__(self, params, config) -> None:
         self.verbose = params["verbose"]
         if self.verbose:
@@ -36,14 +38,13 @@ class PythonRunner():
         else:
             if self.verbose:
                 print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}")
-
+        dependencies = self.config.get("dependencies", [])
-        if "dependencies" in self.config:
+        if len(dependencies) >0:
-            command = [self.vpython, "-m", "pip", "install"] + self.config["dependencies"] + ["--upgrade"]
+            if self.verbose:
-            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                print(f"[PythonRunner] Ensuring dependencies:  {', '.join(dependencies)}")
-                p.wait()
+            PackageManager.getInstance().ensure_more(dependencies, executable=self.vpython)
-                if p.returncode != 0:
+            if self.verbose:
-                    sys.stdout.buffer.write(p.stderr.read())
+                print(f"[PythonRunner] Installation done")
-                    raise(RunnerError(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
 
     def __ghetto_glob(self, command, workdir):
         if self.verbose:

alice-ci/src/alice/runners/pyutils.py

@@ -0,0 +1,79 @@
+import subprocess
+import sys
+from pkg_resources import parse_version 
+import re
+
+from alice.exceptions import RunnerError, ConfigException
+
+class PackageManager:
+    __instance = None
+    @staticmethod
+    def getInstance():
+        """ Static access method. """
+        if PackageManager.__instance == None:
+            PackageManager()
+        return PackageManager.__instance
+    def __init__(self):
+        """ Virtually private constructor. """
+        if PackageManager.__instance != None:
+            raise Exception("This class is a singleton!")
+        else:
+            PackageManager.__instance = self
+        self.package_list = self.__get_packages()
+
+    def __get_packages(self):
+        packages = {}
+        with subprocess.Popen([sys.executable, "-m", "pip", "freeze"],
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines())))
+        for name, version in installed:
+            packages[name] = parse_version(version)
+        return packages
+
+    def ensure_more(self, package_list, executable=sys.executable):
+        to_install = list(filter(lambda x: not self.__has_package(x), package_list))
+        if len(to_install) > 0:
+            command = [executable, "-m", "pip", "install"] + to_install
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    # Assumption: there are more hits in the long run, than misses
+    def ensure(self, package_string, executable=sys.executable):
+        if not self.__has_package(package_string):
+            command = [executable, "-m", "pip", "install", package_string]
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    def __has_package(self, package_string):
+        package_data = re.split("==|>|>=|<|<=", package_string)
+        # check in cache
+        if package_data[0] in self.package_list:
+            # check if version is needed
+            if len(package_data) == 2:
+                required_version = parse_version(package_data[1])
+                installed_version = self.package_list[package_data[0]]
+                comparator = package_string.replace(package_data[0], "").replace(package_data[1], "")
+                if comparator == "==":
+                    return required_version == installed_version
+                elif comparator == ">":
+                    return installed_version > required_version
+                elif comparator == ">=":
+                    return installed_version >= required_version
+                elif comparator == "<":
+                    return installed_version < required_version
+                elif comparator == "<=":
+                    return installed_version <= required_version
+                else:
+                    raise ConfigException(f"Illegal comparator found: {comparator}")
+            else:
+                return True
+        return False

test.py

@@ -0,0 +1,85 @@
+import subprocess
+import sys
+from pkg_resources import parse_version 
+import re
+
+class PackageManager:
+    __instance = None
+    @staticmethod
+    def getInstance():
+        """ Static access method. """
+        if PackageManager.__instance == None:
+            PackageManager()
+        return PackageManager.__instance
+    def __init__(self):
+        """ Virtually private constructor. """
+        if PackageManager.__instance != None:
+            raise Exception("This class is a singleton!")
+        else:
+            PackageManager.__instance = self
+        self.package_list = self.__get_packages()
+
+    def __get_packages(self):
+        packages = {}
+        with subprocess.Popen([sys.executable, "-m", "pip", "freeze"],
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines())))
+        for name, version in installed:
+            packages[name] = parse_version(version)
+        return packages
+
+    def ensure_more(self, package_list, executable=sys.executable):
+        to_install = list(filter(lambda x: not self.__has_package(x), package_list))
+        if len(to_install) > 0:
+            command = [executable, "-m", "pip", "install"] + to_install
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(Exception(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    # Assumption: there are more hits in the long run, than misses
+    def ensure(self, package_string, executable=sys.executable):
+        if not self.__has_package(package_string):
+            command = [executable, "-m", "pip", "install", package_string]
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(Exception(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    def __has_package(self, package_string):
+        package_data = re.split("==|>|>=|<|<=", package_string)
+        # check in cache
+        if package_data[0] in self.package_list:
+            # check if version is needed
+            if len(package_data) == 2:
+                required_version = parse_version(package_data[1])
+                installed_version = self.package_list[package_data[0]]
+                comparator = package_string.replace(package_data[0], "").replace(package_data[1], "")
+                if comparator == "==":
+                    return required_version == installed_version
+                elif comparator == ">":
+                    return installed_version > required_version
+                elif comparator == ">=":
+                    return installed_version >= required_version
+                elif comparator == "<":
+                    return installed_version < required_version
+                elif comparator == "<=":
+                    return installed_version <= required_version
+                else:
+                    raise Exception(f"Illegal comparator found: {comparator}")
+            else:
+                return True
+        return False
+
+
+
+if __name__ == "__main__":
+    p = PackageManager().getInstance()
+
+    print(p.package_list)
+    p.ensure_more(["kubernetes", "minio"])