From 85a2fc85a38145e0d1e4e24d10ae3c5bea384e81 Mon Sep 17 00:00:00 2001
From: gyulaid <gyulaid@gyulai.cloud>
Date: Thu, 7 Apr 2022 18:03:51 +0000
Subject: [PATCH 1/5] PyPiRunner class

---
 alice-ci/src/alice/cli.py                  |  1 -
 alice-ci/src/alice/runnerfactory.py        |  4 ++-
 alice-ci/src/alice/runners/__init__.py     |  1 +
 alice-ci/src/alice/runners/pypirunner.py   | 29 ++++++++++++++++++++++
 alice-ci/src/alice/runners/pythonrunner.py |  2 ++
 ci-examples/full.yaml                      | 14 ++++++++++-
 6 files changed, 48 insertions(+), 3 deletions(-)
 create mode 100644 alice-ci/src/alice/runners/pypirunner.py

diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py
index d80f2c7..0820469 100644
--- a/alice-ci/src/alice/cli.py
+++ b/alice-ci/src/alice/cli.py
@@ -33,7 +33,6 @@ def parse_jobs(args):
                 print(f"[Alice][Step] {step}: {status}")
             else:
                 raise ConfigException(f"Step {step} not found in {args.input}")
-                exit(1)
     except ConfigException as e:
         print(f"Configuration error-> {e}")
         exit(1)
diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py
index 6218362..5d7feee 100644
--- a/alice-ci/src/alice/runnerfactory.py
+++ b/alice-ci/src/alice/runnerfactory.py
@@ -1,4 +1,5 @@
 from alice.runners.pythonrunner import PythonRunner
+from alice.runners.pypirunner import PyPiRunner
 from alice.exceptions import ConfigException
 
 
@@ -15,7 +16,8 @@ class Factory():
         # https://git.gyulai.cloud/gyulaid/alice/issues/4
         # module = __import__("module_file")
         # my_class = getattr(module, "class_name")
-        runners = {"python": PythonRunner}
+        runners = {"python": PythonRunner,
+                   "pypi": PyPiRunner}
 
         if (self.verbose):
             print(f"[Alice] Available runners: {'|'.join(runners.keys())}")
diff --git a/alice-ci/src/alice/runners/__init__.py b/alice-ci/src/alice/runners/__init__.py
index 973c468..b67238e 100644
--- a/alice-ci/src/alice/runners/__init__.py
+++ b/alice-ci/src/alice/runners/__init__.py
@@ -1,2 +1,3 @@
 # flake8: noqa F401
 from alice.runners.pythonrunner import PythonRunner
+from alice.runners.pypirunner import PyPiRunner
diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py
new file mode 100644
index 0000000..3eda5e9
--- /dev/null
+++ b/alice-ci/src/alice/runners/pypirunner.py
@@ -0,0 +1,29 @@
+import json
+import sys
+from urllib import request    
+from pkg_resources import parse_version   
+
+from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
+
+
+class PyPiRunner():
+    def __init__(self, params, user_defaults) -> None:
+        self.verbose = params["verbose"]
+        if self.verbose:
+            print("[PyPiRunner] Initializing")
+        self.workdir = user_defaults["workdir"]
+        # config only contains env and workdir
+        self.config = user_defaults
+        self.default_repo = "https://pypi.python.org/pypi"
+
+    def __versions(self, pkg_name):
+        # TODO: Error handling
+        url = f'{self.default_repo}/{pkg_name}/json'
+        releases = json.loads(request.urlopen(url).read())['releases']
+        return sorted(releases, key=parse_version, reverse=True) 
+
+    def update_config(self, config):
+        print(config)
+
+    def run(self, job_spec):
+        print(self.__versions("alice-ci"))
\ No newline at end of file
diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py
index 4dbcd0b..cda91b8 100644
--- a/alice-ci/src/alice/runners/pythonrunner.py
+++ b/alice-ci/src/alice/runners/pythonrunner.py
@@ -42,6 +42,8 @@ class PythonRunner():
     # Also - dependency install by config is only allowed in this step
     def update_config(self, config):
         if "dependencies" in config:
+            # TODO: Use pip freeze to list installed packages, only install missing
+            # ...or just dump the dependencies in a single list, and let pip deal with it.
             for dependency in config["dependencies"]:
                 # TODO: Check what happens with fixed version
                 command = [self.vpython, "-m", "pip", "install", dependency, "--upgrade"]
diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml
index f5b9191..565c477 100644
--- a/ci-examples/full.yaml
+++ b/ci-examples/full.yaml
@@ -28,6 +28,18 @@ jobs:
     commands:
       - "-c \"import os; print(os.environ)\""
   - name: lint
+    type: python
     workdir: alice-ci
     commands:
-      - "-m flake8 --ignore E501"
\ No newline at end of file
+      - "-m flake8 --ignore E501"
+  - name: pkg
+    type: pypi
+    workdir: .
+    upload: false
+    repo:
+      uri: example.com
+      username: asdf
+      password:
+        from_env: PYPIPASS
+    packages:
+     - alice-ci
\ No newline at end of file
-- 
2.25.1


From afebbc60a30f29be321ec30dc1d90efc0ea7349e Mon Sep 17 00:00:00 2001
From: gyulaid <gyulaid@gyulai.cloud>
Date: Fri, 8 Apr 2022 21:47:36 +0000
Subject: [PATCH 2/5] PackageManager class

---
 alice-ci/src/alice/cli.py                     |  2 +-
 .../src/alice/{utils.py => configparser.py}   |  1 +
 alice-ci/src/alice/runners/pythonrunner.py    | 21 ++---
 alice-ci/src/alice/runners/pyutils.py         | 79 +++++++++++++++++
 test.py                                       | 85 +++++++++++++++++++
 5 files changed, 177 insertions(+), 11 deletions(-)
 rename alice-ci/src/alice/{utils.py => configparser.py} (97%)
 create mode 100644 alice-ci/src/alice/runners/pyutils.py
 create mode 100644 test.py

diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py
index f92d6b3..e03f06a 100644
--- a/alice-ci/src/alice/cli.py
+++ b/alice-ci/src/alice/cli.py
@@ -1,7 +1,7 @@
 import os
 import argparse
 
-from alice.utils import ConfigParser
+from alice.configparser import ConfigParser
 from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError
 
 
diff --git a/alice-ci/src/alice/utils.py b/alice-ci/src/alice/configparser.py
similarity index 97%
rename from alice-ci/src/alice/utils.py
rename to alice-ci/src/alice/configparser.py
index 4ecfb26..4af4848 100644
--- a/alice-ci/src/alice/utils.py
+++ b/alice-ci/src/alice/configparser.py
@@ -1,3 +1,4 @@
+import inspect
 from os import getcwd, path, environ
 import subprocess
 import yaml
diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py
index 3a7b4dd..bd3dc1b 100644
--- a/alice-ci/src/alice/runners/pythonrunner.py
+++ b/alice-ci/src/alice/runners/pythonrunner.py
@@ -1,12 +1,14 @@
+from http.server import executable
 import subprocess
 import os
 import sys
 import shlex
 
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
+from alice.runners.pyutils import PackageManager
 
-
-class PythonRunner():
+# TODO: Handle config like PyPiConfig
+class PythonRunner:
     def __init__(self, params, config) -> None:
         self.verbose = params["verbose"]
         if self.verbose:
@@ -36,14 +38,13 @@ class PythonRunner():
         else:
             if self.verbose:
                 print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}")
-
-        if "dependencies" in self.config:
-            command = [self.vpython, "-m", "pip", "install"] + self.config["dependencies"] + ["--upgrade"]
-            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
-                p.wait()
-                if p.returncode != 0:
-                    sys.stdout.buffer.write(p.stderr.read())
-                    raise(RunnerError(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
+        dependencies = self.config.get("dependencies", [])
+        if len(dependencies) >0:
+            if self.verbose:
+                print(f"[PythonRunner] Ensuring dependencies:  {', '.join(dependencies)}")
+            PackageManager.getInstance().ensure_more(dependencies, executable=self.vpython)
+            if self.verbose:
+                print(f"[PythonRunner] Installation done")
 
     def __ghetto_glob(self, command, workdir):
         if self.verbose:
diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py
new file mode 100644
index 0000000..6631f53
--- /dev/null
+++ b/alice-ci/src/alice/runners/pyutils.py
@@ -0,0 +1,79 @@
+import subprocess
+import sys
+from pkg_resources import parse_version 
+import re
+
+from alice.exceptions import RunnerError, ConfigException
+
+class PackageManager:
+    __instance = None
+    @staticmethod
+    def getInstance():
+        """ Static access method. """
+        if PackageManager.__instance == None:
+            PackageManager()
+        return PackageManager.__instance
+    def __init__(self):
+        """ Virtually private constructor. """
+        if PackageManager.__instance != None:
+            raise Exception("This class is a singleton!")
+        else:
+            PackageManager.__instance = self
+        self.package_list = self.__get_packages()
+
+    def __get_packages(self):
+        packages = {}
+        with subprocess.Popen([sys.executable, "-m", "pip", "freeze"],
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines())))
+        for name, version in installed:
+            packages[name] = parse_version(version)
+        return packages
+
+    def ensure_more(self, package_list, executable=sys.executable):
+        to_install = list(filter(lambda x: not self.__has_package(x), package_list))
+        if len(to_install) > 0:
+            command = [executable, "-m", "pip", "install"] + to_install
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    # Assumption: there are more hits in the long run, than misses
+    def ensure(self, package_string, executable=sys.executable):
+        if not self.__has_package(package_string):
+            command = [executable, "-m", "pip", "install", package_string]
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(RunnerError(f"[PackageManager] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    def __has_package(self, package_string):
+        package_data = re.split("==|>|>=|<|<=", package_string)
+        # check in cache
+        if package_data[0] in self.package_list:
+            # check if version is needed
+            if len(package_data) == 2:
+                required_version = parse_version(package_data[1])
+                installed_version = self.package_list[package_data[0]]
+                comparator = package_string.replace(package_data[0], "").replace(package_data[1], "")
+                if comparator == "==":
+                    return required_version == installed_version
+                elif comparator == ">":
+                    return installed_version > required_version
+                elif comparator == ">=":
+                    return installed_version >= required_version
+                elif comparator == "<":
+                    return installed_version < required_version
+                elif comparator == "<=":
+                    return installed_version <= required_version
+                else:
+                    raise ConfigException(f"Illegal comparator found: {comparator}")
+            else:
+                return True
+        return False
\ No newline at end of file
diff --git a/test.py b/test.py
new file mode 100644
index 0000000..ef1a001
--- /dev/null
+++ b/test.py
@@ -0,0 +1,85 @@
+import subprocess
+import sys
+from pkg_resources import parse_version 
+import re
+
+class PackageManager:
+    __instance = None
+    @staticmethod
+    def getInstance():
+        """ Static access method. """
+        if PackageManager.__instance == None:
+            PackageManager()
+        return PackageManager.__instance
+    def __init__(self):
+        """ Virtually private constructor. """
+        if PackageManager.__instance != None:
+            raise Exception("This class is a singleton!")
+        else:
+            PackageManager.__instance = self
+        self.package_list = self.__get_packages()
+
+    def __get_packages(self):
+        packages = {}
+        with subprocess.Popen([sys.executable, "-m", "pip", "freeze"],
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines())))
+        for name, version in installed:
+            packages[name] = parse_version(version)
+        return packages
+
+    def ensure_more(self, package_list, executable=sys.executable):
+        to_install = list(filter(lambda x: not self.__has_package(x), package_list))
+        if len(to_install) > 0:
+            command = [executable, "-m", "pip", "install"] + to_install
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(Exception(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    # Assumption: there are more hits in the long run, than misses
+    def ensure(self, package_string, executable=sys.executable):
+        if not self.__has_package(package_string):
+            command = [executable, "-m", "pip", "install", package_string]
+            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                p.wait()
+                if p.returncode != 0:
+                    sys.stdout.buffer.write(p.stderr.read())
+                    raise(Exception(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
+            self.package_list = self.__get_packages()
+
+    def __has_package(self, package_string):
+        package_data = re.split("==|>|>=|<|<=", package_string)
+        # check in cache
+        if package_data[0] in self.package_list:
+            # check if version is needed
+            if len(package_data) == 2:
+                required_version = parse_version(package_data[1])
+                installed_version = self.package_list[package_data[0]]
+                comparator = package_string.replace(package_data[0], "").replace(package_data[1], "")
+                if comparator == "==":
+                    return required_version == installed_version
+                elif comparator == ">":
+                    return installed_version > required_version
+                elif comparator == ">=":
+                    return installed_version >= required_version
+                elif comparator == "<":
+                    return installed_version < required_version
+                elif comparator == "<=":
+                    return installed_version <= required_version
+                else:
+                    raise Exception(f"Illegal comparator found: {comparator}")
+            else:
+                return True
+        return False
+
+
+
+if __name__ == "__main__":
+    p = PackageManager().getInstance()
+
+    print(p.package_list)
+    p.ensure_more(["kubernetes", "minio"])
\ No newline at end of file
-- 
2.25.1


From 5120293c294ccb14df3c512560f934b343ee8a09 Mon Sep 17 00:00:00 2001
From: gyulaid <gyulaid@gyulai.cloud>
Date: Fri, 8 Apr 2022 21:55:40 +0000
Subject: [PATCH 3/5] Cleanup + pep8

---
 alice-ci/src/alice/__init__.py             |  2 +-
 alice-ci/src/alice/configparser.py         |  1 -
 alice-ci/src/alice/runnerfactory.py        |  7 +-
 alice-ci/src/alice/runners/pythonrunner.py |  6 +-
 alice-ci/src/alice/runners/pyutils.py      | 13 ++--
 test.py                                    | 85 ----------------------
 6 files changed, 16 insertions(+), 98 deletions(-)
 delete mode 100644 test.py

diff --git a/alice-ci/src/alice/__init__.py b/alice-ci/src/alice/__init__.py
index a442884..3c513fe 100644
--- a/alice-ci/src/alice/__init__.py
+++ b/alice-ci/src/alice/__init__.py
@@ -1,5 +1,5 @@
 # flake8: noqa F401
-from alice.utils import ConfigParser
+from alice.configparser import ConfigParser
 from alice.exceptions import NonZeroRetcode
 from alice.runnerfactory import Factory
 from alice.runners.pythonrunner import PythonRunner
diff --git a/alice-ci/src/alice/configparser.py b/alice-ci/src/alice/configparser.py
index 4af4848..4ecfb26 100644
--- a/alice-ci/src/alice/configparser.py
+++ b/alice-ci/src/alice/configparser.py
@@ -1,4 +1,3 @@
-import inspect
 from os import getcwd, path, environ
 import subprocess
 import yaml
diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py
index 24db0a3..618248c 100644
--- a/alice-ci/src/alice/runnerfactory.py
+++ b/alice-ci/src/alice/runnerfactory.py
@@ -20,8 +20,8 @@ class Factory():
         # https://git.gyulai.cloud/gyulaid/alice/issues/4
         # module = __import__("module_file")
         # my_class = getattr(module, "class_name")
-        self.runners = {"python": PythonRunner,
-                   "pypi": PyPiRunner}
+        self.runnertypes = {"python": PythonRunner,
+                            "pypi": PyPiRunner}
 
         if (self.verbose):
             print(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}")
@@ -50,7 +50,8 @@ class Factory():
                 params = {
                     "verbose": self.verbose
                 }
-                config = self.runner_configs[runnertype]
+                # If there is a runner specific config, use that, else global
+                config = self.runner_configs.get(runnertype, self.globals.copy())
                 self.runners[runnertype] = self.runnertypes[runnertype](params, config)
             else:
                 raise ConfigException(f"Invalid runner type: {runnertype}")
diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py
index bd3dc1b..f69d9cf 100644
--- a/alice-ci/src/alice/runners/pythonrunner.py
+++ b/alice-ci/src/alice/runners/pythonrunner.py
@@ -1,4 +1,3 @@
-from http.server import executable
 import subprocess
 import os
 import sys
@@ -7,6 +6,7 @@ import shlex
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
 from alice.runners.pyutils import PackageManager
 
+
 # TODO: Handle config like PyPiConfig
 class PythonRunner:
     def __init__(self, params, config) -> None:
@@ -39,12 +39,12 @@ class PythonRunner:
             if self.verbose:
                 print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}")
         dependencies = self.config.get("dependencies", [])
-        if len(dependencies) >0:
+        if len(dependencies) > 0:
             if self.verbose:
                 print(f"[PythonRunner] Ensuring dependencies:  {', '.join(dependencies)}")
             PackageManager.getInstance().ensure_more(dependencies, executable=self.vpython)
             if self.verbose:
-                print(f"[PythonRunner] Installation done")
+                print("[PythonRunner] Installation done")
 
     def __ghetto_glob(self, command, workdir):
         if self.verbose:
diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py
index 6631f53..ca95641 100644
--- a/alice-ci/src/alice/runners/pyutils.py
+++ b/alice-ci/src/alice/runners/pyutils.py
@@ -1,21 +1,24 @@
 import subprocess
 import sys
-from pkg_resources import parse_version 
+from pkg_resources import parse_version
 import re
 
 from alice.exceptions import RunnerError, ConfigException
 
+
 class PackageManager:
     __instance = None
+
     @staticmethod
     def getInstance():
         """ Static access method. """
-        if PackageManager.__instance == None:
+        if PackageManager.__instance is None:
             PackageManager()
         return PackageManager.__instance
+
     def __init__(self):
         """ Virtually private constructor. """
-        if PackageManager.__instance != None:
+        if PackageManager.__instance is not None:
             raise Exception("This class is a singleton!")
         else:
             PackageManager.__instance = self
@@ -24,7 +27,7 @@ class PackageManager:
     def __get_packages(self):
         packages = {}
         with subprocess.Popen([sys.executable, "-m", "pip", "freeze"],
-                          stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+                              stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
             p.wait()
             installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines())))
         for name, version in installed:
@@ -76,4 +79,4 @@ class PackageManager:
                     raise ConfigException(f"Illegal comparator found: {comparator}")
             else:
                 return True
-        return False
\ No newline at end of file
+        return False
diff --git a/test.py b/test.py
deleted file mode 100644
index ef1a001..0000000
--- a/test.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import subprocess
-import sys
-from pkg_resources import parse_version 
-import re
-
-class PackageManager:
-    __instance = None
-    @staticmethod
-    def getInstance():
-        """ Static access method. """
-        if PackageManager.__instance == None:
-            PackageManager()
-        return PackageManager.__instance
-    def __init__(self):
-        """ Virtually private constructor. """
-        if PackageManager.__instance != None:
-            raise Exception("This class is a singleton!")
-        else:
-            PackageManager.__instance = self
-        self.package_list = self.__get_packages()
-
-    def __get_packages(self):
-        packages = {}
-        with subprocess.Popen([sys.executable, "-m", "pip", "freeze"],
-                          stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
-            p.wait()
-            installed = list(map(lambda x: x.decode("UTF-8").split("=="), filter(lambda x: b'==' in x, p.stdout.read().splitlines())))
-        for name, version in installed:
-            packages[name] = parse_version(version)
-        return packages
-
-    def ensure_more(self, package_list, executable=sys.executable):
-        to_install = list(filter(lambda x: not self.__has_package(x), package_list))
-        if len(to_install) > 0:
-            command = [executable, "-m", "pip", "install"] + to_install
-            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
-                p.wait()
-                if p.returncode != 0:
-                    sys.stdout.buffer.write(p.stderr.read())
-                    raise(Exception(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
-            self.package_list = self.__get_packages()
-
-    # Assumption: there are more hits in the long run, than misses
-    def ensure(self, package_string, executable=sys.executable):
-        if not self.__has_package(package_string):
-            command = [executable, "-m", "pip", "install", package_string]
-            with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
-                p.wait()
-                if p.returncode != 0:
-                    sys.stdout.buffer.write(p.stderr.read())
-                    raise(Exception(f"[PythonRunner] Could not install dependencies ({p.returncode})"))
-            self.package_list = self.__get_packages()
-
-    def __has_package(self, package_string):
-        package_data = re.split("==|>|>=|<|<=", package_string)
-        # check in cache
-        if package_data[0] in self.package_list:
-            # check if version is needed
-            if len(package_data) == 2:
-                required_version = parse_version(package_data[1])
-                installed_version = self.package_list[package_data[0]]
-                comparator = package_string.replace(package_data[0], "").replace(package_data[1], "")
-                if comparator == "==":
-                    return required_version == installed_version
-                elif comparator == ">":
-                    return installed_version > required_version
-                elif comparator == ">=":
-                    return installed_version >= required_version
-                elif comparator == "<":
-                    return installed_version < required_version
-                elif comparator == "<=":
-                    return installed_version <= required_version
-                else:
-                    raise Exception(f"Illegal comparator found: {comparator}")
-            else:
-                return True
-        return False
-
-
-
-if __name__ == "__main__":
-    p = PackageManager().getInstance()
-
-    print(p.package_list)
-    p.ensure_more(["kubernetes", "minio"])
\ No newline at end of file
-- 
2.25.1


From 32c08cf63e23c886f59e5377815babe3e56be647 Mon Sep 17 00:00:00 2001
From: gyulaid <gyulaid@gyulai.cloud>
Date: Fri, 8 Apr 2022 22:14:43 +0000
Subject: [PATCH 4/5] Added PyPiRunner  - unstable

---
 alice-ci/src/alice/runners/pypirunner.py | 104 ++++++++++++++++++++---
 ci-examples/full.yaml                    |   4 +-
 2 files changed, 92 insertions(+), 16 deletions(-)

diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py
index 3eda5e9..e644210 100644
--- a/alice-ci/src/alice/runners/pypirunner.py
+++ b/alice-ci/src/alice/runners/pypirunner.py
@@ -1,29 +1,105 @@
 import json
-import sys
-from urllib import request    
-from pkg_resources import parse_version   
+from urllib import request
+from pkg_resources import parse_version
+from os import environ, path
+from alice.runners.pyutils import PackageManager
+from alice.exceptions import ConfigException
 
-from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
+
+def grab_from(target):
+    if "from_env" in target:
+        return environ[target["from_env"]]
+    else:
+        raise ConfigException(f"Unsupported grabber: {target.keys()}")
+
+
+def get_uri(config, default):
+    if "repo" in config:
+        return config["repo"].get("uri", default)
+    return default
+
+
+def get_user(config, default):
+    if "repo" in config:
+        if "username" in config["repo"]:
+            data = config["repo"]["username"]
+            if isinstance(data, str):
+                return data
+            else:
+                return grab_from(data)
+    return default
+
+
+def get_pass(config, default):
+    if "repo" in config:
+        if "password" in config["repo"]:
+            data = config["repo"]["password"]
+            if isinstance(data, str):
+                return data
+            else:
+                return grab_from(data)
+    return default
+
+
+# Parses and stores the config from yaml
+class PypiConfig:
+    def __init__(self, config={}) -> None:
+        self.workdir = config.get("workdir", None)
+        self.repo_uri = get_uri(config, "https://pypi.python.org/pypi")
+        self.repo_user = get_user(config, None)
+        self.repo_pass = get_pass(config, None)
+        self.packages = config.get("packages", set())
+        self.upload = config.get("upload", False)
+        print(self.packages)
+
+    # returns a PyPiConfig with merged values
+    def copy(self, job_config={}):
+        p = PypiConfig()
+        p.workdir = job_config.get("workdir", self.workdir)
+        p.repo_uri = get_uri(job_config, self.repo_uri)
+        p.repo_user = get_user(job_config, self.repo_user)
+        p.repo_pass = get_pass(job_config, self.repo_pass)
+        p.packages = set(job_config.get("packages", [])).update(self.packages)
+        p.upload = job_config.get("upload", self.upload)
+        return p
 
 
 class PyPiRunner():
-    def __init__(self, params, user_defaults) -> None:
+    def __init__(self, params, config) -> None:
         self.verbose = params["verbose"]
         if self.verbose:
             print("[PyPiRunner] Initializing")
-        self.workdir = user_defaults["workdir"]
-        # config only contains env and workdir
-        self.config = user_defaults
-        self.default_repo = "https://pypi.python.org/pypi"
+        self.workdir = config["workdir"]
+        self.config = PypiConfig(config)
 
     def __versions(self, pkg_name):
         # TODO: Error handling
-        url = f'{self.default_repo}/{pkg_name}/json'
+        url = f'{self.config.repo_uri}/{pkg_name}/json'
         releases = json.loads(request.urlopen(url).read())['releases']
-        return sorted(releases, key=parse_version, reverse=True) 
+        return sorted(releases, key=parse_version, reverse=True)
+
+    def build(self, path):
+        # TODO: Actual build - silent, unless failure!
+        pass
 
-    def update_config(self, config):
-        print(config)
+    def upload(self, path, repo_uri, repo_pass, repo_user):
+        # TODO: Implement
+        pass
 
     def run(self, job_spec):
-        print(self.__versions("alice-ci"))
\ No newline at end of file
+        print(self.__versions("alice-ci"))
+        job_config = self.config.copy(job_spec)
+        # TODO: This prints out None !!!!!!!!!!!!!
+        print(job_config.packages)
+        return
+
+        PackageManager.getInstance().ensure("build")
+        for package in job_config.packages:
+            self.build(path.join(job_config.workdir, package))
+        if job_config.upload:
+            PackageManager.getInstance().ensure("twine")
+            for package in job_config.packages:
+                self.build(path.join(job_config.workdir, package),
+                           job_config.repo_uri,
+                           job_config.repo_pass,
+                           job_config.repo_user)
diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml
index 3a5d0fa..b6aba40 100644
--- a/ci-examples/full.yaml
+++ b/ci-examples/full.yaml
@@ -7,7 +7,7 @@ runners:
         value: B
       - name: C
         value: C
-    workdir: packages
+    workdir: .
   python:
     env:
       - name: A
@@ -41,6 +41,6 @@ jobs:
       uri: example.com
       username: asdf
       password:
-        from_env: PYPIPASS
+        from_env: COLORTERM
     packages:
      - alice-ci
\ No newline at end of file
-- 
2.25.1


From 406c059cb3f41fd3c6790429132f8187765d29c8 Mon Sep 17 00:00:00 2001
From: gyulaid <gyulaid@gyulai.cloud>
Date: Sun, 10 Apr 2022 17:19:32 +0000
Subject: [PATCH 5/5] PyPiRunner done

---
 alice-ci/setup.cfg                         |   2 +-
 alice-ci/src/alice/runners/pypirunner.py   | 132 ++++++++++++++++-----
 alice-ci/src/alice/runners/pythonrunner.py |  30 +----
 alice-ci/src/alice/runners/pyutils.py      |  33 ++++++
 ci-examples/full.yaml                      |   8 +-
 5 files changed, 143 insertions(+), 62 deletions(-)

diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg
index 5a3170e..b53d679 100644
--- a/alice-ci/setup.cfg
+++ b/alice-ci/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = alice-ci
-version = 0.0.8
+version = 0.0.6
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md
diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py
index e644210..332cdb3 100644
--- a/alice-ci/src/alice/runners/pypirunner.py
+++ b/alice-ci/src/alice/runners/pypirunner.py
@@ -1,9 +1,13 @@
 import json
-from urllib import request
+import os
+import re
+import subprocess
+import sys
+from urllib import request, error
 from pkg_resources import parse_version
 from os import environ, path
-from alice.runners.pyutils import PackageManager
-from alice.exceptions import ConfigException
+from alice.runners.pyutils import PackageManager, glob
+from alice.exceptions import ConfigException, RunnerError
 
 
 def grab_from(target):
@@ -14,9 +18,11 @@ def grab_from(target):
 
 
 def get_uri(config, default):
-    if "repo" in config:
-        return config["repo"].get("uri", default)
-    return default
+    url = config.get("repo", {}).get("uri", default)
+    if url is not None:
+        if not re.match('(?:http|ftp|https)://', url):
+            url = f"https://{url}"
+    return url
 
 
 def get_user(config, default):
@@ -44,26 +50,30 @@ def get_pass(config, default):
 # Parses and stores the config from yaml
 class PypiConfig:
     def __init__(self, config={}) -> None:
-        self.workdir = config.get("workdir", None)
-        self.repo_uri = get_uri(config, "https://pypi.python.org/pypi")
+        self.workdir = path.abspath(config.get("workdir", "."))
+        self.repo_uri = get_uri(config, None)
         self.repo_user = get_user(config, None)
         self.repo_pass = get_pass(config, None)
-        self.packages = config.get("packages", set())
+        self.packages = set(config.get("packages", []))
         self.upload = config.get("upload", False)
-        print(self.packages)
+        self.fail_if_exists = config.get("fail_if_exists", False)
 
     # returns a PyPiConfig with merged values
     def copy(self, job_config={}):
         p = PypiConfig()
-        p.workdir = job_config.get("workdir", self.workdir)
+        p.workdir = path.abspath(path.join(self.workdir, job_config.get("workdir", ".")))
         p.repo_uri = get_uri(job_config, self.repo_uri)
         p.repo_user = get_user(job_config, self.repo_user)
         p.repo_pass = get_pass(job_config, self.repo_pass)
-        p.packages = set(job_config.get("packages", [])).update(self.packages)
+        job_pkg_set = set(job_config["packages"])
+        job_pkg_set.update(self.packages)
+        p.packages = job_pkg_set
         p.upload = job_config.get("upload", self.upload)
+        p.fail_if_exists = job_config.get("fail_if_exists", self.fail_if_exists)
         return p
 
 
+# TODO: consider "--skip-existing" flag for twine
 class PyPiRunner():
     def __init__(self, params, config) -> None:
         self.verbose = params["verbose"]
@@ -72,34 +82,96 @@ class PyPiRunner():
         self.workdir = config["workdir"]
         self.config = PypiConfig(config)
 
-    def __versions(self, pkg_name):
-        # TODO: Error handling
-        url = f'{self.config.repo_uri}/{pkg_name}/json'
-        releases = json.loads(request.urlopen(url).read())['releases']
+    def __versions(self, repo, pkg_name):
+        if repo is not None:
+            url = f'{repo}/{pkg_name}/json'
+        else:
+            url = f"https://pypi.python.org/pypi/{pkg_name}/json"
+        try:
+            releases = json.loads(request.urlopen(url).read())['releases']
+        except error.URLError as e:
+            raise RunnerError(f"{url}: {e}")
+
         return sorted(releases, key=parse_version, reverse=True)
 
-    def build(self, path):
+    def build(self, config, package):
         # TODO: Actual build - silent, unless failure!
-        pass
+        pkg_path = path.join(config.workdir, package)
+        if not path.isdir(pkg_path):
+            raise ConfigException(f"Path does not exists: {pkg_path}")
+        command = [sys.executable, "-m", "build", package]
+        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            if p.returncode != 0:
+                print("STDOUT:")
+                sys.stdout.buffer.write(p.stdout.read())
+                print("STDERR:")
+                sys.stdout.buffer.write(p.stderr.read())
+                raise RunnerError(f"[PyPiRunner] Failed to build {package}")
+
+    def find_unuploaded(self, repo, file_list, pkg_name):
+        versions = self.__versions(repo, pkg_name)
+        unuploaded = []
+        for file in file_list:
+            # flake8: noqa W605
+            re_groups = re.findall("(\d*\.\d*\.\d*)", file)
+            if len(re_groups) < 1:
+                raise RunnerError(f"Unable to determine version of file {file}")
+            file_version = re_groups[0]
+            if file_version not in versions:
+                unuploaded.append(file)
+            else:
+                print(f"[PyPiRunner] File already uploaded: {os.path.basename(file)}")
+        return unuploaded
 
-    def upload(self, path, repo_uri, repo_pass, repo_user):
-        # TODO: Implement
-        pass
+
+    def upload(self, config, package):
+        command = [sys.executable, "-m", "twine", "upload"]
+        if self.verbose:
+            command.append("--verbose")
+        if config.repo_uri is not None:
+            command.append("--repository-url")
+            command.append(config.repo_uri)
+        if config.repo_user is not None:
+            command.append("-u")
+            command.append(config.repo_user)
+        if config.repo_pass is not None:
+            command.append("-p")
+            command.append(config.repo_pass)
+        
+        dist_path = os.path.abspath(os.path.join(config.workdir, package, "dist"))
+        files = glob(os.path.join(dist_path, "*"), config.workdir)
+        for file in files:
+            print(f"[PyPiRunner] Found: {file}")
+
+        to_upload = self.find_unuploaded(config.repo_uri, files, package)
+        if len(to_upload) == 0:
+            return
+        command += to_upload
+        print(command)
+        print(" ".join(command))
+        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            if p.returncode != 0:
+                print("STDOUT:")
+                sys.stdout.buffer.write(p.stdout.read())
+                print("STDERR:")
+                sys.stdout.buffer.write(p.stderr.read())
+                raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})")
 
     def run(self, job_spec):
-        print(self.__versions("alice-ci"))
         job_config = self.config.copy(job_spec)
-        # TODO: This prints out None !!!!!!!!!!!!!
-        print(job_config.packages)
-        return
 
         PackageManager.getInstance().ensure("build")
         for package in job_config.packages:
-            self.build(path.join(job_config.workdir, package))
+            print(f"[PyPiRunner] Building {package}")
+            #self.build(job_config, package)
+            print(f"[PyPiRunner] Package {package} built")
+
         if job_config.upload:
             PackageManager.getInstance().ensure("twine")
             for package in job_config.packages:
-                self.build(path.join(job_config.workdir, package),
-                           job_config.repo_uri,
-                           job_config.repo_pass,
-                           job_config.repo_user)
+                print(f"[PyPiRunner] Uploading {package}")
+                self.upload(job_config, package)
+        else:
+            print(f"[PyPiRunner] Upload disabled, skiping")
diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py
index f69d9cf..268cbd8 100644
--- a/alice-ci/src/alice/runners/pythonrunner.py
+++ b/alice-ci/src/alice/runners/pythonrunner.py
@@ -4,7 +4,7 @@ import sys
 import shlex
 
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
-from alice.runners.pyutils import PackageManager
+from alice.runners.pyutils import PackageManager, glob_command
 
 
 # TODO: Handle config like PyPiConfig
@@ -46,32 +46,6 @@ class PythonRunner:
             if self.verbose:
                 print("[PythonRunner] Installation done")
 
-    def __ghetto_glob(self, command, workdir):
-        if self.verbose:
-            print(f"[PythonRunner][Globbing] Starting command: {' '.join(command)}")
-        new_command = []
-        for item in command:
-            if "*" in item:
-                if self.verbose:
-                    print(f"[PythonRunner][Globbing] Found item: [{item}]")
-                dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item)))
-                base_name = os.path.basename(item)
-                if os.path.isdir(dir):
-                    item_parts = base_name.split("*")
-                    for file in os.listdir(dir):
-                        # TODO: Fix ordering! A*B = B*A = AB*
-                        if item_parts[0] in file and item_parts[1] in file:
-                            new_item = os.path.join(dir, file)
-                            if self.verbose:
-                                print(f"[PythonRunner][Globbing] Substitute: {new_item}")
-                            new_command.append(new_item)
-                else:
-                    if self.verbose:
-                        print(f"[PythonRunner][Globbing] Dir not exists: {dir}")
-            else:
-                new_command.append(item)
-        return new_command
-
     # Executes the given job in the one and only venv
     # parameter shall be the raw jobscpec
     def run(self, job_spec):
@@ -90,7 +64,7 @@ class PythonRunner:
                     print(f"[PythonRunner] Raw command: {command}")
                 # TODO: only split if command is not an array
                 if "*" in command:
-                    run_command = self.__ghetto_glob(shlex.split(command), pwd)
+                    run_command = glob_command(shlex.split(command), pwd, self.verbose)
                 else:
                     run_command = shlex.split(command)
                 if self.verbose:
diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py
index ca95641..62efdf7 100644
--- a/alice-ci/src/alice/runners/pyutils.py
+++ b/alice-ci/src/alice/runners/pyutils.py
@@ -1,3 +1,4 @@
+import os
 import subprocess
 import sys
 from pkg_resources import parse_version
@@ -80,3 +81,35 @@ class PackageManager:
             else:
                 return True
         return False
+
+
+def glob(item, workdir, verbose=False):
+    new_command = []
+    if "*" in item:
+        if verbose:
+            print(f"[Globbing] Found item: [{item}]")
+        dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item)))
+        base_name = os.path.basename(item)
+        if os.path.isdir(dir):
+            item_parts = base_name.split("*")
+            for file in os.listdir(dir):
+                # TODO: Fix ordering! A*B = B*A = AB*
+                if item_parts[0] in file and item_parts[1] in file:
+                    new_item = os.path.join(dir, file)
+                    if verbose:
+                        print(f"[Globbing] Substitute: {new_item}")
+                    new_command.append(new_item)
+        else:
+            raise ConfigException(f"[Globbing] Dir not exists: {dir}")
+        return new_command
+    else:
+        return [item]
+
+
+def glob_command(command, workdir, verbose=False):
+    if verbose:
+        print(f"[Globbing] Starting command: {' '.join(command)}")
+    new_command = []
+    for item in command:
+        new_command += glob(item, workdir, verbose)
+    return new_command
diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml
index b6aba40..a7f8e61 100644
--- a/ci-examples/full.yaml
+++ b/ci-examples/full.yaml
@@ -36,11 +36,13 @@ jobs:
   - name: pkg
     type: pypi
     workdir: .
-    upload: false
+    upload: true
+    fail_if_exists: false
     repo:
       uri: example.com
-      username: asdf
+      username:
+        from_env: PYPIUSER
       password:
-        from_env: COLORTERM
+        from_env: PYPIPASS
     packages:
      - alice-ci
\ No newline at end of file
-- 
2.25.1