PyPiRunner done

alice-ci/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = alice-ci
-version = 0.0.8
+version = 0.0.6
 author = Daniel Gyulai
 description = Alice CI framework
 long_description = file: README.md

alice-ci/src/alice/runners/pypirunner.py

@@ -1,9 +1,13 @@
 import json
-from urllib import request
+import os
+import re
+import subprocess
+import sys
+from urllib import request, error
 from pkg_resources import parse_version
 from os import environ, path
-from alice.runners.pyutils import PackageManager
+from alice.runners.pyutils import PackageManager, glob
-from alice.exceptions import ConfigException
+from alice.exceptions import ConfigException, RunnerError
 
 
 def grab_from(target):
@@ -14,9 +18,11 @@ def grab_from(target):
 
 
 def get_uri(config, default):
-    if "repo" in config:
+    url = config.get("repo", {}).get("uri", default)
-        return config["repo"].get("uri", default)
+    if url is not None:
-    return default
+        if not re.match('(?:http|ftp|https)://', url):
+            url = f"https://{url}"
+    return url
 
 
 def get_user(config, default):
@@ -44,26 +50,30 @@ def get_pass(config, default):
 # Parses and stores the config from yaml
 class PypiConfig:
     def __init__(self, config={}) -> None:
-        self.workdir = config.get("workdir", None)
+        self.workdir = path.abspath(config.get("workdir", "."))
-        self.repo_uri = get_uri(config, "https://pypi.python.org/pypi")
+        self.repo_uri = get_uri(config, None)
         self.repo_user = get_user(config, None)
         self.repo_pass = get_pass(config, None)
-        self.packages = config.get("packages", set())
+        self.packages = set(config.get("packages", []))
         self.upload = config.get("upload", False)
-        print(self.packages)
+        self.fail_if_exists = config.get("fail_if_exists", False)
 
     # returns a PyPiConfig with merged values
     def copy(self, job_config={}):
         p = PypiConfig()
-        p.workdir = job_config.get("workdir", self.workdir)
+        p.workdir = path.abspath(path.join(self.workdir, job_config.get("workdir", ".")))
         p.repo_uri = get_uri(job_config, self.repo_uri)
         p.repo_user = get_user(job_config, self.repo_user)
         p.repo_pass = get_pass(job_config, self.repo_pass)
-        p.packages = set(job_config.get("packages", [])).update(self.packages)
+        job_pkg_set = set(job_config["packages"])
+        job_pkg_set.update(self.packages)
+        p.packages = job_pkg_set
         p.upload = job_config.get("upload", self.upload)
+        p.fail_if_exists = job_config.get("fail_if_exists", self.fail_if_exists)
         return p
 
 
+# TODO: consider "--skip-existing" flag for twine
 class PyPiRunner():
     def __init__(self, params, config) -> None:
         self.verbose = params["verbose"]
@@ -72,34 +82,96 @@ class PyPiRunner():
         self.workdir = config["workdir"]
         self.config = PypiConfig(config)
 
-    def __versions(self, pkg_name):
+    def __versions(self, repo, pkg_name):
-        # TODO: Error handling
+        if repo is not None:
-        url = f'{self.config.repo_uri}/{pkg_name}/json'
+            url = f'{repo}/{pkg_name}/json'
-        releases = json.loads(request.urlopen(url).read())['releases']
+        else:
+            url = f"https://pypi.python.org/pypi/{pkg_name}/json"
+        try:
+            releases = json.loads(request.urlopen(url).read())['releases']
+        except error.URLError as e:
+            raise RunnerError(f"{url}: {e}")
+
         return sorted(releases, key=parse_version, reverse=True)
 
-    def build(self, path):
+    def build(self, config, package):
         # TODO: Actual build - silent, unless failure!
-        pass
+        pkg_path = path.join(config.workdir, package)
+        if not path.isdir(pkg_path):
+            raise ConfigException(f"Path does not exists: {pkg_path}")
+        command = [sys.executable, "-m", "build", package]
+        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            if p.returncode != 0:
+                print("STDOUT:")
+                sys.stdout.buffer.write(p.stdout.read())
+                print("STDERR:")
+                sys.stdout.buffer.write(p.stderr.read())
+                raise RunnerError(f"[PyPiRunner] Failed to build {package}")
+
+    def find_unuploaded(self, repo, file_list, pkg_name):
+        versions = self.__versions(repo, pkg_name)
+        unuploaded = []
+        for file in file_list:
+            # flake8: noqa W605
+            re_groups = re.findall("(\d*\.\d*\.\d*)", file)
+            if len(re_groups) < 1:
+                raise RunnerError(f"Unable to determine version of file {file}")
+            file_version = re_groups[0]
+            if file_version not in versions:
+                unuploaded.append(file)
+            else:
+                print(f"[PyPiRunner] File already uploaded: {os.path.basename(file)}")
+        return unuploaded
 
-    def upload(self, path, repo_uri, repo_pass, repo_user):
+
-        # TODO: Implement
+    def upload(self, config, package):
-        pass
+        command = [sys.executable, "-m", "twine", "upload"]
+        if self.verbose:
+            command.append("--verbose")
+        if config.repo_uri is not None:
+            command.append("--repository-url")
+            command.append(config.repo_uri)
+        if config.repo_user is not None:
+            command.append("-u")
+            command.append(config.repo_user)
+        if config.repo_pass is not None:
+            command.append("-p")
+            command.append(config.repo_pass)
+        
+        dist_path = os.path.abspath(os.path.join(config.workdir, package, "dist"))
+        files = glob(os.path.join(dist_path, "*"), config.workdir)
+        for file in files:
+            print(f"[PyPiRunner] Found: {file}")
+
+        to_upload = self.find_unuploaded(config.repo_uri, files, package)
+        if len(to_upload) == 0:
+            return
+        command += to_upload
+        print(command)
+        print(" ".join(command))
+        with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+            p.wait()
+            if p.returncode != 0:
+                print("STDOUT:")
+                sys.stdout.buffer.write(p.stdout.read())
+                print("STDERR:")
+                sys.stdout.buffer.write(p.stderr.read())
+                raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})")
 
     def run(self, job_spec):
-        print(self.__versions("alice-ci"))
         job_config = self.config.copy(job_spec)
-        # TODO: This prints out None !!!!!!!!!!!!!
-        print(job_config.packages)
-        return
 
         PackageManager.getInstance().ensure("build")
         for package in job_config.packages:
-            self.build(path.join(job_config.workdir, package))
+            print(f"[PyPiRunner] Building {package}")
+            #self.build(job_config, package)
+            print(f"[PyPiRunner] Package {package} built")
+
         if job_config.upload:
             PackageManager.getInstance().ensure("twine")
             for package in job_config.packages:
-                self.build(path.join(job_config.workdir, package),
+                print(f"[PyPiRunner] Uploading {package}")
-                           job_config.repo_uri,
+                self.upload(job_config, package)
-                           job_config.repo_pass,
+        else:
-                           job_config.repo_user)
+            print(f"[PyPiRunner] Upload disabled, skiping")

alice-ci/src/alice/runners/pythonrunner.py

@@ -4,7 +4,7 @@ import sys
 import shlex
 
 from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException
-from alice.runners.pyutils import PackageManager
+from alice.runners.pyutils import PackageManager, glob_command
 
 
 # TODO: Handle config like PyPiConfig
@@ -46,32 +46,6 @@ class PythonRunner:
             if self.verbose:
                 print("[PythonRunner] Installation done")
 
-    def __ghetto_glob(self, command, workdir):
-        if self.verbose:
-            print(f"[PythonRunner][Globbing] Starting command: {' '.join(command)}")
-        new_command = []
-        for item in command:
-            if "*" in item:
-                if self.verbose:
-                    print(f"[PythonRunner][Globbing] Found item: [{item}]")
-                dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item)))
-                base_name = os.path.basename(item)
-                if os.path.isdir(dir):
-                    item_parts = base_name.split("*")
-                    for file in os.listdir(dir):
-                        # TODO: Fix ordering! A*B = B*A = AB*
-                        if item_parts[0] in file and item_parts[1] in file:
-                            new_item = os.path.join(dir, file)
-                            if self.verbose:
-                                print(f"[PythonRunner][Globbing] Substitute: {new_item}")
-                            new_command.append(new_item)
-                else:
-                    if self.verbose:
-                        print(f"[PythonRunner][Globbing] Dir not exists: {dir}")
-            else:
-                new_command.append(item)
-        return new_command
-
     # Executes the given job in the one and only venv
     # parameter shall be the raw jobscpec
     def run(self, job_spec):
@@ -90,7 +64,7 @@ class PythonRunner:
                     print(f"[PythonRunner] Raw command: {command}")
                 # TODO: only split if command is not an array
                 if "*" in command:
-                    run_command = self.__ghetto_glob(shlex.split(command), pwd)
+                    run_command = glob_command(shlex.split(command), pwd, self.verbose)
                 else:
                     run_command = shlex.split(command)
                 if self.verbose:

alice-ci/src/alice/runners/pyutils.py

@@ -1,3 +1,4 @@
+import os
 import subprocess
 import sys
 from pkg_resources import parse_version
@@ -80,3 +81,35 @@ class PackageManager:
             else:
                 return True
         return False
+
+
+def glob(item, workdir, verbose=False):
+    new_command = []
+    if "*" in item:
+        if verbose:
+            print(f"[Globbing] Found item: [{item}]")
+        dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item)))
+        base_name = os.path.basename(item)
+        if os.path.isdir(dir):
+            item_parts = base_name.split("*")
+            for file in os.listdir(dir):
+                # TODO: Fix ordering! A*B = B*A = AB*
+                if item_parts[0] in file and item_parts[1] in file:
+                    new_item = os.path.join(dir, file)
+                    if verbose:
+                        print(f"[Globbing] Substitute: {new_item}")
+                    new_command.append(new_item)
+        else:
+            raise ConfigException(f"[Globbing] Dir not exists: {dir}")
+        return new_command
+    else:
+        return [item]
+
+
+def glob_command(command, workdir, verbose=False):
+    if verbose:
+        print(f"[Globbing] Starting command: {' '.join(command)}")
+    new_command = []
+    for item in command:
+        new_command += glob(item, workdir, verbose)
+    return new_command

ci-examples/full.yaml

@@ -36,11 +36,13 @@ jobs:
   - name: pkg
     type: pypi
     workdir: .
-    upload: false
+    upload: true
+    fail_if_exists: false
     repo:
       uri: example.com
-      username: asdf
+      username:
+        from_env: PYPIUSER
       password:
-        from_env: COLORTERM
+        from_env: PYPIPASS
     packages:
      - alice-ci