From 406c059cb3f41fd3c6790429132f8187765d29c8 Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 10 Apr 2022 17:19:32 +0000 Subject: [PATCH] PyPiRunner done --- alice-ci/setup.cfg | 2 +- alice-ci/src/alice/runners/pypirunner.py | 132 ++++++++++++++++----- alice-ci/src/alice/runners/pythonrunner.py | 30 +---- alice-ci/src/alice/runners/pyutils.py | 33 ++++++ ci-examples/full.yaml | 8 +- 5 files changed, 143 insertions(+), 62 deletions(-) diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index 5a3170e..b53d679 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = alice-ci -version = 0.0.8 +version = 0.0.6 author = Daniel Gyulai description = Alice CI framework long_description = file: README.md diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py index e644210..332cdb3 100644 --- a/alice-ci/src/alice/runners/pypirunner.py +++ b/alice-ci/src/alice/runners/pypirunner.py @@ -1,9 +1,13 @@ import json -from urllib import request +import os +import re +import subprocess +import sys +from urllib import request, error from pkg_resources import parse_version from os import environ, path -from alice.runners.pyutils import PackageManager -from alice.exceptions import ConfigException +from alice.runners.pyutils import PackageManager, glob +from alice.exceptions import ConfigException, RunnerError def grab_from(target): @@ -14,9 +18,11 @@ def grab_from(target): def get_uri(config, default): - if "repo" in config: - return config["repo"].get("uri", default) - return default + url = config.get("repo", {}).get("uri", default) + if url is not None: + if not re.match('(?:http|ftp|https)://', url): + url = f"https://{url}" + return url def get_user(config, default): @@ -44,26 +50,30 @@ def get_pass(config, default): # Parses and stores the config from yaml class PypiConfig: def __init__(self, config={}) -> None: - self.workdir = config.get("workdir", None) - self.repo_uri = get_uri(config, "https://pypi.python.org/pypi") + self.workdir = path.abspath(config.get("workdir", ".")) + self.repo_uri = get_uri(config, None) self.repo_user = get_user(config, None) self.repo_pass = get_pass(config, None) - self.packages = config.get("packages", set()) + self.packages = set(config.get("packages", [])) self.upload = config.get("upload", False) - print(self.packages) + self.fail_if_exists = config.get("fail_if_exists", False) # returns a PyPiConfig with merged values def copy(self, job_config={}): p = PypiConfig() - p.workdir = job_config.get("workdir", self.workdir) + p.workdir = path.abspath(path.join(self.workdir, job_config.get("workdir", "."))) p.repo_uri = get_uri(job_config, self.repo_uri) p.repo_user = get_user(job_config, self.repo_user) p.repo_pass = get_pass(job_config, self.repo_pass) - p.packages = set(job_config.get("packages", [])).update(self.packages) + job_pkg_set = set(job_config["packages"]) + job_pkg_set.update(self.packages) + p.packages = job_pkg_set p.upload = job_config.get("upload", self.upload) + p.fail_if_exists = job_config.get("fail_if_exists", self.fail_if_exists) return p +# TODO: consider "--skip-existing" flag for twine class PyPiRunner(): def __init__(self, params, config) -> None: self.verbose = params["verbose"] @@ -72,34 +82,96 @@ class PyPiRunner(): self.workdir = config["workdir"] self.config = PypiConfig(config) - def __versions(self, pkg_name): - # TODO: Error handling - url = f'{self.config.repo_uri}/{pkg_name}/json' - releases = json.loads(request.urlopen(url).read())['releases'] + def __versions(self, repo, pkg_name): + if repo is not None: + url = f'{repo}/{pkg_name}/json' + else: + url = f"https://pypi.python.org/pypi/{pkg_name}/json" + try: + releases = json.loads(request.urlopen(url).read())['releases'] + except error.URLError as e: + raise RunnerError(f"{url}: {e}") + return sorted(releases, key=parse_version, reverse=True) - def build(self, path): + def build(self, config, package): # TODO: Actual build - silent, unless failure! - pass + pkg_path = path.join(config.workdir, package) + if not path.isdir(pkg_path): + raise ConfigException(f"Path does not exists: {pkg_path}") + command = [sys.executable, "-m", "build", package] + with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + print("STDOUT:") + sys.stdout.buffer.write(p.stdout.read()) + print("STDERR:") + sys.stdout.buffer.write(p.stderr.read()) + raise RunnerError(f"[PyPiRunner] Failed to build {package}") + + def find_unuploaded(self, repo, file_list, pkg_name): + versions = self.__versions(repo, pkg_name) + unuploaded = [] + for file in file_list: + # flake8: noqa W605 + re_groups = re.findall("(\d*\.\d*\.\d*)", file) + if len(re_groups) < 1: + raise RunnerError(f"Unable to determine version of file {file}") + file_version = re_groups[0] + if file_version not in versions: + unuploaded.append(file) + else: + print(f"[PyPiRunner] File already uploaded: {os.path.basename(file)}") + return unuploaded - def upload(self, path, repo_uri, repo_pass, repo_user): - # TODO: Implement - pass + + def upload(self, config, package): + command = [sys.executable, "-m", "twine", "upload"] + if self.verbose: + command.append("--verbose") + if config.repo_uri is not None: + command.append("--repository-url") + command.append(config.repo_uri) + if config.repo_user is not None: + command.append("-u") + command.append(config.repo_user) + if config.repo_pass is not None: + command.append("-p") + command.append(config.repo_pass) + + dist_path = os.path.abspath(os.path.join(config.workdir, package, "dist")) + files = glob(os.path.join(dist_path, "*"), config.workdir) + for file in files: + print(f"[PyPiRunner] Found: {file}") + + to_upload = self.find_unuploaded(config.repo_uri, files, package) + if len(to_upload) == 0: + return + command += to_upload + print(command) + print(" ".join(command)) + with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + print("STDOUT:") + sys.stdout.buffer.write(p.stdout.read()) + print("STDERR:") + sys.stdout.buffer.write(p.stderr.read()) + raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})") def run(self, job_spec): - print(self.__versions("alice-ci")) job_config = self.config.copy(job_spec) - # TODO: This prints out None !!!!!!!!!!!!! - print(job_config.packages) - return PackageManager.getInstance().ensure("build") for package in job_config.packages: - self.build(path.join(job_config.workdir, package)) + print(f"[PyPiRunner] Building {package}") + #self.build(job_config, package) + print(f"[PyPiRunner] Package {package} built") + if job_config.upload: PackageManager.getInstance().ensure("twine") for package in job_config.packages: - self.build(path.join(job_config.workdir, package), - job_config.repo_uri, - job_config.repo_pass, - job_config.repo_user) + print(f"[PyPiRunner] Uploading {package}") + self.upload(job_config, package) + else: + print(f"[PyPiRunner] Upload disabled, skiping") diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index f69d9cf..268cbd8 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -4,7 +4,7 @@ import sys import shlex from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException -from alice.runners.pyutils import PackageManager +from alice.runners.pyutils import PackageManager, glob_command # TODO: Handle config like PyPiConfig @@ -46,32 +46,6 @@ class PythonRunner: if self.verbose: print("[PythonRunner] Installation done") - def __ghetto_glob(self, command, workdir): - if self.verbose: - print(f"[PythonRunner][Globbing] Starting command: {' '.join(command)}") - new_command = [] - for item in command: - if "*" in item: - if self.verbose: - print(f"[PythonRunner][Globbing] Found item: [{item}]") - dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item))) - base_name = os.path.basename(item) - if os.path.isdir(dir): - item_parts = base_name.split("*") - for file in os.listdir(dir): - # TODO: Fix ordering! A*B = B*A = AB* - if item_parts[0] in file and item_parts[1] in file: - new_item = os.path.join(dir, file) - if self.verbose: - print(f"[PythonRunner][Globbing] Substitute: {new_item}") - new_command.append(new_item) - else: - if self.verbose: - print(f"[PythonRunner][Globbing] Dir not exists: {dir}") - else: - new_command.append(item) - return new_command - # Executes the given job in the one and only venv # parameter shall be the raw jobscpec def run(self, job_spec): @@ -90,7 +64,7 @@ class PythonRunner: print(f"[PythonRunner] Raw command: {command}") # TODO: only split if command is not an array if "*" in command: - run_command = self.__ghetto_glob(shlex.split(command), pwd) + run_command = glob_command(shlex.split(command), pwd, self.verbose) else: run_command = shlex.split(command) if self.verbose: diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py index ca95641..62efdf7 100644 --- a/alice-ci/src/alice/runners/pyutils.py +++ b/alice-ci/src/alice/runners/pyutils.py @@ -1,3 +1,4 @@ +import os import subprocess import sys from pkg_resources import parse_version @@ -80,3 +81,35 @@ class PackageManager: else: return True return False + + +def glob(item, workdir, verbose=False): + new_command = [] + if "*" in item: + if verbose: + print(f"[Globbing] Found item: [{item}]") + dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item))) + base_name = os.path.basename(item) + if os.path.isdir(dir): + item_parts = base_name.split("*") + for file in os.listdir(dir): + # TODO: Fix ordering! A*B = B*A = AB* + if item_parts[0] in file and item_parts[1] in file: + new_item = os.path.join(dir, file) + if verbose: + print(f"[Globbing] Substitute: {new_item}") + new_command.append(new_item) + else: + raise ConfigException(f"[Globbing] Dir not exists: {dir}") + return new_command + else: + return [item] + + +def glob_command(command, workdir, verbose=False): + if verbose: + print(f"[Globbing] Starting command: {' '.join(command)}") + new_command = [] + for item in command: + new_command += glob(item, workdir, verbose) + return new_command diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml index b6aba40..a7f8e61 100644 --- a/ci-examples/full.yaml +++ b/ci-examples/full.yaml @@ -36,11 +36,13 @@ jobs: - name: pkg type: pypi workdir: . - upload: false + upload: true + fail_if_exists: false repo: uri: example.com - username: asdf + username: + from_env: PYPIUSER password: - from_env: COLORTERM + from_env: PYPIPASS packages: - alice-ci \ No newline at end of file