From 2515b1e0f6e870c65e3bc426e273cc2e0dc6b3ca Mon Sep 17 00:00:00 2001 From: gyulaid Date: Sun, 24 Apr 2022 17:39:57 +0200 Subject: [PATCH] mInOr fIxEs aNd iMpRoVeMeNtS --- .gitignore | 2 + alice-ci/src/alice/cli.py | 5 +- alice-ci/src/alice/config.py | 20 +- alice-ci/src/alice/configparser.py | 4 +- alice-ci/src/alice/runnerfactory.py | 4 +- alice-ci/src/alice/runners/pypirepo.py | 120 +++++++++ alice-ci/src/alice/runners/pypirunner.py | 283 +++++++++++++++++---- alice-ci/src/alice/runners/pythonrunner.py | 42 ++- alice-ci/src/alice/runners/pyutils.py | 5 +- ci-examples/full.yaml | 5 + 10 files changed, 420 insertions(+), 70 deletions(-) create mode 100644 alice-ci/src/alice/runners/pypirepo.py diff --git a/.gitignore b/.gitignore index f4f8132..817529a 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,5 @@ dmypy.json # Cython debug symbols cython_debug/ +# persistency dir +.alice diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py index 5630055..8386b15 100644 --- a/alice-ci/src/alice/cli.py +++ b/alice-ci/src/alice/cli.py @@ -39,7 +39,7 @@ def parse_jobs(args): def main(): parser = argparse.ArgumentParser(prog="alice") parser.add_argument("steps", nargs='*', default=["default"]) - parser.add_argument("-i", "--input", default="alice-ci.yaml") + parser.add_argument("-i", "--input", default=".alice-ci.yaml") parser.add_argument("-e", "--env", nargs='*', default=[]) parser.add_argument("-a", "--addrunner", nargs='*', default=[]) parser.add_argument('--verbose', '-v', action='count', default=0) @@ -51,6 +51,9 @@ def main(): if not os.path.isfile(args.input): print(f"No such file: {args.input}") exit(1) + persistency_path = os.path.join(os.getcwd(), ".alice") + if not os.path.isdir(persistency_path): + os.mkdir(persistency_path) parse_jobs(args) diff --git a/alice-ci/src/alice/config.py b/alice-ci/src/alice/config.py index d0d473b..fb1bd6e 100644 --- a/alice-ci/src/alice/config.py +++ b/alice-ci/src/alice/config.py @@ -6,7 +6,7 @@ from .exceptions import ConfigException class ConfigHolder: __instance = None - file_name = ".alice" + file_name = os.path.join(os.getcwd(), ".alice", "vars") @staticmethod def getInstance(): @@ -25,7 +25,8 @@ class ConfigHolder: self.vars = {} if os.path.isfile(config): with open(config) as f: - for line in f: + for _line in f: + line = _line.strip() items = line.split("=") if len(items) > 1: self.vars[items[0]] = line.replace(f"{items[0]}=", "") @@ -33,6 +34,19 @@ class ConfigHolder: def get(self, key): try: - self.vars[key] + return self.vars[key] except KeyError: raise ConfigException(f"{key} not defined in .conf!") + + def set(self, key, value): + self.vars[key] = value + self.commit() + + + def soft_set(self, key, value): + self.vars[key] = value + + def commit(self): + with open(self.file_name, 'w') as f: + for k, v in self.vars.items(): + f.write(f"{k}={v if v is not None else ''}\n") diff --git a/alice-ci/src/alice/configparser.py b/alice-ci/src/alice/configparser.py index d65b092..b2ec886 100644 --- a/alice-ci/src/alice/configparser.py +++ b/alice-ci/src/alice/configparser.py @@ -81,8 +81,8 @@ class ConfigParser: def execute_pipeline(self, pipeline_name): if pipeline_name in self.pipelines: print(f"[Alice][Pipeline] {pipeline_name}: Start") - for job in self.pipelines[pipeline_name]: - self.execute_job(job) + for task in self.pipelines[pipeline_name]: + self.execute(task) print(f"[Alice][Pipeline] {pipeline_name}: Success") def execute_job(self, job_name): diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py index 424d213..3661d25 100644 --- a/alice-ci/src/alice/runnerfactory.py +++ b/alice-ci/src/alice/runnerfactory.py @@ -4,6 +4,7 @@ from os.path import join, abspath from .runners.pythonrunner import PythonRunner from .runners.pypirunner import PyPiRunner from .runners.dockerrunner import DockerRunner +from .runners.pypirepo import PypiRepoRunner from .exceptions import ConfigException @@ -23,7 +24,8 @@ class Factory(): # my_class = getattr(module, "class_name") self.runnertypes = {"python": PythonRunner, "pypi": PyPiRunner, - "docker": DockerRunner} + "docker": DockerRunner, + "pypirepo": PypiRepoRunner} logging.info(f"[Alice] Available runners: {'|'.join(self.runnertypes.keys())}") diff --git a/alice-ci/src/alice/runners/pypirepo.py b/alice-ci/src/alice/runners/pypirepo.py new file mode 100644 index 0000000..7c02814 --- /dev/null +++ b/alice-ci/src/alice/runners/pypirepo.py @@ -0,0 +1,120 @@ +import logging +from operator import truediv +from xml.dom import NoModificationAllowedErr +import docker +import time +from os.path import join, isdir +from os import getcwd, mkdir +import os + +from ..exceptions import NonZeroRetcode, RunnerError, ConfigException +from ..config import ConfigHolder + +pipconf = """[global] +index-url = URL +trusted-host = BASE + pypi.org +extra-index-url= http://pypi.org/simple""" + + +class RepoConfig: + def __init__(self, config={}) -> None: + self.port = config.get("port", 8888) + self.enabled = config.get("enabled", True) + self.container_name = config.get("container_name", "alice-pypiserver") + + def copy(self, job_config): + r = RepoConfig() + r.container_name = job_config.get("container_name", self.container_name) + r.enabled = job_config.get("enabled", self.enabled) + r.port = job_config.get("port", self.port) + return r + + +class PypiRepoRunner: + def __init__(self, config) -> None: + logging.info("[PythonRunner] Initializing") + self.config = RepoConfig(config) + self.client = docker.from_env() + self.user = "alice" + self.passwd = "alice" + self.htpasswd = 'alice:{SHA}UisnajVr3zkBPfq+os1D4UHsyeg=' + + def __is_running(self, name): + try: + self.client.containers.get(name) + return True + except docker.errors.NotFound: + return False + + def run(self, job_spec): + job_config = self.config.copy(job_spec) + running = self.__is_running(job_config.container_name) + print(f"[PyPiRepo] {job_config.container_name} running: {running}") + + persistency_dir = join(getcwd(), ".alice", "pypirepo") + if not isdir(persistency_dir): + mkdir(persistency_dir) + + package_dir = join(persistency_dir, "packages") + if not isdir(package_dir): + mkdir(package_dir) + + htpasswd_file = join(persistency_dir, ".htpasswd") + with open(htpasswd_file, 'w') as f: + f.write(self.htpasswd) + + docker_host_ip = None + for network in self.client.networks.list(): + if network.name == "bridge": + docker_host_ip = network.attrs["IPAM"]["Config"][0]["Gateway"] + if docker_host_ip is None: + raise RunnerError("Unable to determine Docker host IP") + + if job_config.enabled: + if not running: + c = self.client.containers.run( + name = job_config.container_name, + image="pypiserver/pypiserver:latest", + detach=True, + labels={"app": "alice"}, + command=["--overwrite", "-P", ".htpasswd", "packages"], + ports={"8080/tcp": job_config.port}, + volumes={ + package_dir: { + "bind": "/data/packages", + "mode": "rw" + }, + htpasswd_file:{ + "bind": "/data/.htpasswd", + "mode": "ro" + } + }, + restart_policy={ + "Name": "unless-stopped" + } + ) + c.reload() + print(f"[PyPiRepo] {job_config.container_name} : {c.status}") + cfgh = ConfigHolder.getInstance() + cfgh.soft_set("PYPI_USER", self.user) + cfgh.soft_set("PYPI_PASS", self.passwd) + cfgh.soft_set("PYPI_REPO", f"http://localhost:{job_config.port}") + cfgh.soft_set("DOCKER_PYPI_USER", self.user) + cfgh.soft_set("DOCKER_PYPI_PASS", self.passwd) + cfgh.soft_set("DOCKER_PYPI_REPO", f"http://{docker_host_ip}:{job_config.port}") + cfgh.commit() + + venv = join(os.getcwd(), "venv") + if os.path.isdir(venv): + netloc = f"localhost:{job_config.port}" + url = f"http://{self.user}:{self.passwd}@{netloc}" + conf = pipconf.replace("URL", url).replace("BASE", netloc) + + if os.name == "nt": # Windows + filename = join(venv, "pip.ini") + else: # Linux & Mac + filename = join(venv, "pip.conf") + with open(filename, 'w') as f: + f.write(conf) + print(f"[PyPiRepo] pip conf written to {filename}") diff --git a/alice-ci/src/alice/runners/pypirunner.py b/alice-ci/src/alice/runners/pypirunner.py index 63d0d04..75b9329 100644 --- a/alice-ci/src/alice/runners/pypirunner.py +++ b/alice-ci/src/alice/runners/pypirunner.py @@ -1,19 +1,62 @@ +from distutils.command.config import config +from distutils.log import debug import json import logging +from ntpath import join import os import re import subprocess import sys -from urllib import request, error from pkg_resources import parse_version +from requests import get +from requests.auth import HTTPBasicAuth from os import environ, path +from html.parser import HTMLParser from alice.runners.pyutils import PackageManager, glob, grab_from from alice.exceptions import ConfigException, RunnerError +import hashlib +from pathlib import Path + + +def md5_update_from_file(filename, hash): + assert Path(filename).is_file() + with open(str(filename), "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash.update(chunk) + return hash + + +def md5_file(filename): + return md5_update_from_file(filename, hashlib.md5()).hexdigest() + + +def md5_update_from_dir(directory, hash, exclude_dirs, exclude_extensions, exclude_dirs_wildcard): + assert Path(directory).is_dir() + for _path in os.listdir(directory): + path = os.path.join(directory, _path) + if os.path.isfile(path) : + hash.update(_path.encode()) + logging.debug(f"[PyPiRunner][Hash] File: {path}") + hash = md5_update_from_file(path, hash) + elif os.path.isdir(path): + skip = False + for name in exclude_dirs: + if name in os.path.basename(_path): + skip = True + if not skip: + hash = md5_update_from_dir(path, hash, exclude_dirs, exclude_extensions, exclude_dirs_wildcard) + return hash + + +def md5_dir(directory, exclude_dirs=[], exclude_extensions=[], exclude_dirs_wildcard=[]): + return md5_update_from_dir(directory, hashlib.sha1(), exclude_dirs, exclude_extensions, exclude_dirs_wildcard).hexdigest() def get_uri(config, default): url = config.get("repo", {}).get("uri", default) if url is not None: + if not isinstance(url, str): + url = grab_from(url) if not re.match('(?:http|ftp|https)://', url): url = f"https://{url}" return url @@ -41,6 +84,19 @@ def get_pass(config, default): return default +class SimpleRepoParser(HTMLParser): + def __init__(self): + HTMLParser.__init__(self) + self.packages = [] + + def handle_data(self, data): + re_groups = re.findall("(\d*\.\d*\.\d*)", data) + if len(re_groups) == 1: + file_version = re_groups[0] + if file_version not in self.packages: + self.packages.append(file_version) + + # Parses and stores the config from yaml class PypiConfig: def __init__(self, config={}) -> None: @@ -67,41 +123,106 @@ class PypiConfig: return p +# TODO: Refactor to something sensible, more flexible +class PackageMeta: + def __init__(self): + self.conf_dir = path.join(os.getcwd(), ".alice", "pypirunner") + self.metafile = path.join(self.conf_dir, "packagemeta.json") + if not path.isdir(self.conf_dir): + os.mkdir(self.conf_dir) + if path.isfile(self.metafile): + with open(self.metafile) as f: + self.metadata = json.load(f) + else: + self.metadata = {} + self.__save() + + def __save(self): + with open(self.metafile, 'w') as f: + json.dump(self.metadata, f) + + def get(self, package, key): + return self.metadata.get(package, {}).get(key, "") + + def set(self, package, key, value): + if package not in self.metadata: + self.metadata[package] = {} + self.metadata[package][key] = value + self.__save() + + # TODO: consider "--skip-existing" flag for twine class PyPiRunner(): def __init__(self, config) -> None: logging.info("[PyPiRunner] Initializing") self.workdir = config["workdir"] self.config = PypiConfig(config) + self.metadata = PackageMeta() - def __versions(self, repo, pkg_name): - if repo is not None: - url = f'{repo}/{pkg_name}/json' + def __versions(self, config, pkg_name): + repo = config.repo_uri + if repo is None: + repo = "https://pypi.python.org/pypi" + + if config.repo_pass is not None and config.repo_user is not None: + logging.info(f"[PyPiRunner][Versions] Set auth headers from config") + logging.debug(f"[PyPiRunner][Versions] Auth: {config.repo_user}:{config.repo_pass}") + auth = HTTPBasicAuth(config.repo_user, config.repo_pass) else: - url = f"https://pypi.python.org/pypi/{pkg_name}/json" - try: - releases = json.loads(request.urlopen(url).read())['releases'] - except error.URLError as e: - raise RunnerError(f"{url}: {e}") + logging.info(f"[PyPiRunner][Versions] No auth headers in config, skip") + logging.debug(f"[PyPiRunner][Versions] Auth: {config.repo_user}:{config.repo_pass}") + auth = None - return sorted(releases, key=parse_version, reverse=True) + try: + if repo.endswith("pypi"): + url = f'{repo}/{pkg_name}/json' + logging.info(f"[PyPiRunner][Versions] Trying JSON API at {url}") + response = get(url, auth=auth) + if response.status_code == 200: + releases = json.loads(response.text)["releases"] + return sorted(releases, key=parse_version, reverse=True) + else: + logging.info(f"[PyPiRunner][Versions] JSON failed: [{response.status_code}]") + logging.debug(response.text) + repo = f"{repo}/simple" + url = f"{repo}/{pkg_name}" + logging.info(f"[PyPiRunner][Versions] Trying Simple API at {url}") + response = get(url, auth=auth) + if response.status_code == 200: + parser = SimpleRepoParser() + parser.feed(response.text) + return sorted(parser.packages, key=parse_version, reverse=True) + if response.status_code == 404: + return [] + else: + logging.info(f"[PyPiRunner][Versions] Simple failed: [{response.status_code}]") + logging.debug(response.text) + raise Exception("Failed to fetch available versions") + + except Exception as e: + raise RunnerError(f"{url}: {e}") def build(self, config, package): + print(f"[PyPiRunner] Building {package}") pkg_path = path.join(config.workdir, package) if not path.isdir(pkg_path): raise ConfigException(f"Path does not exists: {pkg_path}") + PackageManager.getInstance().ensure("build") command = [sys.executable, "-m", "build", package] - with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - p.wait() - if p.returncode != 0: - print("STDOUT:") - sys.stdout.buffer.write(p.stdout.read()) - print("STDERR:") - sys.stdout.buffer.write(p.stderr.read()) - raise RunnerError(f"[PyPiRunner] Failed to build {package}") - - def find_unuploaded(self, repo, file_list, pkg_name): - versions = self.__versions(repo, pkg_name) + if logging.root.isEnabledFor(logging.DEBUG): + with subprocess.Popen(command, cwd=config.workdir) as p: + p.wait() + if p.returncode != 0: + raise RunnerError(f"[PyPiRunner] Failed to build {package}") + else: + with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + raise RunnerError(f"[PyPiRunner] Failed to build {package}") + print(f"[PyPiRunner] Package {package} built") + + def find_unuploaded(self, config, file_list, pkg_name): + versions = self.__versions(config, pkg_name) unuploaded = [] for file in file_list: # flake8: noqa W605 @@ -113,52 +234,112 @@ class PyPiRunner(): unuploaded.append(file) else: print(f"[PyPiRunner] File already uploaded: {os.path.basename(file)}") + print(f"[PyPiRunner] Packages to publish: {', '.join(unuploaded) if len(unuploaded) > 1 else 'None'}") return unuploaded + def upload_command(self, config, package, _command, to_upload): + unregistered = False + command = _command + to_upload + with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + for line in map(lambda x: x.decode('utf8').strip(), p.stderr): + if "405 Method Not Allowed" in line: + unregistered = True + if not unregistered: + print("STDOUT:") + sys.stdout.buffer.write(p.stdout.read()) + print("STDERR:") + sys.stdout.buffer.write(p.stderr.read()) + raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})") + if unregistered: + print("[PyPiRunner] Registering package") + register_command = [sys.executable, "-m", "twine", "register", "--verbose", "--non-interactive"] + if config.repo_uri is not None: + register_command.append("--repository-url") + register_command.append(config.repo_uri) + if config.repo_user is not None and config.repo_pass is not None: + register_command.append("-u") + register_command.append(config.repo_user) + register_command.append("-p") + register_command.append(config.repo_pass) + register_command.append(to_upload[0]) + with subprocess.Popen(register_command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + print("STDOUT:") + sys.stdout.buffer.write(p.stdout.read()) + print("STDERR:") + sys.stdout.buffer.write(p.stderr.read()) + raise RunnerError(f"[PyPiRunner] Failed to register {package} ({p.returncode})") + self.upload_command(config, package, _command, to_upload) - def upload(self, config, package): - command = [sys.executable, "-m", "twine", "upload", "--verbose"] + def upload(self, config, package, current_version): + print(f"[PyPiRunner] Uploading {package}") + PackageManager.getInstance().ensure("twine") + command = [sys.executable, "-m", "twine", "upload", "--verbose", "--non-interactive"] if config.repo_uri is not None: command.append("--repository-url") command.append(config.repo_uri) - if config.repo_user is not None: + if config.repo_user is not None and config.repo_pass is not None: command.append("-u") command.append(config.repo_user) - if config.repo_pass is not None: command.append("-p") command.append(config.repo_pass) + else: + raise RunnerError("[PyPiRunner] Can't upload without credentials!") dist_path = os.path.abspath(os.path.join(config.workdir, package, "dist")) - files = glob(os.path.join(dist_path, "*"), config.workdir) - for file in files: - print(f"[PyPiRunner] Found: {file}") + _files = glob(os.path.join(dist_path, "*"), config.workdir) + files = [] + for file in _files: + if current_version in os.path.basename(file): + files.append(file) + print(f"[PyPiRunner] Found: {file}") + else: + logging.info(f"[PyPiRunner] Dropped: {file} doesn't match current version: {current_version}") - to_upload = self.find_unuploaded(config.repo_uri, files, package) + to_upload = self.find_unuploaded(config, files, package) if len(to_upload) == 0: return - command += to_upload - with subprocess.Popen(command, cwd=config.workdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - p.wait() - if p.returncode != 0: - print("STDOUT:") - sys.stdout.buffer.write(p.stdout.read()) - print("STDERR:") - sys.stdout.buffer.write(p.stderr.read()) - raise RunnerError(f"[PyPiRunner] Failed to upload {package} ({p.returncode})") + #command += to_upload + self.upload_command(config, package, command, to_upload) + print(f"[PyPiRunner] Uploaded {package}") + + def package_version(self, config, package): + cfg_path = path.join(config.workdir, package, "setup.cfg") + with open(cfg_path) as f: + for line in f: + if line.startswith("version"): + re_groups = re.findall("(\d*\.\d*\.\d*)", line) + if len(re_groups) < 1: + raise RunnerError(f"Unable to determine version of package: |{line}|") + return re_groups[0] def run(self, job_spec): job_config = self.config.copy(job_spec) - - PackageManager.getInstance().ensure("build") + for package in job_config.packages: - print(f"[PyPiRunner] Building {package}") - self.build(job_config, package) - print(f"[PyPiRunner] Package {package} built") - - if job_config.upload: - PackageManager.getInstance().ensure("twine") - for package in job_config.packages: - print(f"[PyPiRunner] Uploading {package}") - self.upload(job_config, package) - else: - print(f"[PyPiRunner] Upload disabled, skiping") + pkg_dir = path.join(job_config.workdir, package) + pkg_hash = md5_dir(pkg_dir, exclude_dirs=["pycache", "pytest_cache", "build", "dist", "egg-info"]) + logging.debug(f"[PyPiRunner] {package} hash: {pkg_hash}") + pkg_version = self.package_version(job_config, package) + logging.debug(f"[PyPiRunner] {package} local version: {pkg_version}") + repo_versions = self.__versions(job_config, package) + logging.debug(f"[PyPiRunner] {package} remote version: {repo_versions}") + + if pkg_version not in repo_versions: + print(f"[PyPiRunner] {package} not found in repo") + self.build(job_config, package) + self.metadata.set(package, pkg_version, pkg_hash) + else: + if pkg_hash != self.metadata.get(package, pkg_version): + self.build(job_config, package) + self.metadata.set(package, pkg_version, pkg_hash) + else: + print(f"[PyPiRunner] {package} Unchanged since last build") + + if job_config.upload: + self.upload(job_config, package, pkg_version) + else: + print(f"[PyPiRunner] Upload disabled, skipping") diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index c2854bd..cc7e6bf 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -5,7 +5,7 @@ import sys import shlex from ..exceptions import NonZeroRetcode, RunnerError, ConfigException -from .pyutils import PackageManager, glob_command +from .pyutils import PackageManager, glob_command, grab_from # TODO: Handle config like PyPiConfig @@ -27,12 +27,14 @@ class PythonRunner: if not os.path.exists(self.vpython): logging.debug(f"[PythonRunner] Venv not found at {self.vpython}") logging.info("[PythonRunner] Initializing venv") + output = [] with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + stdout=subprocess.PIPE) as p: p.wait() + for line in p.stdout: + output.append(line.decode('utf8').strip()) if p.returncode != 0: - sys.stdout.buffer.write(p.stderr.read()) - sys.stdout.buffer.write(p.stdout.read()) + print("\n".join(output)) raise RunnerError("[PythonRunner] Could not create virtualenv") else: logging.info(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}") @@ -42,11 +44,21 @@ class PythonRunner: if len(dependencies) > 0: logging.info(f"[PythonRunner] Ensuring dependencies: {', '.join(dependencies)}") command = [self.vpython, "-m", "pip", "install"] + dependencies - with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - p.wait() - if p.returncode != 0: - sys.stdout.buffer.write(p.stderr.read()) - raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})")) + if logging.root.isEnabledFor(logging.DEBUG): + with subprocess.Popen(command) as p: + p.wait() + if p.returncode != 0: + raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})")) + else: + output = [] + with subprocess.Popen(command, stdout=subprocess.PIPE) as p: + for line in p.stdout: + output.append(line.decode('utf8').strip()) + p.wait() + if p.returncode != 0: + #sys.stdout.buffer.write(p.stderr.read()) + print("\n".join(output)) + raise(RunnerError(f"[PythonRunner] Could not install dependencies: {dependencies} ({p.returncode})")) logging.info("[PythonRunner] Installation done") # Executes the given job in the one and only venv @@ -56,10 +68,18 @@ class PythonRunner: pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"])) else: pwd = self.workdir - run_env = self.config["env"].copy() + run_env = {} + for k, v in self.config["env"].items(): + if isinstance(v, str): + run_env[k] = v + else: + run_env[k] = grab_from(v) if "env" in job_spec: for env_var in job_spec["env"]: - run_env[env_var["name"]] = env_var["value"] + if isinstance(job_spec["env"], str): + run_env[env_var["name"]] = env_var["value"] + else: + run_env[env_var["name"]] = grab_from(env_var["value"]) if "commands" in job_spec: commands = job_spec["commands"] for command in commands: diff --git a/alice-ci/src/alice/runners/pyutils.py b/alice-ci/src/alice/runners/pyutils.py index d8bb685..1dff2fd 100644 --- a/alice-ci/src/alice/runners/pyutils.py +++ b/alice-ci/src/alice/runners/pyutils.py @@ -125,7 +125,10 @@ def grab_from(target): except KeyError: raise ConfigException(f"Env var unset: {target['from_env']}") elif "from_cfg" in target: - ConfigHolder.getInstance().get(target["from_cfg"]) + value = ConfigHolder.getInstance().get(target["from_cfg"]) + if len(value) == 0: + value = None + return value else: raise ConfigException(f"Unsupported grabber: {target.keys()}") diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml index 0e5714f..c06827c 100644 --- a/ci-examples/full.yaml +++ b/ci-examples/full.yaml @@ -84,6 +84,11 @@ jobs: credentials: username: B password: B + - name: pypi_init + type: pypirepo + enabled: true + port: 8888 + container_name: pypiserver pipelines: default: