diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..7ab2f96 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,11 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu/.devcontainer/base.Dockerfile + +# [Choice] Ubuntu version (use hirsuite or bionic on local arm64/Apple Silicon): hirsute, focal, bionic +ARG VARIANT="hirsute" +FROM mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT} + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..6a078e8 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.3/containers/ubuntu +{ + "name": "Ubuntu", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick an Ubuntu version: hirsute, focal, bionic + // Use hirsute or bionic on local arm64/Apple Silicon. + "args": { "VARIANT": "focal" } + }, + + // Set *default* container specific settings.json values on container create. + "settings": {}, + + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "uname -a", + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode", + "features": { + "python": "latest" + } +} diff --git a/.drone.yml b/.drone.yml index 23346d0..7fe5a67 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,32 +1,32 @@ -kind: pipeline -type: docker -name: default - -steps: -- name: static-test - image: alpine/flake8 - commands: - - python3 -m flake8 --ignore E501,W503 - -- name: build - image: python - commands: - - python3 -m pip install build - - python3 -m build alice-ci - -- name: publish - image: python - environment: - TWINE_PASSWORD: - from_secret: pypi_username - TWINE_USERNAME: - from_secret: pypi_password - commands: - - python3 -m pip install twine - - python3 -m twine upload --verbose alice-ci/dist/* - when: - branch: - - master - event: - exclude: +kind: pipeline +type: docker +name: default + +steps: +- name: static-test + image: alpine/flake8 + commands: + - python3 -m flake8 --ignore E501,W503 + +- name: build + image: python + commands: + - python3 -m pip install build + - python3 -m build alice-ci + +- name: publish + image: python + environment: + TWINE_PASSWORD: + from_secret: pypi_username + TWINE_USERNAME: + from_secret: pypi_password + commands: + - python3 -m pip install twine + - python3 -m twine upload --verbose alice-ci/dist/* + when: + branch: + - master + event: + exclude: - pull_request \ No newline at end of file diff --git a/.gitignore b/.gitignore index f8b73e7..f4f8132 100644 --- a/.gitignore +++ b/.gitignore @@ -1,140 +1,140 @@ -# ---> Python -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + diff --git a/LICENSE b/LICENSE index c2f5393..b633e6c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,9 @@ -MIT License - -Copyright (c) 2022 Daniel Gyulai - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +MIT License + +Copyright (c) 2022 Daniel Gyulai + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index f485afa..5bfc5ca 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# alice - -CI framework with support for local running. - -Main repo [here](https://git.gyulai.cloud/gyulaid/alice). - -[![Build Status](https://ci.gyulai.cloud/api/badges/gyulaid/alice/status.svg)](https://ci.gyulai.cloud/gyulaid/alice) -[![PyPI version](https://badge.fury.io/py/alice-ci.svg)](https://badge.fury.io/py/alice-ci) - -* [Basic usage](alice-ci/README.md) -* [Runners](docs/runners.md) +# alice + +CI framework with support for local running. + +Main repo [here](https://git.gyulai.cloud/gyulaid/alice). + +[![Build Status](https://ci.gyulai.cloud/api/badges/gyulaid/alice/status.svg)](https://ci.gyulai.cloud/gyulaid/alice) +[![PyPI version](https://badge.fury.io/py/alice-ci.svg)](https://badge.fury.io/py/alice-ci) + +* [Basic usage](alice-ci/README.md) +* [Runners](docs/runners.md) * [CI syntax](docs/syntax.md) \ No newline at end of file diff --git a/alice-ci/README.md b/alice-ci/README.md index eb18524..121fa27 100644 --- a/alice-ci/README.md +++ b/alice-ci/README.md @@ -1,16 +1,16 @@ -# Alice-CI - -Continous Integration framework with the goal of using the exact same code in CI and local env. Steps can be defined in yaml files, for syntax see the docs. Runs on LInux and Windows, Mac should work too, but not yet tested. - -## Usage - -Install with pip: -``` -pythom3 -m pip install alice-ci -``` - -To run: - -``` -pythom3 -m alice [-i ] STEPS +# Alice-CI + +Continous Integration framework with the goal of using the exact same code in CI and local env. Steps can be defined in yaml files, for syntax see the docs. Runs on LInux and Windows, Mac should work too, but not yet tested. + +## Usage + +Install with pip: +``` +pythom3 -m pip install alice-ci +``` + +To run: + +``` +pythom3 -m alice [-i ] STEPS ``` \ No newline at end of file diff --git a/alice-ci/pyproject.toml b/alice-ci/pyproject.toml index b5a3c46..0ad39d0 100644 --- a/alice-ci/pyproject.toml +++ b/alice-ci/pyproject.toml @@ -1,6 +1,6 @@ -[build-system] -requires = [ - "setuptools>=42", - "wheel" -] +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/alice-ci/setup.cfg b/alice-ci/setup.cfg index d1e593d..8bd98b5 100644 --- a/alice-ci/setup.cfg +++ b/alice-ci/setup.cfg @@ -1,26 +1,30 @@ -[metadata] -name = alice-ci -version = 0.0.6 -author = Daniel Gyulai -description = Alice CI framework -long_description = file: README.md -long_description_content_type = text/markdown -url = https://git.gyulai.cloud/gyulaid/alice -project_urls = - Bug Tracker = https://git.gyulai.cloud/gyulaid/alice/issues -classifiers = - Programming Language :: Python :: 3 - License :: OSI Approved :: MIT License - Operating System :: OS Independent - -[options] -package_dir = - = src -packages = find: -python_requires = >=3.6 -install_requires = - PyYAML==6.0 - virtualenv==20.14.0 - -[options.packages.find] +[metadata] +name = alice-ci +version = 0.0.7 +author = Daniel Gyulai +description = Alice CI framework +long_description = file: README.md +long_description_content_type = text/markdown +url = https://git.gyulai.cloud/gyulaid/alice +project_urls = + Bug Tracker = https://git.gyulai.cloud/gyulaid/alice/issues +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Operating System :: OS Independent + +[options] +package_dir = + = src +packages = alice +python_requires = >=3.6 +install_requires = + PyYAML==6.0 + virtualenv==20.14.0 + +[options.entry_points] +console_scripts = + alice = alice.cli:main + +[options.packages.find] where = src \ No newline at end of file diff --git a/alice-ci/src/alice/__init__.py b/alice-ci/src/alice/__init__.py index 5423544..a442884 100644 --- a/alice-ci/src/alice/__init__.py +++ b/alice-ci/src/alice/__init__.py @@ -1,10 +1,10 @@ -# flake8: noqa F401 -from alice.utils import ConfigParser -from alice.exceptions import NonZeroRetcode -from alice.runnerfactory import Factory -from alice.runners.pythonrunner import PythonRunner -from alice.exceptions import NonZeroRetcode -from alice.exceptions import RunnerError -from alice.exceptions import ConfigException - +# flake8: noqa F401 +from alice.utils import ConfigParser +from alice.exceptions import NonZeroRetcode +from alice.runnerfactory import Factory +from alice.runners.pythonrunner import PythonRunner +from alice.exceptions import NonZeroRetcode +from alice.exceptions import RunnerError +from alice.exceptions import ConfigException + name = "alice" \ No newline at end of file diff --git a/alice-ci/src/alice/__main__.py b/alice-ci/src/alice/__main__.py index 41aa9b8..a4d3a54 100644 --- a/alice-ci/src/alice/__main__.py +++ b/alice-ci/src/alice/__main__.py @@ -1,3 +1,3 @@ -from alice.cli import main - -main() +from alice.cli import main + +main() diff --git a/alice-ci/src/alice/cli.py b/alice-ci/src/alice/cli.py index bd958c4..d80f2c7 100644 --- a/alice-ci/src/alice/cli.py +++ b/alice-ci/src/alice/cli.py @@ -1,63 +1,62 @@ -import os -import argparse - -from alice.utils import ConfigParser -from alice.runnerfactory import Factory -from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError - - -def gen_env(self, param_list): - env_vars = {} - for item in param_list: - item_parts = item.split("=") - if len(item_parts) == 2: - env_vars[item_parts[0]] = item_parts[1] - else: - raise ConfigException(f"Invalid parameter: {item}") - return env_vars - - -def parse_jobs(args): - try: - factory = Factory(args.verbose) - if len(args.env) > 0: - envs = gen_env(args.env) - if args.verbose: - print(f"[Alice] Env vars from CLI: {envs}") - factory.update_runners({"env": envs}) - jobParser = ConfigParser(args.input, factory, args.verbose) - - print("Begin pipeline steps...") - for step in args.steps: - if step in jobParser.jobs: - status = jobParser.execute_job(step) - print(f"[Step] {step}: {status}") - else: - print(f"Step {step} not found in {args.input}") - exit(1) - except ConfigException as e: - print(f"Configuration error-> {e}") - exit(1) - except NonZeroRetcode: - print("FAILED") - exit(1) - except RunnerError as e: - print(f"RunnerError-> {e}") - - -def main(): - parser = argparse.ArgumentParser(prog="alice") - parser.add_argument("steps", nargs='+') - parser.add_argument("-i", "--input", default="alice-ci.yaml") - parser.add_argument("-e", "--env", nargs='*', default=[]) - parser.add_argument("-a", "--addrunner", nargs='*', default=[]) - parser.add_argument("-v", "--verbose", action='store_true') - args = parser.parse_args() - if not os.path.isfile(args.input): - print(f"No such file: {args.input}") - exit(1) - parse_jobs(args) - - -if __name__ == "__main__": - main() +import os +import argparse + +from alice.utils import ConfigParser +from alice.runnerfactory import Factory +from alice.exceptions import ConfigException, NonZeroRetcode, RunnerError + + +def gen_env(param_list): + env_vars = {} + for item in param_list: + item_parts = item.split("=") + if len(item_parts) == 2: + env_vars[item_parts[0]] = item_parts[1] + else: + raise ConfigException(f"Invalid parameter: {item}") + return env_vars + + +def parse_jobs(args): + try: + factory = Factory(args.verbose) + if len(args.env) > 0: + envs = gen_env(args.env) + if args.verbose: + print(f"[Alice] Env vars from CLI: {envs}") + jobParser = ConfigParser(args.input, factory, gen_env(args.env), args.verbose) + + print("[Alice] Begin pipeline steps") + for step in args.steps: + if step in jobParser.jobs: + status = jobParser.execute_job(step) + print(f"[Alice][Step] {step}: {status}") + else: + raise ConfigException(f"Step {step} not found in {args.input}") + exit(1) + except ConfigException as e: + print(f"Configuration error-> {e}") + exit(1) + except NonZeroRetcode: + print("[Alice] FAILED") + exit(1) + except RunnerError as e: + print(f"RunnerError-> {e}") + + +def main(): + parser = argparse.ArgumentParser(prog="alice") + parser.add_argument("steps", nargs='+') + parser.add_argument("-i", "--input", default="alice-ci.yaml") + parser.add_argument("-e", "--env", nargs='*', default=[]) + parser.add_argument("-a", "--addrunner", nargs='*', default=[]) + parser.add_argument("-v", "--verbose", action='store_true') + args = parser.parse_args() + if not os.path.isfile(args.input): + print(f"No such file: {args.input}") + exit(1) + parse_jobs(args) + + +if __name__ == "__main__": + main() diff --git a/alice-ci/src/alice/exceptions.py b/alice-ci/src/alice/exceptions.py index a30fb70..8a52bc8 100644 --- a/alice-ci/src/alice/exceptions.py +++ b/alice-ci/src/alice/exceptions.py @@ -1,10 +1,10 @@ -class NonZeroRetcode(Exception): - pass - - -class RunnerError(Exception): - pass - - -class ConfigException(Exception): - pass +class NonZeroRetcode(Exception): + pass + + +class RunnerError(Exception): + pass + + +class ConfigException(Exception): + pass diff --git a/alice-ci/src/alice/runnerfactory.py b/alice-ci/src/alice/runnerfactory.py index f9012e5..6218362 100644 --- a/alice-ci/src/alice/runnerfactory.py +++ b/alice-ci/src/alice/runnerfactory.py @@ -1,49 +1,45 @@ -from os import getcwd - -from alice.runners.pythonrunner import PythonRunner -from alice.exceptions import ConfigException - - -class Factory(): - def __init__(self, verbose) -> None: - self.verbose = verbose - self.runnertypes = self.__load_runners() - self.runners = {} - self.workdir = getcwd() - self.globals = {} - - def __load_runners(self): - # TODO: Runners can be imported via cli too - # module = __import__("module_file") - # my_class = getattr(module, "class_name") - runners = {"python": PythonRunner} - - if (self.verbose): - print(f"[Alice] Available runners: {'|'.join(runners.keys())}") - return runners - - def set_globals(self, globals): - self.globals = globals - - def update_globals(self, update): - if "env" in update: - self.globals["env"].update(update["env"]) - - def update_runners(self, config): - for runnertype, runnerconfig in config.items(): - if runnertype != "global": - if (self.verbose): - print(f"[Alice] Configuring runner {runnertype}") - self.get_runner(runnertype).update_config(runnerconfig) - - def get_runner(self, runnertype): - if runnertype not in self.runners: - if runnertype in self.runnertypes: - if (self.verbose): - print(f"[Alice] Initializing runner: {runnertype}") - self.runners[runnertype] = self.runnertypes[runnertype](self.workdir, - self.globals, - self.verbose) - else: - raise ConfigException(f"Invalid runner type: {runnertype}") - return self.runners[runnertype] +from alice.runners.pythonrunner import PythonRunner +from alice.exceptions import ConfigException + + +class Factory(): + def __init__(self, verbose) -> None: + self.verbose = verbose + self.runnertypes = self.__load_runners() + self.runner_configs = {} + self.runners = {} + self.globals = {} + + def __load_runners(self): + # TODO: Runners can be imported via cli too + # https://git.gyulai.cloud/gyulaid/alice/issues/4 + # module = __import__("module_file") + # my_class = getattr(module, "class_name") + runners = {"python": PythonRunner} + + if (self.verbose): + print(f"[Alice] Available runners: {'|'.join(runners.keys())}") + return runners + + def set_globals(self, globals): + self.globals = globals + + def update_runners(self, config): + for runnertype, runnerconfig in config.items(): + if runnertype != "global": + if (self.verbose): + print(f"[Alice] Configuring runner: {runnertype}") + self.get_runner(runnertype).update_config(runnerconfig) + + def get_runner(self, runnertype): + if runnertype not in self.runners: + if runnertype in self.runnertypes: + if (self.verbose): + print(f"[Alice] Initializing runner: {runnertype}") + params = { + "verbose": self.verbose + } + self.runners[runnertype] = self.runnertypes[runnertype](params, self.globals) + else: + raise ConfigException(f"Invalid runner type: {runnertype}") + return self.runners[runnertype] diff --git a/alice-ci/src/alice/runners/__init__.py b/alice-ci/src/alice/runners/__init__.py index e6df5ea..973c468 100644 --- a/alice-ci/src/alice/runners/__init__.py +++ b/alice-ci/src/alice/runners/__init__.py @@ -1,3 +1,2 @@ -from alice.runners.pythonrunner import PythonRunner - -__all__ = ["PythonRunner"] +# flake8: noqa F401 +from alice.runners.pythonrunner import PythonRunner diff --git a/alice-ci/src/alice/runners/dockerrunner.py b/alice-ci/src/alice/runners/dockerrunner.py index b1dd741..50a3260 100644 --- a/alice-ci/src/alice/runners/dockerrunner.py +++ b/alice-ci/src/alice/runners/dockerrunner.py @@ -1 +1 @@ -# TODO Implement +# TODO Implement diff --git a/alice-ci/src/alice/runners/pythonrunner.py b/alice-ci/src/alice/runners/pythonrunner.py index 30367c5..4dbcd0b 100644 --- a/alice-ci/src/alice/runners/pythonrunner.py +++ b/alice-ci/src/alice/runners/pythonrunner.py @@ -1,116 +1,117 @@ -import subprocess -import os -import sys -import shlex -from tabnanny import verbose - -from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException - - -# same venv across all runs! -class PythonRunner(): - def __init__(self, workdir, defaults, verbose) -> None: - self.workdir = workdir - self.virtual_dir = os.path.abspath(os.path.join(workdir, "venv")) - self.config = defaults - self.env_vars = os.environ.copy() - for env_var in defaults["env"]: - self.env_vars[env_var["name"]] = env_var["value"] - self.verbose = verbose - - self.__init_venv() - - def __init_venv(self): - if os.name == "nt": # Windows - self.vpython = os.path.join(self.virtual_dir, "Scripts", "python.exe") - else: # Linux & Mac - self.vpython = os.path.join(self.virtual_dir, "bin", "python3") - - if not os.path.exists(self.vpython): - with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - p.wait() - if p.returncode != 0: - sys.stdout.buffer.write(p.stderr.read()) - raise RunnerError("[PythonRunner] Could not create virtualenv") - else: - if self.verbose: - print(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}") - else: - if self.verbose: - print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}") - - # Stores common defaults for all jobs - all types! - # Also - dependency install by config is only allowed in this step - def update_config(self, config): - if "dependencies" in config: - for dependency in config["dependencies"]: - # TODO: Check what happens with fixed version - command = [self.vpython, "-m", "pip", "install", dependency, "--upgrade"] - with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - p.wait() - if p.returncode != 0: - sys.stdout.buffer.write(p.stderr.read()) - raise(RunnerError(f"[PythonRunner] Could not install dependency: {dependency} ({p.returncode})")) - if "env" in config: - for env_var in config["env"]: - self.env_vars[env_var["name"]] = env_var["value"] - if "workdir" in config and config["workdir"] is not None: - self.workdir = os.path.join(self.workdir, config["workdir"]) - - def __ghetto_glob(self, command, workdir): - if self.verbose: - print(f"[PythonRunner][Globbing] Starting command: {' '.join(command)}") - new_command = [] - for item in command: - if "*" in item: - if self.verbose: - print(f"[PythonRunner][Globbing] Found item: [{item}]") - dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item))) - base_name = os.path.basename(item) - if os.path.isdir(dir): - item_parts = base_name.split("*") - for file in os.listdir(dir): - # TODO: Fix ordering! A*B = B*A = AB* - if item_parts[0] in file and item_parts[1] in file: - new_item = os.path.join(dir, file) - if self.verbose: - print(f"[PythonRunner][Globbing] Substitute: {new_item}") - new_command.append(new_item) - else: - if self.verbose: - print(f"[PythonRunner][Globbing] Dir not exists: {dir}") - else: - new_command.append(item) - return new_command - - # Executes the given job in the one and only venv - # parameter shall be the raw jobscpec - def run(self, job_spec): - if "workdir" in job_spec: - pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"])) - else: - pwd = self.workdir - run_env = self.env_vars.copy() - if "env" in job_spec: - for env_var in job_spec["env"]: - run_env[env_var["name"]] = env_var["value"] - if "commands" in job_spec: - commands = job_spec["commands"] - for command in commands: - if self.verbose: - print(f"[PythonRunner] Raw command: {command}") - # TODO: only split if command is not an array - run_command = self.__ghetto_glob(shlex.split(command), pwd) - if self.verbose: - print(f"[PythonRunner] Command to execute: {run_command}") - print(f"[PythonRunner] Workdir: {pwd}") - if os.path.isdir(pwd): - with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p: - p.wait() - if p.returncode != 0: - raise NonZeroRetcode(f"Command {command} returned code {p.returncode}") - else: - raise RunnerError(f"[PythonRunner] Invalid path for shell command: {pwd}") - else: - raise ConfigException(f"[PythonRunner] No commands specified in step {job_spec['name']}") +import subprocess +import os +import sys +import shlex + +from alice.exceptions import NonZeroRetcode, RunnerError, ConfigException + + +class PythonRunner(): + def __init__(self, params, user_defaults) -> None: + self.verbose = params["verbose"] + if self.verbose: + print("[PythonRunner] Initializing") + self.workdir = user_defaults["workdir"] + self.virtual_dir = os.path.abspath(os.path.join(self.workdir, "venv")) + self.config = user_defaults + + self.__init_venv() + + def __init_venv(self): + if os.name == "nt": # Windows + self.vpython = os.path.join(self.virtual_dir, "Scripts", "python.exe") + else: # Linux & Mac + self.vpython = os.path.join(self.virtual_dir, "bin", "python3") + + if not os.path.exists(self.vpython): + print("[PythonRunner] Initializing venv") + with subprocess.Popen([sys.executable, "-m", "virtualenv", self.virtual_dir], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + sys.stdout.buffer.write(p.stderr.read()) + raise RunnerError("[PythonRunner] Could not create virtualenv") + else: + if self.verbose: + print(f"[PythonRunner] Virtualenv initialized at {self.virtual_dir}") + else: + if self.verbose: + print(f"[PythonRunner] Found virtualenv at {self.virtual_dir}") + + # Stores common defaults for all jobs - all types! + # Also - dependency install by config is only allowed in this step + def update_config(self, config): + if "dependencies" in config: + for dependency in config["dependencies"]: + # TODO: Check what happens with fixed version + command = [self.vpython, "-m", "pip", "install", dependency, "--upgrade"] + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + if p.returncode != 0: + sys.stdout.buffer.write(p.stderr.read()) + raise(RunnerError(f"[PythonRunner] Could not install dependency: {dependency} ({p.returncode})")) + if "env" in config: + for env_var in config["env"]: + self.config["env"][env_var["name"]] = env_var["value"] + if "workdir" in config and config["workdir"] is not None: + self.workdir = os.path.join(self.workdir, config["workdir"]) + + def __ghetto_glob(self, command, workdir): + if self.verbose: + print(f"[PythonRunner][Globbing] Starting command: {' '.join(command)}") + new_command = [] + for item in command: + if "*" in item: + if self.verbose: + print(f"[PythonRunner][Globbing] Found item: [{item}]") + dir = os.path.abspath(os.path.join(workdir, os.path.dirname(item))) + base_name = os.path.basename(item) + if os.path.isdir(dir): + item_parts = base_name.split("*") + for file in os.listdir(dir): + # TODO: Fix ordering! A*B = B*A = AB* + if item_parts[0] in file and item_parts[1] in file: + new_item = os.path.join(dir, file) + if self.verbose: + print(f"[PythonRunner][Globbing] Substitute: {new_item}") + new_command.append(new_item) + else: + if self.verbose: + print(f"[PythonRunner][Globbing] Dir not exists: {dir}") + else: + new_command.append(item) + return new_command + + # Executes the given job in the one and only venv + # parameter shall be the raw jobscpec + def run(self, job_spec): + if "workdir" in job_spec: + pwd = os.path.abspath(os.path.join(self.workdir, job_spec["workdir"])) + else: + pwd = self.workdir + run_env = self.config["env"].copy() + if "env" in job_spec: + for env_var in job_spec["env"]: + run_env[env_var["name"]] = env_var["value"] + if "commands" in job_spec: + commands = job_spec["commands"] + for command in commands: + if self.verbose: + print(f"[PythonRunner] Raw command: {command}") + # TODO: only split if command is not an array + if "*" in command: + run_command = self.__ghetto_glob(shlex.split(command), pwd) + else: + run_command = shlex.split(command) + if self.verbose: + print(f"[PythonRunner] Command to execute: {run_command}") + print(f"[PythonRunner] Workdir: {pwd}") + if os.path.isdir(pwd): + with subprocess.Popen([self.vpython] + run_command, cwd=pwd, env=run_env) as p: + p.wait() + if p.returncode != 0: + raise NonZeroRetcode(f"Command {command} returned code {p.returncode}") + else: + raise RunnerError(f"[PythonRunner] Invalid path for shell command: {pwd}") + else: + raise ConfigException(f"[PythonRunner] No commands specified in step {job_spec['name']}") diff --git a/alice-ci/src/alice/utils.py b/alice-ci/src/alice/utils.py index af1e1c0..edaf61f 100644 --- a/alice-ci/src/alice/utils.py +++ b/alice-ci/src/alice/utils.py @@ -1,86 +1,88 @@ -import os -import subprocess -import yaml - -from alice.exceptions import ConfigException - - -class ConfigParser: - def __init__(self, file_path, factory, verbose=False) -> None: - self.verbose = verbose - with open(file_path) as f: - self.config = yaml.safe_load(f) - self.factory = factory - self.factory.set_globals(self.__gen_globals()) - if "runners" in self.config: - self.factory.update_runners(self.config["runners"]) - self.jobs = self.__get_jobs() - - # Initialize env, workdir if not present - def __gen_globals(self): - globals = { - "env": [], - "workdir": None - } - if "runners" in self.config: - if "global" in self.config["runners"]: - if "env" in self.config["runners"]["global"]: - globals["env"] = self.config["runners"]["global"]["env"] - if "workdir" in self.config["runners"]["global"]: - globals["workdir"] = self.config["runners"]["global"]["workdir"] - - if (self.verbose): - print(f"[Alice] Configured globals: {globals}") - return globals - - def __get_jobs(self): - if "jobs" in self.config: - jobs = {} - for job_spec in self.config["jobs"]: - name = job_spec["name"] - if name in jobs: - raise ConfigException(f"Job with name {name} already exists!") - - jobs[name] = job_spec - if (self.verbose): - print(f"[Alice] Parsed jobs: {', '.join(jobs.keys())}") - return jobs - else: - raise ConfigException("No jobs defined in config") - - def __is_changed(self, changes): - try: - target = changes["branch"] - paths = [] - for path in changes["paths"]: - paths.append(os.path.abspath(path)) - # TODO: Error handling - command = ["git", "diff", "--name-only", target] - with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: - p.wait() - for line in p.stdout: - change_path = os.path.abspath(line.decode("UTF-8").strip()) - for path in paths: - spec_path = os.path.abspath(path) - if change_path.startswith(spec_path): - if self.verbose: - print(f"[Alice] Modified file: {change_path}") - print(f"[Alice] Path match: {path}") - return True - except KeyError: - raise ConfigException(f"Invalid 'changes' config: {changes}") - return False - - def execute_job(self, job_name): - if job_name in self.jobs: - job_spec = self.jobs[job_name] - should_run = True - if "changes" in job_spec: - should_run = self.__is_changed(job_spec["changes"]) - if should_run: - runner = self.factory.get_runner(job_spec["type"]) - runner.run(job_spec) - return "SUCCESS" - else: - return "SKIP, no change detected" - +from os import getcwd, path, environ +import subprocess +import yaml + +from alice.exceptions import ConfigException + + +class ConfigParser: + def __init__(self, file_path, factory, cli_env_vars, verbose=False) -> None: + self.verbose = verbose + with open(file_path) as f: + self.config = yaml.safe_load(f) + self.factory = factory + self.factory.set_globals(self.__gen_globals(cli_env_vars)) + if "runners" in self.config: + self.factory.update_runners(self.config["runners"]) + self.jobs = self.__get_jobs() + + # Initialize env and workdir if not present in global + def __gen_globals(self, cli_vars): + env_vars = environ.copy() + env_vars.update(cli_vars) + globals = { + "env": env_vars, + "workdir": getcwd() + } + if "runners" in self.config: + if "global" in self.config["runners"]: + if "env" in self.config["runners"]["global"]: + for var in self.config["runners"]["global"]["env"]: + globals["env"][var["name"]] = var["value"] + if "workdir" in self.config["runners"]["global"]: + globals["workdir"] = self.config["runners"]["global"]["workdir"] + + if (self.verbose): + print(f"[Alice] Configured globals: {globals}") + return globals + + def __get_jobs(self): + if "jobs" in self.config: + jobs = {} + for job_spec in self.config["jobs"]: + name = job_spec["name"] + if name in jobs: + raise ConfigException(f"Job with name {name} already exists!") + + jobs[name] = job_spec + if (self.verbose): + print(f"[Alice] Parsed jobs: {', '.join(jobs.keys())}") + return jobs + else: + raise ConfigException("No jobs defined in config") + + def __is_changed(self, changes): + try: + target = changes["branch"] + paths = [] + for _path in changes["paths"]: + paths.append(path.abspath(_path)) + # TODO: Error handling + command = ["git", "diff", "--name-only", target] + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p: + p.wait() + for line in p.stdout: + change_path = path.abspath(line.decode("UTF-8").strip()) + for _path in paths: + spec_path = path.abspath(_path) + if change_path.startswith(spec_path): + if self.verbose: + print(f"[Alice] Modified file: {change_path}") + print(f"[Alice] Path match: {_path}") + return True + except KeyError: + raise ConfigException(f"Invalid 'changes' config: {changes}") + return False + + def execute_job(self, job_name): + if job_name in self.jobs: + job_spec = self.jobs[job_name] + should_run = True + if "changes" in job_spec: + should_run = self.__is_changed(job_spec["changes"]) + if should_run: + runner = self.factory.get_runner(job_spec["type"]) + runner.run(job_spec) + return "SUCCESS" + else: + return "SKIP, no change detected" diff --git a/ci-examples/full.yaml b/ci-examples/full.yaml index 3d2dfee..f5b9191 100644 --- a/ci-examples/full.yaml +++ b/ci-examples/full.yaml @@ -1,33 +1,33 @@ -runners: - global: - env: - - name: A - value: A - - name: B - value: B - - name: C - value: C - workdir: packages - python: - env: - - name: A - value: D - dependencies: - - flake8 - - build -jobs: - - name: env - type: python - changes: - branch: origin/master - paths: - - "docs" - env: - - name: B - value: E - commands: - - "-c \"import os; print(os.environ)\"" - - name: lint - workdir: alice-ci - commands: +runners: + global: + env: + - name: A + value: A + - name: B + value: B + - name: C + value: C + workdir: packages + python: + env: + - name: A + value: D + dependencies: + - flake8 + - build +jobs: + - name: env + type: python + changes: + branch: origin/master + paths: + - "docs" + env: + - name: B + value: E + commands: + - "-c \"import os; print(os.environ)\"" + - name: lint + workdir: alice-ci + commands: - "-m flake8 --ignore E501" \ No newline at end of file diff --git a/ci-examples/python1.yaml b/ci-examples/python1.yaml index b562f56..ec7ed68 100644 --- a/ci-examples/python1.yaml +++ b/ci-examples/python1.yaml @@ -1,18 +1,18 @@ -runners: - python: - dependencies: - - flake8 - - build - - twine -jobs: - - name: selfcheck - type: python - workdir: ci - commands: - - "-m flake8 --ignore E501 --exclude venv" - - - name: lint - type: python - workdir: alice-ci/src - commands: +runners: + python: + dependencies: + - flake8 + - build + - twine +jobs: + - name: selfcheck + type: python + workdir: ci + commands: + - "-m flake8 --ignore E501 --exclude venv" + + - name: lint + type: python + workdir: alice-ci/src + commands: - "-m flake8 --ignore E501" \ No newline at end of file diff --git a/docs/examples.md b/docs/examples.md index d181b42..34fc38d 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,24 +1,24 @@ -# alice-ci.yaml examples - -## Python lint - -Installes flake8 package in a virtual elvironment, then lints the contents of the packages directory in the current working dir. - -``` -runners: - python: - dependencies: - - name: flake8 -jobs: - - name: lint - type: python - workdir: packages - commands: - - "-m flake8" -``` - -To run this job: - -``` -pythom3 -m alice lint +# alice-ci.yaml examples + +## Python lint + +Installes flake8 package in a virtual elvironment, then lints the contents of the packages directory in the current working dir. + +``` +runners: + python: + dependencies: + - name: flake8 +jobs: + - name: lint + type: python + workdir: packages + commands: + - "-m flake8" +``` + +To run this job: + +``` +pythom3 -m alice lint ``` \ No newline at end of file diff --git a/docs/runners.md b/docs/runners.md index 08a4b8e..800fa16 100644 --- a/docs/runners.md +++ b/docs/runners.md @@ -1,8 +1,45 @@ -# Runners - -Runners are responsible to execute a list of commands in a set environment defined in the CI yaml file. - -## List of runners - -* Python - executes python commands in a virtual environment -* Docker - executes each job in a separate Docker container - unimplemented \ No newline at end of file +# Runners + +Runners are responsible to execute a list of commands in a set environment defined in the CI yaml file. + +## List of runners + +* Python - executes python commands in a virtual environment +* Docker - executes each job in a separate Docker container - unimplemented + +## Import schema + +What you need to do to make Alice recognise and import your custom Runners +TODO + +## Runner API + +Each runner has to support the following functions: + +### __init__(params, user_defaults) + +* params: dict of runtime variables for the program itself. +* user_defaults: raw data from the CI file's global dict, augmented with an "env" dict, which contains environment variables from the host sytem, the CLI params and the pipeline global config, and the "workdir" value, which is the absolute path of the directory that the runner shall recognize as the current working directory. + +#### Params + +Currently the only param used is the dict is "verbose", whichis a boolean. The intended purpose is to show debug output if set to True. + +#### Workdir +workdir can be assigned at CI yaml level as global +Order: + By default: os.cwd() + if overwritten in global + ------------------------------- Below this level is the runner's responsibility + if owerwritten in runner config + if overwritten in job + +Runner shall receive the current working directory, unless stated otherwise in global config + +### update_config(config) + +The function takes the raw data from the parsed yaml under runners.(runnername). Handling its own config is the sole responsibility of the runner. This function may be called at any point of the running lifecycle, so the runner has to support changing its own configuration. + +### run(job_spec) + +This function executes one job attributed ith the type of the runner called. As the only hard requirement for Alice is the "type" field in a job (or the optional "changes"), everything else is handled by the runner. \ No newline at end of file diff --git a/docs/syntax.md b/docs/syntax.md index cc05de8..18967f6 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -1,39 +1,39 @@ -# alice-ci.yaml - -This yaml file defines the job steps executed by Alice. The jobs are called by names for each passed parameter on CLI. For example the following command searches for a job called lint defined in the `alice-ci.yaml` file in the current working directory, then runs it. - -``` -pythom3 -m alice lint -``` - -[Example configs](examples.md) - -## runners - -Contains global configuration for various runners. Currently the only supported runner is `python`. - -### Python - -#### Dependencies - -List of dependencies installed in the virtual environment. Each dependency has a `name` and an `import_name`, as Alice checks the availability of each package by trying to import `import_name`, and if it fails, calls pip to install `name`. - -## jobs - -List of jobs. Each job has a mandatory name, type and a list of commands, optional parameter is workdir. - -### name - -Mandatory value, string. Has to be unique in the current file. - -### type - -Job type, selects the runner executing the commands. Currently the only supported type is `python`. - -### comands - -List of strings, each executed one by one from top to bottom in the current context. - -### workdir - -Optional, defines Working directory relative to PWD. The default working directory is the current directory. +# alice-ci.yaml + +This yaml file defines the job steps executed by Alice. The jobs are called by names for each passed parameter on CLI. For example the following command searches for a job called lint defined in the `alice-ci.yaml` file in the current working directory, then runs it. + +``` +pythom3 -m alice lint +``` + +[Example configs](examples.md) + +## runners + +Contains global configuration for various runners. Currently the only supported runner is `python`. + +### Python + +#### Dependencies + +List of dependencies installed in the virtual environment. Each dependency has a `name` and an `import_name`, as Alice checks the availability of each package by trying to import `import_name`, and if it fails, calls pip to install `name`. + +## jobs + +List of jobs. Each job has a mandatory name, type and a list of commands, optional parameter is workdir. + +### name + +Mandatory value, string. Has to be unique in the current file. + +### type + +Job type, selects the runner executing the commands. Currently the only supported type is `python`. + +### comands + +List of strings, each executed one by one from top to bottom in the current context. + +### workdir + +Optional, defines Working directory relative to PWD. The default working directory is the current directory.