From 4854bb240fc08954445ce7ae5d18ed3770e97593 Mon Sep 17 00:00:00 2001 From: mrjk Date: Sat, 15 Jan 2022 17:44:02 -0500 Subject: [PATCH] Update: schema management across plugins, trailing spaces cleanup --- albero/app.py | 26 +++---- albero/cli.py | 2 +- albero/managers.py | 103 +++++++++++++-------------- albero/plugin/backend/loop.py | 115 ++++++++++++++++++++++++------- albero/plugin/common.py | 18 ++--- albero/plugin/strategy/schema.py | 34 ++++----- albero/utils.py | 52 +++++++------- pyproject.toml | 1 + 8 files changed, 212 insertions(+), 139 deletions(-) diff --git a/albero/app.py b/albero/app.py index 366d82d..d0dbc0c 100755 --- a/albero/app.py +++ b/albero/app.py @@ -22,15 +22,15 @@ log = logging.getLogger(__name__) class App(): schema = { - "$schema": 'http://json-schema.org/draft-04/schema#', + "$schema": 'http://json-schema.org/draft-07/schema#', "type": "object", "additionalProperties": False, "default": {}, "$def" :{ - 'backends_items': None, - 'backends_config': None, - 'rules_items': None, - 'rules_config': None, + 'backends_items': {}, + 'backends_config': {}, + 'rules_items': {}, + 'rules_config': {}, }, "patternProperties": { ".*": { @@ -65,17 +65,20 @@ class App(): "default": {}, }, }, - + }, "tree": { "type": "array", "default": [], - "arrayItem": { "$ref": "#/$defs/backends_items" }, + "items": { + "type": "object", + "properties": { "$ref": "#/$defs/backends_items" }, + }, }, "rules": { "type": "array", "default": [], - "arrayItem": { "$ref": "#/$defs/rules_items" }, + # "arrayItem": { "$ref": "#/$defs/rules_items" }, }, }, }, @@ -122,12 +125,9 @@ class App(): r2 = RulesManager.get_schema(AlberoPlugins) d = self.schema - d['$def']['backends_items'] = r1 - d['$def']['rules_items'] = r2 - d['$def']['backends_config'] = None - d['$def']['rules_config'] = None + d['patternProperties']['.*']['properties'] ['tree']['items']['properties'] = r1 + d['patternProperties']['.*']['properties'] ['tree']['items'] = r2 - #pprint(d) print(json.dumps(d, indent=2)) diff --git a/albero/cli.py b/albero/cli.py index 2ed8cb5..037ea2e 100644 --- a/albero/cli.py +++ b/albero/cli.py @@ -153,7 +153,7 @@ class CmdApp: config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml' # self.log.debug(f"Command line vars: {vars(self.args)}") - keys = self.args.key or [None] + keys = self.args.key or [None] # Parse payload from enf file: new_params = {} diff --git a/albero/managers.py b/albero/managers.py index fbbf8d7..c714188 100644 --- a/albero/managers.py +++ b/albero/managers.py @@ -1,4 +1,6 @@ +import dpath.util + import copy import json import textwrap @@ -47,37 +49,32 @@ class LoadPlugin(): class Manager(): plugins_kind = [] - - _schema_props_default = {} - _schema_props_default = {} + _schema_props_default = None @classmethod def get_schema(cls, plugins_db): - pprint (cls.plugins_kind) - - ret = {} - ret3 = [] + # Properties + ret3 = {} for kind in cls.plugins_kind: - ret[kind] = {} + # ret[kind] = {} plugin_kind = getattr(plugins_db, kind) for plugin_name in [i for i in dir(plugin_kind) if not i.startswith('_')]: plugin = getattr(plugin_kind, plugin_name) - print (plugin.Plugin) - #pprint (dir(plugin)) plugin_cls = getattr(plugin, 'Plugin', None) if plugin_cls: schema_props = getattr(plugin_cls, '_schema_props_new', 'MISSING ITEM') if schema_props: - ret[kind][plugin_name] = schema_props - print (plugin_name) - ret3.append( schema_props ) - - ret3.append( cls._schema_props_new ) + # ret[kind][plugin_name] = schema_props + ret3.update( schema_props ) + ret3.update( cls._schema_props_new ) + # Injection ret1 = cls._schema_props_default - ret1["$def"]["items"] = ret3 + position = cls._props_position + dpath.util.set(ret1, position, ret3) + return ret1 @@ -86,37 +83,42 @@ class BackendsManager(Manager): plugins_kind = ['engine', 'backend'] _schema_props_new = { - "engine": { - "type": "string", - "default": "jerakia", - "optional": False, - }, - "value": { + "engine": { + "type": "string", + "default": "jerakia", + "optional": False, + }, + "value": { "default": 'UNSET', - "optional": False, - }, - #### INSERT HERE SUBSCHEMA !!!!! + "optional": False, + }, } - _schema_props_default = { - "$schema": 'http://json-schema.org/draft-04/schema#', + _props_position = 'oneOf/0/properties' + _schema_props_default = { + "$schema": 'http://json-schema.org/draft-07/schema#', "default": "", - "$def": { - "items": {}, - }, + # This does not work :( + #"$def": { + # "props": {}, + # }, "oneOf": [ { - "type": "string", - "default": "BLAAAAHHH" + "type": "object", + "additionalProperties": True, + "default": {}, + "title": "object", + "properties": {}, + "description": "Object to configure a bacjend item", }, { - "type": "object", - "additionalProperties": True, - "default": {}, - "properties": { "$ref": "#/$defs/name" }, + "type": "string", + "default": "BLAAAAHHH", + "title": "string", + "description": "Enter a simple string configuration value for default engine", }, ] - } + } def _validate_item(self, item): if isinstance(item, str): @@ -251,28 +253,29 @@ class RulesManager(Manager): }, } - _schema_props_default = { - "$schema": 'http://json-schema.org/draft-04/schema#', + _props_position = 'oneOf/1/properties' + _schema_props_default = { + "$schema": 'http://json-schema.org/draft-07/schema#', "default": "", "$def": { "items": {}, }, "oneOf": [ { - "type": "string", + "type": "string", "default": "BLAAAAHHH" }, { - "type": "object", - "additionalProperties": True, + "type": "object", + "additionalProperties": True, "default": {}, - "properties": { "$ref": "#/$defs/name" }, + "properties": { "$ref": "#/$defs/name" }, }, ] - } + } OLD_rule_schema = { - "$schema": 'http://json-schema.org/draft-04/schema#', + "$schema": 'http://json-schema.org/draft-07/schema#', "type": "object", "additionalProperties": False, "properties": { @@ -285,13 +288,13 @@ class RulesManager(Manager): "type": "null", }, { - "type": "string", + "type": "string", }, { - "type": "object", - "additionalProperties": True, + "type": "object", + "additionalProperties": True, "default": {}, - "properties": { "$ref": "#/$defs/name" }, + "properties": { "$ref": "#/$defs/name" }, }, ], }, @@ -331,7 +334,7 @@ class RulesManager(Manager): else: matched_rule = rule[0] log.debug(f"Matcher rule for {key}: {matched_rule}") - + matched_rule['trace'] = trace matched_rule['explain'] = explain schema_validate(matched_rule, self._schema_props_default) diff --git a/albero/plugin/backend/loop.py b/albero/plugin/backend/loop.py index 2d0c0d9..760e26a 100644 --- a/albero/plugin/backend/loop.py +++ b/albero/plugin/backend/loop.py @@ -14,41 +14,110 @@ import textwrap class Plugin(PluginBackendClass): _plugin_name = "loop" + _plugin_help = """ + This module helps to loop over a backend + """, _schema_props_new = { "loop": { + "description": _plugin_help, "default": None, "optional": True, + "examples": [ + { + "value": "site/{{ loop_env }}/config/{{ os }}", + "loop": { + "var": "loop_env", + "data": [ + "dev", + "preprod", + "prod", + ], + }, + "comment": "The module will loop three time over the value, and the variable `loop_env` will consecutely have `dev`, `preprod` and `prod` as value", + }, + { + "value": "site/{{ loop_env2 }}/config/{{ os }}", + "loop": { + "var": "loop_env2", + "data": "my_scope_var", + }, + "comment": "Like the previous example, but it will fetch the list from any scope variables", + }, + { + "loop": None, + "comment": "Disable this module, no loop will operate", + }, + + + # "loop": { + # "var": "my_var", + # }, + # }, + # "loop": { + # "var": "my_var", + # }, + # "example": "", + # }, + # "loop": { + # "var": "my_var", + # }, + # "example": "", + # }, + ], "oneOf": [ { - "type": "null", - }, - { - "type": "string", - }, - { - "type": "object", - "additionalProperties": True, + "type": "object", + "additionalProperties": False, "default": {}, - "properties": { - "data": { + "title": "Complete config", + "description": "", + + "properties": { + "data": { "default": None, - "optional": False, + "optional": False, + "title": "Module configuration", + "description": "Data list used for iterations. It only accept lists as type. It disable the module if set to `null`.", "anyOf":[ - {"type": "null"}, - {"type": "string"}, - {"type": "array"}, + { + "type": "null", + "title": "Disable Module", + "description": "Disable the module", + }, + { + "type": "string", + "title": "Scope variable", + "description": "Will look the value of the loop list from the scope. TOFIX: What if variablle does not exists?", + }, + { + "type": "array", + "title": "Hardcoded list", + "description": "Simply enter the list of value to be iterated to.", + }, ] - }, - "var": { - "type": "string", - "default": "loop_item", - "optional": True, - }, - }, + }, + "var": { + "type": "string", + "default": "loop_item", + "optional": True, + "title": "Module configuration", + "description": "Name of the variable to be used in templating language", + }, + }, + }, + { + "type": "string", + "title": "Short config", + "description": "If set to string, it will define the name of the variable to lookup into the scope.", + }, + { + "type": "null", + "title": "Disable", + "description": "If set to null, it disable the module", }, ] } - } + } def process(self, backends: list, ctx: dict) -> (list, dict): @@ -84,7 +153,7 @@ class Plugin(PluginBackendClass): _cand['_run']['scope'][loop_var] = item #_cand.scope[loop_var] = item ret.append(_cand) - + new_backends.extend(ret) return new_backends, ctx diff --git a/albero/plugin/common.py b/albero/plugin/common.py index 1d9ec9f..e44f53d 100644 --- a/albero/plugin/common.py +++ b/albero/plugin/common.py @@ -16,10 +16,10 @@ import copy # Candidate Classes # ============================= -class Candidate(): - engine = None - found = False - data = None +class Candidate(): + engine = None + found = False + data = None run = None scope = None key = None @@ -27,9 +27,9 @@ class Candidate(): def __init__(self, run): self.run = copy.deepcopy(run) - def __repr__(self): - return f"{self.__dict__}" - + def __repr__(self): + return f"{self.__dict__}" + def _report_data(self, data=None): default_data = { #"rule": self.config, @@ -149,7 +149,7 @@ class PluginEngineClass(PluginClass): schema = getattr(self, key) props.update(schema) self.schema = { - "$schema": 'http://json-schema.org/draft-04/schema#', + "$schema": 'http://json-schema.org/draft-07/schema#', "type": "object", "additionalProperties": True, "properties": props, @@ -205,7 +205,7 @@ class PluginFileGlob(): } def _glob(self, item): - + # DIRECT CALL TO APP< TOFIX app_config = self.app.conf2 root = app_config.get("default", {}).get("config", {}).get("root", f"{Path.cwd()}/tree") diff --git a/albero/plugin/strategy/schema.py b/albero/plugin/strategy/schema.py index 6d07942..4cfe9b4 100644 --- a/albero/plugin/strategy/schema.py +++ b/albero/plugin/strategy/schema.py @@ -20,41 +20,41 @@ class Plugin(PluginStrategyClass): "optional": True, "oneOf": [ { - "type": "null", + "type": "null", }, { - "type": "string", + "type": "string", }, { - "type": "array", + "type": "array", }, { - "type": "object", - "additionalProperties": True, + "type": "object", + "additionalProperties": True, "default": {}, - "properties": { - "data": { + "properties": { + "data": { "default": None, - "optional": False, + "optional": False, "anyOf":[ {"type": "null"}, {"type": "string"}, {"type": "array"}, ] - }, - "var": { - "type": "string", - "default": "loop_item", - "optional": True, - }, - }, + }, + "var": { + "type": "string", + "default": "loop_item", + "optional": True, + }, + }, }, ] } - } + } default_merge_schema = { - "$schema": 'http://json-schema.org/draft-04/schema#', + "$schema": 'http://json-schema.org/draft-07/schema#', "oneOf": [ { "type": "array", diff --git a/albero/utils.py b/albero/utils.py index d7a7280..ae88ae2 100644 --- a/albero/utils.py +++ b/albero/utils.py @@ -15,46 +15,46 @@ log = logging.getLogger(__name__) # # File parsers # # ===================== -# -# class FileParserClass(): -# -# def __init__(self, path): -# self.path = path -# +# +# class FileParserClass(): +# +# def __init__(self, path): +# self.path = path +# # def from_file(self, file): # raise Exception ("Not implemented") -# +# # def from_string(self, data): # raise Exception ("Not implemented") -# +# # def from_dict(self, data): # raise Exception ("Not implemented") -# -# class FilesYAMLParser(FileParserClass): -# def get_data(self): -# with open(self.path, "r") as stream: -# try: -# return yaml.safe_load(stream) -# except yaml.YAMLError as exc: -# raise Exception(exc) -# print(exc) -# -# -# class FilesJSONParser(FileParserClass): +# +# class FilesYAMLParser(FileParserClass): +# def get_data(self): +# with open(self.path, "r") as stream: +# try: +# return yaml.safe_load(stream) +# except yaml.YAMLError as exc: +# raise Exception(exc) +# print(exc) +# +# +# class FilesJSONParser(FileParserClass): # pass -# class FilesRawParser(FileParserClass): +# class FilesRawParser(FileParserClass): # pass -# class FilesTOMLParser(FileParserClass): +# class FilesTOMLParser(FileParserClass): # pass -# class FilesCSVParser(FileParserClass): +# class FilesCSVParser(FileParserClass): # pass -# class FilesINIParser(FileParserClass): +# class FilesINIParser(FileParserClass): # pass -# +# # format_db = { # ".raw": FilesRawParser, # ".yml": FilesYAMLParser, -# ".yaml": FilesYAMLParser, +# ".yaml": FilesYAMLParser, # ".json": FilesJSONParser, # } diff --git a/pyproject.toml b/pyproject.toml index efa0b17..ee08507 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ jsonmerge = "^1.8.0" anyconfig = "^0.12.0" python-box = "^5.4.1" prettytable = "^3.0.0" +dpath = "^2.0.5" [tool.poetry.dev-dependencies] json-schema-for-humans = "^0.40"