diff --git a/albero/app.py b/albero/app.py index d0dbc0c..a36dcc4 100755 --- a/albero/app.py +++ b/albero/app.py @@ -12,26 +12,28 @@ from pprint import pprint from albero.query import Query from albero.utils import schema_validate import anyconfig + # from box import Box from pathlib import Path import logging + log = logging.getLogger(__name__) -class App(): +class App: schema = { - "$schema": 'http://json-schema.org/draft-07/schema#', + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": False, "default": {}, - "$def" :{ - 'backends_items': {}, - 'backends_config': {}, - 'rules_items': {}, - 'rules_config': {}, - }, + "$def": { + "backends_items": {}, + "backends_config": {}, + "rules_items": {}, + "rules_config": {}, + }, "patternProperties": { ".*": { "type": "object", @@ -39,53 +41,51 @@ class App(): "additionalProperties": False, "properties": { "config": { - "type": "object", - "default": {}, - "additionalProperties": False, - "properties": { - "app": { - "type": "object", - "default": {}, - "additionalProperties": False, - "properties": { - "root": { - "type": "string", - "default": None, - }, - }, - }, - - "tree": { - #"additionalProperties": False, - "type": "object", - "default": {}, - }, - "rules": { - "type": "object", - "default": {}, - }, - }, - - }, - "tree": { - "type": "array", - "default": [], - "items": { + "type": "object", + "default": {}, + "additionalProperties": False, + "properties": { + "app": { "type": "object", - "properties": { "$ref": "#/$defs/backends_items" }, + "default": {}, + "additionalProperties": False, + "properties": { + "root": { + "type": "string", + "default": None, + }, }, - }, - "rules": { - "type": "array", - "default": [], - # "arrayItem": { "$ref": "#/$defs/rules_items" }, + }, + "tree": { + # "additionalProperties": False, + "type": "object", + "default": {}, + }, + "rules": { + "type": "object", + "default": {}, + }, }, }, + "tree": { + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": {"$ref": "#/$defs/backends_items"}, + }, + }, + "rules": { + "type": "array", + "default": [], + # "arrayItem": { "$ref": "#/$defs/rules_items" }, + }, }, - } - } + }, + }, + } - def __init__(self, config="albero.yml", namespace='default'): + def __init__(self, config="albero.yml", namespace="default"): conf2 = anyconfig.load(config) # Validate configuration @@ -93,27 +93,26 @@ class App(): try: conf2 = conf2[namespace] except KeyError: - log.error (f"Can't find namespace '{namespace}' in config '{config}'") + log.error(f"Can't find namespace '{namespace}' in config '{config}'") sys.exit(1) # Init - if not conf2['config']['app']['root']: - conf2['config']['app']['root'] = Path(config).parent + if not conf2["config"]["app"]["root"]: + conf2["config"]["app"]["root"] = Path(config).parent else: - conf2['config']['app']['root'] = Path(conf2['config']['app']['root']) + conf2["config"]["app"]["root"] = Path(conf2["config"]["app"]["root"]) # Finish self.conf2 = dict(conf2) def lookup(self, key=None, policy=None, scope=None, trace=False, explain=False): log.debug(f"Lookup key {key} with scope: {scope}") - q = Query(app = self) - r = q.exec(key=key, scope=scope , policy=policy, trace=trace, explain=explain) - - print ("=== Query Result ===") - print(anyconfig.dumps(r, ac_parser='yaml')) - print ("=== Query Result ===") + q = Query(app=self) + r = q.exec(key=key, scope=scope, policy=policy, trace=trace, explain=explain) + print("=== Query Result ===") + print(anyconfig.dumps(r, ac_parser="yaml")) + print("=== Query Result ===") def dump_schema(self): @@ -125,10 +124,7 @@ class App(): r2 = RulesManager.get_schema(AlberoPlugins) d = self.schema - d['patternProperties']['.*']['properties'] ['tree']['items']['properties'] = r1 - d['patternProperties']['.*']['properties'] ['tree']['items'] = r2 + d["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = r1 + d["patternProperties"][".*"]["properties"]["tree"]["items"] = r2 print(json.dumps(d, indent=2)) - - - diff --git a/albero/cli.py b/albero/cli.py index 037ea2e..4098b4c 100644 --- a/albero/cli.py +++ b/albero/cli.py @@ -18,6 +18,7 @@ sys.path.append("/home/jez/prj/bell/training/tiger-ansible/ext/ansible-tree") import albero.app as Albero + class CmdApp: """Main CmdApp""" @@ -99,7 +100,9 @@ class CmdApp: """Prepare command line""" # Manage main parser - parser = argparse.ArgumentParser(description="Albero, to lookup hierarchical data") + parser = argparse.ArgumentParser( + description="Albero, to lookup hierarchical data" + ) parser.add_argument( "-v", "--verbose", action="count", default=0, help="Increase verbosity" ) @@ -112,13 +115,19 @@ class CmdApp: # Manage command: demo add_p = subparsers.add_parser("lookup") - add_p.add_argument("-n", "--namespace", help="Namespace name", default='default') - add_p.add_argument("-f", "--file", help="File with params as dict. Can be stdin - .") - add_p.add_argument("-e", "--scope", dest="scope_param", action="append", default=[]) + add_p.add_argument( + "-n", "--namespace", help="Namespace name", default="default" + ) + add_p.add_argument( + "-f", "--file", help="File with params as dict. Can be stdin - ." + ) + add_p.add_argument( + "-e", "--scope", dest="scope_param", action="append", default=[] + ) add_p.add_argument("-p", "--policy") add_p.add_argument("-t", "--trace", action="store_true") add_p.add_argument("-x", "--explain", action="store_true") - add_p.add_argument("key", default=None, nargs="*") + add_p.add_argument("key", default=None, nargs="*") # Manage command: demo add_p = subparsers.add_parser("demo") @@ -150,9 +159,9 @@ class CmdApp: def cli_lookup(self): """Display how to use logging""" - config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml' + config = "/home/jez/prj/bell/training/tiger-ansible/tree.yml" -# self.log.debug(f"Command line vars: {vars(self.args)}") + # self.log.debug(f"Command line vars: {vars(self.args)}") keys = self.args.key or [None] # Parse payload from enf file: @@ -162,7 +171,7 @@ class CmdApp: # Parse cli params for i in self.args.scope_param: - r = i.split('=') + r = i.split("=") if len(r) != 2: raise Exception("Malformed params") new_params[r[0]] = r[1] @@ -171,18 +180,19 @@ class CmdApp: app = Albero.App(config=config, namespace=self.args.namespace) for key in keys: - app.lookup(key=key, - scope=new_params, - trace=self.args.trace, - explain=self.args.explain - ) + app.lookup( + key=key, + scope=new_params, + trace=self.args.trace, + explain=self.args.explain, + ) def cli_schema(self): """Display configuration schema""" - config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml' + config = "/home/jez/prj/bell/training/tiger-ansible/tree.yml" - app = Albero.App(config=config) #, namespace=self.args.namespace) + app = Albero.App(config=config) # , namespace=self.args.namespace) app.dump_schema() diff --git a/albero/managers.py b/albero/managers.py index c714188..804f2bf 100644 --- a/albero/managers.py +++ b/albero/managers.py @@ -1,33 +1,22 @@ - import dpath.util -import copy -import json -import textwrap -from prettytable import PrettyTable -from pathlib import Path -# from box import Box -from jsonmerge import Merger -import re import logging from pprint import pprint -import collections -from albero.utils import schema_validate, str_ellipsis +from albero.utils import schema_validate import albero.plugin as AlberoPlugins log = logging.getLogger(__name__) -class LoadPlugin(): - +class LoadPlugin: def __init__(self, plugins): self.plugins = plugins def load(self, kind, name): - assert (isinstance(name, str)), f"Got: {name}" + assert isinstance(name, str), f"Got: {name}" # Get plugin kind try: @@ -41,18 +30,25 @@ class LoadPlugin(): except Exception as e: raise Exception(f"Unknown module '{kind}.{name}': {e}") - assert (hasattr(plugin_cls, 'Plugin')), f'Plugin {kind}/{name} is not a valid plugin' + assert hasattr( + plugin_cls, "Plugin" + ), f"Plugin {kind}/{name} is not a valid plugin" # Return plugin Classe return plugin_cls.Plugin -class Manager(): + +class Manager: + """Generic manager class""" plugins_kind = [] _schema_props_default = None + _schema_props_new = None + _props_position = None @classmethod def get_schema(cls, plugins_db): + """Retrieve configuration schema""" # Properties ret3 = {} @@ -60,15 +56,17 @@ class Manager(): # ret[kind] = {} plugin_kind = getattr(plugins_db, kind) - for plugin_name in [i for i in dir(plugin_kind) if not i.startswith('_')]: + for plugin_name in [i for i in dir(plugin_kind) if not i.startswith("_")]: plugin = getattr(plugin_kind, plugin_name) - plugin_cls = getattr(plugin, 'Plugin', None) + plugin_cls = getattr(plugin, "Plugin", None) if plugin_cls: - schema_props = getattr(plugin_cls, '_schema_props_new', 'MISSING ITEM') + schema_props = getattr( + plugin_cls, "_schema_props_new", "MISSING ITEM" + ) if schema_props: # ret[kind][plugin_name] = schema_props - ret3.update( schema_props ) - ret3.update( cls._schema_props_new ) + ret3.update(schema_props) + ret3.update(cls._schema_props_new) # Injection ret1 = cls._schema_props_default @@ -79,76 +77,76 @@ class Manager(): class BackendsManager(Manager): + """Backend Manager""" - plugins_kind = ['engine', 'backend'] + plugins_kind = ["engine", "backend"] _schema_props_new = { - "engine": { - "type": "string", - "default": "jerakia", - "optional": False, - }, - "value": { - "default": 'UNSET', - "optional": False, - }, - } + "engine": { + "type": "string", + "default": "jerakia", + "optional": False, + }, + "value": { + "default": "UNSET", + "optional": False, + }, + } - _props_position = 'oneOf/0/properties' + _props_position = "oneOf/0/properties" _schema_props_default = { - "$schema": 'http://json-schema.org/draft-07/schema#', - "default": "", - # This does not work :( - #"$def": { - # "props": {}, - # }, - "oneOf": [ - { - "type": "object", - "additionalProperties": True, - "default": {}, - "title": "object", - "properties": {}, - "description": "Object to configure a bacjend item", - }, - { - "type": "string", - "default": "BLAAAAHHH", - "title": "string", - "description": "Enter a simple string configuration value for default engine", - }, - ] - } + "$schema": "http://json-schema.org/draft-07/schema#", + "default": "", + # This does not work :( + # "$def": { + # "props": {}, + # }, + "oneOf": [ + { + "type": "object", + "additionalProperties": True, + "default": {}, + "title": "object", + "properties": {}, + "description": "Object to configure a bacjend item", + }, + { + "type": "string", + "default": "BLAAAAHHH", + "title": "string", + "description": "Enter a simple string configuration value for default engine", + }, + ], + } def _validate_item(self, item): + """Private method to validate sub class""" if isinstance(item, str): item = { - "engine": self.config_main.default_engine, - "value": item, - } + "engine": self.config_main.default_engine, + "value": item, + } item = schema_validate(item, self._schema_props_default) - assert (isinstance(item, dict)) + assert isinstance(item, dict) return item def __init__(self, app): self.app = app - self.config_app = app.conf2['config']['app'] - self.config_main = app.conf2['config']['tree'] - self.config_items = list(app.conf2['tree']) + self.config_app = app.conf2["config"]["app"] + self.config_main = app.conf2["config"]["tree"] + self.config_items = list(app.conf2["tree"]) # THIS MAKE A BUG !!!! self.plugin_loader = LoadPlugin(AlberoPlugins) - self.plugins = [ - 'init', - 'loop', - 'hier', - ] + "init", + "loop", + "hier", + ] # Auto init self.backends = self.config_items - def query(self, key=None, scope=None, trace=False): backends = self.get_backends(key=key, scope=scope, trace=trace) ret = self.get_results(backends, trace=trace) @@ -160,34 +158,31 @@ class BackendsManager(Manager): # Prepare plugins plugin_loader = LoadPlugin(AlberoPlugins) _run = { - "key": key, - "scope": scope, - } + "key": key, + "scope": scope, + } # Preprocess backends plugins backends = self.config_items log.debug(f"Backend preprocessing of {len(backends)} elements") for plugin in self.plugins: - #backend_cls = plugin_loader.load('backend', plugin) - plugin = plugin_loader.load( - 'backend', plugin - )() + # backend_cls = plugin_loader.load('backend', plugin) + plugin = plugin_loader.load("backend", plugin)() log.debug(f"Run {plugin}") new_backend, _run = plugin.process(backends, _run) - assert(isinstance(new_backend, list)), f"Got: {new_backend}" - assert(isinstance(_run, dict)), f"Got: {_run}" + assert isinstance(new_backend, list), f"Got: {new_backend}" + assert isinstance(_run, dict), f"Got: {_run}" backends = new_backend # pprint (backends) for i in backends: - assert (i.get('engine')), f"Got: {i}" + assert i.get("engine"), f"Got: {i}" log.debug(f"Backend preprocessing made {len(backends)} elements") return backends - def get_results(self, backends, trace=False): # Prepare plugins @@ -195,20 +190,18 @@ class BackendsManager(Manager): new_results = [] for backend in backends: - #result_cls = result_loader.load('result', result) + # result_cls = result_loader.load('result', result) # print ("BACKKENNDNNDNDNDND") # pprint(backend) - engine = plugin_loader.load( - 'engine', backend['engine'] - )( - backend, - parent=self, app=self.app) + engine = plugin_loader.load("engine", backend["engine"])( + backend, parent=self, app=self.app + ) log.debug(f"Run engine: {engine}") new_result = engine.process() - assert(isinstance(new_result, list)), f"Got: {new_result}" + assert isinstance(new_result, list), f"Got: {new_result}" new_results.extend(new_result) # Filter out? Not here !new_results = [i for i in new_results if i['found'] ] @@ -217,65 +210,60 @@ class BackendsManager(Manager): return new_results - class RulesManager(Manager): - plugins_kind = ['strategy'] + plugins_kind = ["strategy"] _schema_props_new = { - "rule": { - "default": ".*", - "optional": True, - "oneOf": [ - { - "type": "string", - }, - { - "type": "null", - }, - ], - }, - "strategy": { - "type": "string", - "default": "schema", - # "default": "last", - "optional": True, - # "enum": ["first", "last", "merge"], - }, - - "trace": { - "type": "boolean", - "default": False, - }, - "explain": { - "type": "boolean", - "default": False, - }, - } - - _props_position = 'oneOf/1/properties' - _schema_props_default = { - "$schema": 'http://json-schema.org/draft-07/schema#', - "default": "", - "$def": { - "items": {}, - }, + "rule": { + "default": ".*", + "optional": True, "oneOf": [ { "type": "string", - "default": "BLAAAAHHH" }, { - "type": "object", - "additionalProperties": True, - "default": {}, - "properties": { "$ref": "#/$defs/name" }, + "type": "null", }, - ] - } + ], + }, + "strategy": { + "type": "string", + "default": "schema", + # "default": "last", + "optional": True, + # "enum": ["first", "last", "merge"], + }, + "trace": { + "type": "boolean", + "default": False, + }, + "explain": { + "type": "boolean", + "default": False, + }, + } + + _props_position = "oneOf/1/properties" + _schema_props_default = { + "$schema": "http://json-schema.org/draft-07/schema#", + "default": "", + "$def": { + "items": {}, + }, + "oneOf": [ + {"type": "string", "default": "BLAAAAHHH"}, + { + "type": "object", + "additionalProperties": True, + "default": {}, + "properties": {"$ref": "#/$defs/name"}, + }, + ], + } OLD_rule_schema = { - "$schema": 'http://json-schema.org/draft-07/schema#', + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": False, "properties": { @@ -286,73 +274,70 @@ class RulesManager(Manager): "oneOf": [ { "type": "null", - }, + }, { "type": "string", - }, + }, { "type": "object", "additionalProperties": True, "default": {}, - "properties": { "$ref": "#/$defs/name" }, - }, - ], - }, - } - } - + "properties": {"$ref": "#/$defs/name"}, + }, + ], + }, + }, + } def __init__(self, app): self.app = app - self.config_app = app.conf2['config']['app'] - self.config_main = app.conf2['config']['rules'] - self.config_items = list(app.conf2['rules']) + self.config_app = app.conf2["config"]["app"] + self.config_main = app.conf2["config"]["rules"] + self.config_items = list(app.conf2["rules"]) def get_result(self, candidates, key=None, scope=None, trace=False, explain=False): - #trace=False + # trace=False rules = self.config_items - key = key or '' + key = key or "" # Filter out invalid candidates - matched_candidates = [i for i in candidates if i['found'] == True] + matched_candidates = [i for i in candidates if i["found"] == True] if len(matched_candidates) == 0: log.debug("No matched candidates") return None - # Look for matching key in rules defiunitions regex_support = False matched_rule = {} if regex_support: raise Exception("Not Implemented") - else: - rule = [ i for i in rules if i.get('rule') == key ] - if len(rule) == 0: - log.debug(f"No matched rule for {key}, applying defaults") - else: - matched_rule = rule[0] - log.debug(f"Matcher rule for {key}: {matched_rule}") - matched_rule['trace'] = trace - matched_rule['explain'] = explain + rule = [i for i in rules if i.get("rule") == key] + if len(rule) == 0: + log.debug(f"No matched rule for %s, applying defaults", key) + else: + matched_rule = rule[0] + log.debug(f"Matcher rule for {key}: {matched_rule}") + + matched_rule["trace"] = trace + matched_rule["explain"] = explain schema_validate(matched_rule, self._schema_props_default) - - # Prepare plugins - assert(isinstance(matched_candidates, list)), f"Got: {matched_candidates}" - assert(isinstance(matched_rule, dict)), f"Got: {matched_rule}" - strategy = matched_rule.get('strategy', 'schema') + assert isinstance(matched_candidates, list), f"Got: {matched_candidates}" + assert isinstance(matched_rule, dict), f"Got: {matched_rule}" + strategy = matched_rule.get("strategy", "schema") log.debug(f"Key '{key}' matched rule '{rule}' with '{strategy}' strategy") # Load plugin log.debug(f"Run strategy: {strategy}") plugin_loader = LoadPlugin(AlberoPlugins) - strategy = plugin_loader.load('strategy', - strategy, - )(parent=self, app=self.app) + strategy = plugin_loader.load( + "strategy", + strategy, + )(parent=self, app=self.app) new_result = strategy.process(matched_candidates, matched_rule) return new_result diff --git a/albero/plugin/backend/hier.py b/albero/plugin/backend/hier.py index 401b9b4..2239813 100644 --- a/albero/plugin/backend/hier.py +++ b/albero/plugin/backend/hier.py @@ -1,11 +1,10 @@ - - import copy + # from pathlib import Path # from albero.utils import render_template # from albero.plugin.common import PluginBackendClass # from pprint import pprint -# +# # import logging # import anyconfig # import textwrap @@ -13,55 +12,55 @@ import copy from albero.plugin.common import PluginBackendClass from pprint import pprint import logging + log = logging.getLogger(__name__) + class Plugin(PluginBackendClass): _plugin_name = "hier" _schema_props_new = { - "hier": { - "default": None, - "optional": True, - "oneOf": [ - { - "type": "null", - }, - { - "type": "string", - }, - { - "additionalProperties": True, - "properties": { - "data": { - "default": None, - "anyOf": [ - { "type": "null" }, - { "type": "string" }, - { "type": "array" }, - ] - }, - "var": { - "type": "string", - "default": "hier_item", - "optional": True, - }, - "separator": { - "type": "string", - "default": "/", - "optional": True, - }, - "reversed": { - "type": "boolean", - "default": False, - "optional": True, - }, + "hier": { + "default": None, + "optional": True, + "oneOf": [ + { + "type": "null", + }, + { + "type": "string", + }, + { + "additionalProperties": True, + "properties": { + "data": { + "default": None, + "anyOf": [ + {"type": "null"}, + {"type": "string"}, + {"type": "array"}, + ], + }, + "var": { + "type": "string", + "default": "hier_item", + "optional": True, + }, + "separator": { + "type": "string", + "default": "/", + "optional": True, + }, + "reversed": { + "type": "boolean", + "default": False, + "optional": True, }, }, - ] - } + }, + ], } - - + } def process(self, backends: list, ctx: dict) -> (list, dict): @@ -79,13 +78,13 @@ class Plugin(PluginBackendClass): hier_var = plugin_config.get("var", "item") hier_sep = plugin_config.get("separator", "/") if isinstance(hier_data, str): - hier_data = cand['_run']['scope'].get(hier_data, None) + hier_data = cand["_run"]["scope"].get(hier_data, None) # Build a new list if isinstance(hier_data, str): r = hier_data.split(hier_sep) - assert (isinstance(r, list)), f"Got: {r}" + assert isinstance(r, list), f"Got: {r}" ret1 = [] for index, part in enumerate(r): @@ -93,7 +92,7 @@ class Plugin(PluginBackendClass): try: prefix = ret1[index - 1] except IndexError: - prefix = f'{hier_sep}' + prefix = f"{hier_sep}" prefix = "" item = f"{prefix}{part}{hier_sep}" ret1.append(item) @@ -102,15 +101,13 @@ class Plugin(PluginBackendClass): for item in ret1: _cand = copy.deepcopy(cand) run = { - "index": index, - "hier_value": item, - "hier_var": hier_var, - } - _cand['_run']['hier'] = run - _cand['_run']['scope'][hier_var] = item + "index": index, + "hier_value": item, + "hier_var": hier_var, + } + _cand["_run"]["hier"] = run + _cand["_run"]["scope"][hier_var] = item ret2.append(_cand) new_backends.extend(ret2) return new_backends, ctx - - diff --git a/albero/plugin/backend/init.py b/albero/plugin/backend/init.py index 0f8e149..b6a290b 100644 --- a/albero/plugin/backend/init.py +++ b/albero/plugin/backend/init.py @@ -1,37 +1,35 @@ - - from albero.plugin.common import PluginBackendClass from pprint import pprint import logging + log = logging.getLogger(__name__) import copy + class Plugin(PluginBackendClass): _plugin_name = "init" _schema_props_new = None - default_engine = 'jerakia' + default_engine = "jerakia" def process(self, backends: list, ctx: dict) -> (list, dict): new_backends = [] for index, item in enumerate(backends): default = { - "value": item, - } + "value": item, + } if not isinstance(item, dict): item = default - item['engine'] = item.get('engine', self.default_engine ) - item['_run'] = copy.deepcopy(ctx) - item['_run']['backend'] = { - "index": index, - } + item["engine"] = item.get("engine", self.default_engine) + item["_run"] = copy.deepcopy(ctx) + item["_run"]["backend"] = { + "index": index, + } new_backends.append(item) return new_backends, ctx - - diff --git a/albero/plugin/backend/loop.py b/albero/plugin/backend/loop.py index 760e26a..4fb191d 100644 --- a/albero/plugin/backend/loop.py +++ b/albero/plugin/backend/loop.py @@ -1,5 +1,3 @@ - - import copy from pathlib import Path from albero.utils import render_template @@ -14,56 +12,56 @@ import textwrap class Plugin(PluginBackendClass): _plugin_name = "loop" - _plugin_help = """ + _plugin_help = ( + """ This module helps to loop over a backend """, + ) _schema_props_new = { "loop": { "description": _plugin_help, "default": None, "optional": True, "examples": [ - { - "value": "site/{{ loop_env }}/config/{{ os }}", - "loop": { - "var": "loop_env", - "data": [ - "dev", - "preprod", - "prod", - ], - }, - "comment": "The module will loop three time over the value, and the variable `loop_env` will consecutely have `dev`, `preprod` and `prod` as value", + { + "value": "site/{{ loop_env }}/config/{{ os }}", + "loop": { + "var": "loop_env", + "data": [ + "dev", + "preprod", + "prod", + ], }, - { - "value": "site/{{ loop_env2 }}/config/{{ os }}", - "loop": { - "var": "loop_env2", - "data": "my_scope_var", - }, - "comment": "Like the previous example, but it will fetch the list from any scope variables", + "comment": "The module will loop three time over the value, and the variable `loop_env` will consecutely have `dev`, `preprod` and `prod` as value", + }, + { + "value": "site/{{ loop_env2 }}/config/{{ os }}", + "loop": { + "var": "loop_env2", + "data": "my_scope_var", }, - { - "loop": None, - "comment": "Disable this module, no loop will operate", - }, - - - # "loop": { - # "var": "my_var", - # }, - # }, - # "loop": { - # "var": "my_var", - # }, - # "example": "", - # }, - # "loop": { - # "var": "my_var", - # }, - # "example": "", - # }, - ], + "comment": "Like the previous example, but it will fetch the list from any scope variables", + }, + { + "loop": None, + "comment": "Disable this module, no loop will operate", + }, + # "loop": { + # "var": "my_var", + # }, + # }, + # "loop": { + # "var": "my_var", + # }, + # "example": "", + # }, + # "loop": { + # "var": "my_var", + # }, + # "example": "", + # }, + ], "oneOf": [ { "type": "object", @@ -71,30 +69,29 @@ class Plugin(PluginBackendClass): "default": {}, "title": "Complete config", "description": "", - "properties": { "data": { "default": None, "optional": False, "title": "Module configuration", "description": "Data list used for iterations. It only accept lists as type. It disable the module if set to `null`.", - "anyOf":[ - { - "type": "null", - "title": "Disable Module", - "description": "Disable the module", - }, - { - "type": "string", - "title": "Scope variable", - "description": "Will look the value of the loop list from the scope. TOFIX: What if variablle does not exists?", - }, - { - "type": "array", - "title": "Hardcoded list", - "description": "Simply enter the list of value to be iterated to.", - }, - ] + "anyOf": [ + { + "type": "null", + "title": "Disable Module", + "description": "Disable the module", + }, + { + "type": "string", + "title": "Scope variable", + "description": "Will look the value of the loop list from the scope. TOFIX: What if variablle does not exists?", + }, + { + "type": "array", + "title": "Hardcoded list", + "description": "Simply enter the list of value to be iterated to.", + }, + ], }, "var": { "type": "string", @@ -115,14 +112,12 @@ class Plugin(PluginBackendClass): "title": "Disable", "description": "If set to null, it disable the module", }, - ] + ], } } - def process(self, backends: list, ctx: dict) -> (list, dict): - new_backends = [] for cand in backends: cand = dict(cand) @@ -137,26 +132,23 @@ class Plugin(PluginBackendClass): # Retrieve config data loop_var = loop_config.get("var", "item") if isinstance(loop_data, str): - loop_data = cand['_run']['scope'].get(loop_data, None) - assert (isinstance(loop_data, list)), f"Got: {loop_data}" + loop_data = cand["_run"]["scope"].get(loop_data, None) + assert isinstance(loop_data, list), f"Got: {loop_data}" # Build a new list ret = [] for idx, item in enumerate(loop_data): _cand = copy.deepcopy(cand) run = { - "loop_index": idx, - "loop_value": item, - "loop_var": loop_var, - } - _cand['_run']['loop'] = run - _cand['_run']['scope'][loop_var] = item - #_cand.scope[loop_var] = item + "loop_index": idx, + "loop_value": item, + "loop_var": loop_var, + } + _cand["_run"]["loop"] = run + _cand["_run"]["scope"][loop_var] = item + # _cand.scope[loop_var] = item ret.append(_cand) new_backends.extend(ret) return new_backends, ctx - - - diff --git a/albero/plugin/common.py b/albero/plugin/common.py index e44f53d..aa34916 100644 --- a/albero/plugin/common.py +++ b/albero/plugin/common.py @@ -1,4 +1,3 @@ - # from box import Box import textwrap from pprint import pprint @@ -9,6 +8,7 @@ import yaml import json import logging + log = logging.getLogger(__name__) from albero.utils import schema_validate @@ -16,7 +16,7 @@ import copy # Candidate Classes # ============================= -class Candidate(): +class Candidate: engine = None found = False data = None @@ -32,40 +32,36 @@ class Candidate(): def _report_data(self, data=None): default_data = { - #"rule": self.config, - "value": self.engine._plugin_value, - "data": self.data, - } + # "rule": self.config, + "value": self.engine._plugin_value, + "data": self.data, + } data = data or default_data - d = json.dumps(data, indent=2) #, sort_keys=True, ) + d = json.dumps(data, indent=2) # , sort_keys=True, ) return d - - - - # Generic Classes # ============================= -class PluginClass(): +class PluginClass: _plugin_type = "none" _plugin_value = None _schema_props_new = "UNSET PLUGIN PROPRIETIES" _schema_props_plugin = { - "engine": { - "type": "string", - # TODO: Fix this ug - "default": "jerakia" - }, - "value": {}, - } + "engine": { + "type": "string", + # TODO: Fix this ug + "default": "jerakia", + }, + "value": {}, + } def __repr__(self): kind = self._plugin_type name = self._plugin_name - value = getattr(self, 'value', 'NO VALUE') + value = getattr(self, "value", "NO VALUE") return f"{kind}.{name}:{value}" def __init__(self, config=None, parent=None, app=None): @@ -79,61 +75,63 @@ class PluginClass(): def _init(self): pass + def _validate(self): pass + class PluginBackendClass(PluginClass): _plugin_type = "backend" def _init(self): pass + class PluginStrategyClass(PluginClass): _plugin_type = "strategy" def _init(self): pass + class PluginEngineClass(PluginClass): _plugin_type = "engine" _schema_props_default = { - "value": { - "default": "UNSET", + "value": { + "default": "UNSET", + }, + #### SHOULD NOT BE HERE + "hier": { + "additionalProperties": True, + "optional": True, + "properties": { + "var": { + "type": "string", + "default": "item", + "optional": True, }, - - #### SHOULD NOT BE HERE - "hier": { - "additionalProperties": True, - "optional": True, - "properties": { - "var": { - "type": "string", - "default": "item", - "optional": True, - }, - "data": { - "default": None, - "anyOf": [ - { "type": "null" }, - { "type": "string" }, - { "type": "array" }, - ] - }, - "separator": { - "type": "string", - "default": "/", - "optional": True, - }, - "reversed": { - "type": "boolean", - "default": False, - "optional": True, - }, - } - } - } - + "data": { + "default": None, + "anyOf": [ + {"type": "null"}, + {"type": "string"}, + {"type": "array"}, + ], + }, + "separator": { + "type": "string", + "default": "/", + "optional": True, + }, + "reversed": { + "type": "boolean", + "default": False, + "optional": True, + }, + }, + }, + } # Default plugin API Methods # ===================== @@ -143,13 +141,13 @@ class PluginEngineClass(PluginClass): def _validate(self): # Build schema - schema_keys = [a for a in dir(self) if a.startswith('_schema_props_')] + schema_keys = [a for a in dir(self) if a.startswith("_schema_props_")] props = {} for key in schema_keys: schema = getattr(self, key) props.update(schema) self.schema = { - "$schema": 'http://json-schema.org/draft-07/schema#', + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": True, "properties": props, @@ -159,72 +157,75 @@ class PluginEngineClass(PluginClass): self.config = schema_validate(self.config, self.schema) return True - # Public Methods # ===================== def dump(self): ret = { "config": self.config, - } + } return ret def lookup_candidates(self, key=None, scope=None): - raise Exception (f"Module does not implement this method :(") + raise Exception(f"Module does not implement this method :(") # It must always return a list of `Candidate` instances return [] def _example(self): - print (f"Module does not implement this method :(") + print(f"Module does not implement this method :(") return None # File plugins Extensions # ============================= -class PluginFileGlob(): + +class PluginFileGlob: _schema_props_glob = { - "glob": { - "additionalProperties": False, - "default": { - "file": "ansible.yaml", - }, - "properties": { - "file": { - "type": "string", - "default": "ansible", - "optional": True, - }, - "ext": { - "type": "array", - "default": [ "yml", "yaml" ], - "optional": True, - }, - } - } + "glob": { + "additionalProperties": False, + "default": { + "file": "ansible.yaml", + }, + "properties": { + "file": { + "type": "string", + "default": "ansible", + "optional": True, + }, + "ext": { + "type": "array", + "default": ["yml", "yaml"], + "optional": True, + }, + }, } + } def _glob(self, item): # DIRECT CALL TO APP< TOFIX app_config = self.app.conf2 - root = app_config.get("default", {}).get("config", {}).get("root", f"{Path.cwd()}/tree") - #root = self.app.conf2.config.app.root + root = ( + app_config.get("default", {}) + .get("config", {}) + .get("root", f"{Path.cwd()}/tree") + ) + # root = self.app.conf2.config.app.root # TOFIX print ("ITEM! %s" % type(root)) # TOFIX print ("ITEM2 %s" % self.app.conf2.config.app.root) glob_config = self.config.get("glob", {}) - glob_file = glob_config['file'] - #glob_ext = glob_config['ext'] + glob_file = glob_config["file"] + # glob_ext = glob_config['ext'] item = Path(root) / Path(item) / Path(glob_file) item = f"{item}" - #file = f"{glob_file}.{glob_ext}" + # file = f"{glob_file}.{glob_ext}" - #print ("ITEM %s" % item) + # print ("ITEM %s" % item) files = glob.glob(item) - log.debug (f"Matched file for glob '{item}': {files}") + log.debug(f"Matched file for glob '{item}': {files}") return files - diff --git a/albero/plugin/engine/jerakia.py b/albero/plugin/engine/jerakia.py index b84e5e0..89f05fc 100644 --- a/albero/plugin/engine/jerakia.py +++ b/albero/plugin/engine/jerakia.py @@ -1,4 +1,3 @@ - from pathlib import Path from albero.utils import render_template from albero.plugin.common import PluginEngineClass, PluginFileGlob, Candidate @@ -10,55 +9,55 @@ import textwrap log = logging.getLogger(__name__) + class FileCandidate(Candidate): path = None def _report_data(self): data = { - #"rule": self.config, - "value": self.engine._plugin_value, - "data": self.data, - "path": str(self.path.relative_to(Path.cwd())), - } + # "rule": self.config, + "value": self.engine._plugin_value, + "data": self.data, + "path": str(self.path.relative_to(Path.cwd())), + } data = dict(self.config) return super()._report_data(data) - class Plugin(PluginEngineClass, PluginFileGlob): - _plugin_name = 'jerakia' + _plugin_name = "jerakia" ### OLD _plugin_engine = "jerakia" # _schema_props_files = { _schema_props_new = { - "path": { - "anyOf": [ - { + "path": { + "anyOf": [ + { + "type": "string", + }, + { + "type": "array", + "items": { "type": "string", }, - { - "type": "array", - "items": { - "type": "string", - } - }, - ] - } + }, + ] } - - + } def _init(self): - paths = self.config.get('path', self.config.get('value')) + paths = self.config.get("path", self.config.get("value")) if isinstance(paths, str): paths = [paths] elif isinstance(paths, list): pass else: - raise Exception (f"Unsupported path value, expected str or dict, got: {paths} in {self.config}") + raise Exception( + f"Unsupported path value, expected str or dict, got: {paths} in {self.config}" + ) self.paths = paths self.value = paths @@ -78,7 +77,6 @@ class Plugin(PluginEngineClass, PluginFileGlob): return ret - def _show_paths(self, scope): parsed = self._preprocess(scope) @@ -93,14 +91,12 @@ class Plugin(PluginEngineClass, PluginFileGlob): return ret3 - def process(self): - - #scope = self.scope + # scope = self.scope # pprint (self.config) - scope = dict(self.config['_run']['scope']) - key = self.config['_run']['key'] + scope = dict(self.config["_run"]["scope"]) + key = self.config["_run"]["key"] assert isinstance(scope, dict), f"Got: {scope}" assert isinstance(key, (str, type(None))), f"Got: {key}" @@ -125,13 +121,13 @@ class Plugin(PluginEngineClass, PluginFileGlob): # Build result object result = {} - result['run'] = { - 'path': path, - 'rel_path': str(Path(path).relative_to(Path.cwd())), - } - result['parent'] = self.config - result['data'] = data - result['found'] = found + result["run"] = { + "path": path, + "rel_path": str(Path(path).relative_to(Path.cwd())), + } + result["parent"] = self.config + result["data"] = data + result["found"] = found ret.append(result) @@ -141,7 +137,7 @@ class Plugin(PluginEngineClass, PluginFileGlob): # # Read raw file content # data = anyconfig.load(path, ac_parser="yaml") - # + # # ret_obj2 ={ # "_run": _run, @@ -183,7 +179,3 @@ class Plugin(PluginEngineClass, PluginFileGlob): # #log.debug(f"Found value: {ret_obj}") # ret_obj.found = found # ret.append(ret_obj) - - - - diff --git a/albero/plugin/strategy/last.py b/albero/plugin/strategy/last.py index 689beef..324dde5 100644 --- a/albero/plugin/strategy/last.py +++ b/albero/plugin/strategy/last.py @@ -1,9 +1,9 @@ - import logging from albero.plugin.common import PluginStrategyClass log = logging.getLogger(__name__) + class Plugin(PluginStrategyClass): _plugin_name = "last" @@ -12,4 +12,3 @@ class Plugin(PluginStrategyClass): def process(self, candidates: list, rule=None) -> (list, dict): return candidates[-1] - diff --git a/albero/plugin/strategy/schema.py b/albero/plugin/strategy/schema.py index 4cfe9b4..e1d2fdc 100644 --- a/albero/plugin/strategy/schema.py +++ b/albero/plugin/strategy/schema.py @@ -1,5 +1,3 @@ - - import logging from albero.plugin.common import PluginStrategyClass from albero.utils import schema_validate, str_ellipsis @@ -11,6 +9,7 @@ from pprint import pprint from jsonmerge import Merger from prettytable import PrettyTable + class Plugin(PluginStrategyClass): _plugin_name = "schema" @@ -36,11 +35,11 @@ class Plugin(PluginStrategyClass): "data": { "default": None, "optional": False, - "anyOf":[ - {"type": "null"}, - {"type": "string"}, - {"type": "array"}, - ] + "anyOf": [ + {"type": "null"}, + {"type": "string"}, + {"type": "array"}, + ], }, "var": { "type": "string", @@ -49,17 +48,17 @@ class Plugin(PluginStrategyClass): }, }, }, - ] + ], } } default_merge_schema = { - "$schema": 'http://json-schema.org/draft-07/schema#', + "$schema": "http://json-schema.org/draft-07/schema#", "oneOf": [ { "type": "array", "mergeStrategy": "append", -# "mergeStrategy": "arrayMergeById", + # "mergeStrategy": "arrayMergeById", }, { "type": "object", @@ -88,76 +87,123 @@ class Plugin(PluginStrategyClass): ], } - def process(self, candidates: list, rule=None) -> (list, dict): - trace = rule['trace'] - explain = rule['explain'] - schema = rule.get('schema', None) or self.default_merge_schema + trace = rule["trace"] + explain = rule["explain"] + schema = rule.get("schema", None) or self.default_merge_schema merger = Merger(schema) t = PrettyTable() t1 = PrettyTable() new_candidate = None for index, item in enumerate(candidates): - new_value = item['data'] + new_value = item["data"] result = merger.merge(new_candidate, new_value) - backend_info = dict(item['parent']) + backend_info = dict(item["parent"]) backend_run = backend_info.pop("_run") if explain: - t1.add_row([ - index, - '\nBackendRun: ' + str_ellipsis(json.dumps( - backend_run, - default=lambda o: '', indent=2), 70), - '\nRuleRun: ' + str_ellipsis(json.dumps( - item['run'], - default=lambda o: '', indent=2), 70), - '---\nResult: ' + str_ellipsis(json.dumps( - result, - default=lambda o: '', indent=2), 70), - ]) + t1.add_row( + [ + index, + "\nBackendRun: " + + str_ellipsis( + json.dumps( + backend_run, + default=lambda o: "", + indent=2, + ), + 70, + ), + "\nRuleRun: " + + str_ellipsis( + json.dumps( + item["run"], + default=lambda o: "", + indent=2, + ), + 70, + ), + "---\nResult: " + + str_ellipsis( + json.dumps( + result, default=lambda o: "", indent=2 + ), + 70, + ), + ] + ) if trace: - t.add_row([ - index, - '---\nBackendConfig: ' + str_ellipsis(json.dumps( - backend_info, - default=lambda o: '', indent=2), 70) + - '\nBackendRun: ' + str_ellipsis(json.dumps( - backend_run, - default=lambda o: '', indent=2), 70), - - '---\nRuleConfig: ' + str_ellipsis(json.dumps( - rule, - default=lambda o: '', indent=2), 70) + - '\nRuleRun: ' + str_ellipsis(json.dumps( - item['run'], - default=lambda o: '', indent=2), 70) + - #'\nSource: ' + str_ellipsis(json.dumps( - # new_candidate, - # default=lambda o: '', indent=2), 70) + - '\nNew data: ' + str_ellipsis(json.dumps( - new_value, - default=lambda o: '', indent=2), 70), - - '---\nResult: ' + str_ellipsis(json.dumps( - result, - default=lambda o: '', indent=2), 70), - ] + t.add_row( + [ + index, + "---\nBackendConfig: " + + str_ellipsis( + json.dumps( + backend_info, + default=lambda o: "", + indent=2, + ), + 70, ) + + "\nBackendRun: " + + str_ellipsis( + json.dumps( + backend_run, + default=lambda o: "", + indent=2, + ), + 70, + ), + "---\nRuleConfig: " + + str_ellipsis( + json.dumps( + rule, default=lambda o: "", indent=2 + ), + 70, + ) + + "\nRuleRun: " + + str_ellipsis( + json.dumps( + item["run"], + default=lambda o: "", + indent=2, + ), + 70, + ) + + + #'\nSource: ' + str_ellipsis(json.dumps( + # new_candidate, + # default=lambda o: '', indent=2), 70) + + "\nNew data: " + + str_ellipsis( + json.dumps( + new_value, + default=lambda o: "", + indent=2, + ), + 70, + ), + "---\nResult: " + + str_ellipsis( + json.dumps( + result, default=lambda o: "", indent=2 + ), + 70, + ), + ] + ) new_candidate = result if trace: t.field_names = ["Index", "Backend", "Rule", "Data"] - t.align = 'l' - print (t) + t.align = "l" + print(t) if explain: t1.field_names = ["Index", "Backend", "Rule", "Data"] - t1.align = 'l' - print('Explain:\n' + repr(t1)) + t1.align = "l" + print("Explain:\n" + repr(t1)) return new_candidate - - diff --git a/albero/query.py b/albero/query.py index 1339fc7..6bb05db 100755 --- a/albero/query.py +++ b/albero/query.py @@ -12,19 +12,20 @@ from pprint import pprint from albero.managers import BackendsManager, RulesManager from albero.utils import schema_validate import anyconfig + # from box import Box from pathlib import Path import logging -log = logging.getLogger(__name__) +log = logging.getLogger(__name__) # Query ########################################## -class Query(): +class Query: def __init__(self, app): self.app = app @@ -43,9 +44,7 @@ class Query(): ret = {} for i in dir(self): - if not i.startswith('_'): + if not i.startswith("_"): ret[i] = getattr(self, i) - pprint (ret) - - + pprint(ret) diff --git a/albero/utils.py b/albero/utils.py index ae88ae2..ff857cb 100644 --- a/albero/utils.py +++ b/albero/utils.py @@ -1,4 +1,3 @@ - from pathlib import Path from jinja2 import Template import yaml @@ -10,6 +9,7 @@ import collections import logging + log = logging.getLogger(__name__) @@ -62,19 +62,21 @@ log = logging.getLogger(__name__) # Utils Methods # ===================== + def render_template(path, params): """Render template for a given string""" - assert (isinstance(params, dict)), f"Got: {params}" + assert isinstance(params, dict), f"Got: {params}" t = Template(path) return t.render(**params) -#def read_file(file): + +# def read_file(file): # with open(file, 'r') as f: # data = f.read().replace('\n', '') # return data # # -#def parse_file(file, fmt='auto'): +# def parse_file(file, fmt='auto'): # print ("DEPRECATED") # raise Exception ("parse_file is deprecated") # @@ -99,6 +101,7 @@ def render_template(path, params): # Schema Methods # ===================== + def _extend_with_default(validator_class): validate_properties = validator_class.VALIDATORS["properties"] @@ -110,38 +113,43 @@ def _extend_with_default(validator_class): try: for error in validate_properties( - validator, properties, instance, schema, + validator, + properties, + instance, + schema, ): continue except Exception as e: - print ("CATCHED2222 ", e) + print("CATCHED2222 ", e) return validators.extend( - validator_class, {"properties" : set_defaults}, + validator_class, + {"properties": set_defaults}, ) def schema_validate(config, schema): - # Validate the schema - DefaultValidatingDraft7Validator = _extend_with_default(Draft7Validator) - try: - DefaultValidatingDraft7Validator(schema).validate(config) - except Exception as e: - print (e) - p = list(collections.deque(e.schema_path)) - p = '/'.join([ str(i) for i in p ]) - p = f"schema/{p}" - raise Exception( - f"Failed validating {p} for resource with content: {config} with !!!!!! schema: {schema}" - ) - return config + # Validate the schema + DefaultValidatingDraft7Validator = _extend_with_default(Draft7Validator) + try: + DefaultValidatingDraft7Validator(schema).validate(config) + except Exception as e: + print(e) + p = list(collections.deque(e.schema_path)) + p = "/".join([str(i) for i in p]) + p = f"schema/{p}" + raise Exception( + f"Failed validating {p} for resource with content: {config} with !!!!!! schema: {schema}" + ) + return config + def str_ellipsis(txt, length=120): txt = str(txt) ret = [] for string in txt.splitlines(): - string = (string[:length - 4 ] + ' ...') if len(string) > length else string + string = (string[: length - 4] + " ...") if len(string) > length else string ret.append(string) - ret = '\n'.join(ret) + ret = "\n".join(ret) return ret