pro-barbu-foss/kheops
pro-barbu-foss/kheops
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Lint: Whole codebase with black

Browse Source
This commit is contained in:
mrjk 2022-01-15 22:54:13 -05:00
parent 3ab48fab3a
commit 76925684c7
12 changed files with 629 additions and 606 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
albero/app.py
View File

@ -12,25 +12,27 @@ from pprint import pprint
from albero.query import Query from albero.query import Query
from albero.utils import schema_validate from albero.utils import schema_validate
import anyconfig import anyconfig
# from box import Box # from box import Box
from pathlib import Path from pathlib import Path
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class App(): class App:
schema = { schema = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
"default": {}, "default": {},
"$def": { "$def": {
'backends_items': {}, "backends_items": {},
'backends_config': {}, "backends_config": {},
'rules_items': {}, "rules_items": {},
'rules_config': {}, "rules_config": {},
}, },
"patternProperties": { "patternProperties": {
".*": { ".*": {
@ -54,7 +56,6 @@ class App():
}, },
}, },
}, },
"tree": { "tree": {
# "additionalProperties": False, # "additionalProperties": False,
"type": "object", "type": "object",
@ -65,7 +66,6 @@ class App():
"default": {}, "default": {},
}, },
}, },
}, },
"tree": { "tree": {
"type": "array", "type": "array",
@ -82,10 +82,10 @@ class App():
}, },
}, },
}, },
} },
} }
def __init__(self, config="albero.yml", namespace='default'): def __init__(self, config="albero.yml", namespace="default"):
conf2 = anyconfig.load(config) conf2 = anyconfig.load(config)
# Validate configuration # Validate configuration
@ -97,10 +97,10 @@ class App():
sys.exit(1) sys.exit(1)
# Init # Init
if not conf2['config']['app']['root']: if not conf2["config"]["app"]["root"]:
conf2['config']['app']['root'] = Path(config).parent conf2["config"]["app"]["root"] = Path(config).parent
else: else:
conf2['config']['app']['root'] = Path(conf2['config']['app']['root']) conf2["config"]["app"]["root"] = Path(conf2["config"]["app"]["root"])
# Finish # Finish
self.conf2 = dict(conf2) self.conf2 = dict(conf2)
@ -111,10 +111,9 @@ class App():
r = q.exec(key=key, scope=scope, policy=policy, trace=trace, explain=explain) r = q.exec(key=key, scope=scope, policy=policy, trace=trace, explain=explain)
print("=== Query Result ===") print("=== Query Result ===")
print(anyconfig.dumps(r, ac_parser='yaml')) print(anyconfig.dumps(r, ac_parser="yaml"))
print("=== Query Result ===") print("=== Query Result ===")
def dump_schema(self): def dump_schema(self):
import json import json
@ -125,10 +124,7 @@ class App():
r2 = RulesManager.get_schema(AlberoPlugins) r2 = RulesManager.get_schema(AlberoPlugins)
d = self.schema d = self.schema
d['patternProperties']['.*']['properties'] ['tree']['items']['properties'] = r1 d["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = r1
d['patternProperties']['.*']['properties'] ['tree']['items'] = r2 d["patternProperties"][".*"]["properties"]["tree"]["items"] = r2
print(json.dumps(d, indent=2)) print(json.dumps(d, indent=2))

28
albero/cli.py
View File

@ -18,6 +18,7 @@ sys.path.append("/home/jez/prj/bell/training/tiger-ansible/ext/ansible-tree")
import albero.app as Albero import albero.app as Albero
class CmdApp: class CmdApp:
"""Main CmdApp""" """Main CmdApp"""
@ -99,7 +100,9 @@ class CmdApp:
"""Prepare command line""" """Prepare command line"""
# Manage main parser # Manage main parser
parser = argparse.ArgumentParser(description="Albero, to lookup hierarchical data") parser = argparse.ArgumentParser(
description="Albero, to lookup hierarchical data"
)
parser.add_argument( parser.add_argument(
"-v", "--verbose", action="count", default=0, help="Increase verbosity" "-v", "--verbose", action="count", default=0, help="Increase verbosity"
) )
@ -112,9 +115,15 @@ class CmdApp:
# Manage command: demo # Manage command: demo
add_p = subparsers.add_parser("lookup") add_p = subparsers.add_parser("lookup")
add_p.add_argument("-n", "--namespace", help="Namespace name", default='default') add_p.add_argument(
add_p.add_argument("-f", "--file", help="File with params as dict. Can be stdin - .") "-n", "--namespace", help="Namespace name", default="default"
add_p.add_argument("-e", "--scope", dest="scope_param", action="append", default=[]) )
add_p.add_argument(
"-f", "--file", help="File with params as dict. Can be stdin - ."
)
add_p.add_argument(
"-e", "--scope", dest="scope_param", action="append", default=[]
)
add_p.add_argument("-p", "--policy") add_p.add_argument("-p", "--policy")
add_p.add_argument("-t", "--trace", action="store_true") add_p.add_argument("-t", "--trace", action="store_true")
add_p.add_argument("-x", "--explain", action="store_true") add_p.add_argument("-x", "--explain", action="store_true")
@ -150,7 +159,7 @@ class CmdApp:
def cli_lookup(self): def cli_lookup(self):
"""Display how to use logging""" """Display how to use logging"""
config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml' config = "/home/jez/prj/bell/training/tiger-ansible/tree.yml"
# self.log.debug(f"Command line vars: {vars(self.args)}") # self.log.debug(f"Command line vars: {vars(self.args)}")
keys = self.args.key or [None] keys = self.args.key or [None]
@ -162,7 +171,7 @@ class CmdApp:
# Parse cli params # Parse cli params
for i in self.args.scope_param: for i in self.args.scope_param:
r = i.split('=') r = i.split("=")
if len(r) != 2: if len(r) != 2:
raise Exception("Malformed params") raise Exception("Malformed params")
new_params[r[0]] = r[1] new_params[r[0]] = r[1]
@ -171,16 +180,17 @@ class CmdApp:
app = Albero.App(config=config, namespace=self.args.namespace) app = Albero.App(config=config, namespace=self.args.namespace)
for key in keys: for key in keys:
app.lookup(key=key, app.lookup(
key=key,
scope=new_params, scope=new_params,
trace=self.args.trace, trace=self.args.trace,
explain=self.args.explain explain=self.args.explain,
) )
def cli_schema(self): def cli_schema(self):
"""Display configuration schema""" """Display configuration schema"""
config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml' config = "/home/jez/prj/bell/training/tiger-ansible/tree.yml"
app = Albero.App(config=config) # , namespace=self.args.namespace) app = Albero.App(config=config) # , namespace=self.args.namespace)
app.dump_schema() app.dump_schema()

137
albero/managers.py
View File

@ -1,33 +1,22 @@
import dpath.util import dpath.util
import copy
import json
import textwrap
from prettytable import PrettyTable
from pathlib import Path
# from box import Box
from jsonmerge import Merger
import re
import logging import logging
from pprint import pprint from pprint import pprint
import collections
from albero.utils import schema_validate, str_ellipsis from albero.utils import schema_validate
import albero.plugin as AlberoPlugins import albero.plugin as AlberoPlugins
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class LoadPlugin(): class LoadPlugin:
def __init__(self, plugins): def __init__(self, plugins):
self.plugins = plugins self.plugins = plugins
def load(self, kind, name): def load(self, kind, name):
assert (isinstance(name, str)), f"Got: {name}" assert isinstance(name, str), f"Got: {name}"
# Get plugin kind # Get plugin kind
try: try:
@ -41,18 +30,25 @@ class LoadPlugin():
except Exception as e: except Exception as e:
raise Exception(f"Unknown module '{kind}.{name}': {e}") raise Exception(f"Unknown module '{kind}.{name}': {e}")
assert (hasattr(plugin_cls, 'Plugin')), f'Plugin {kind}/{name} is not a valid plugin' assert hasattr(
plugin_cls, "Plugin"
), f"Plugin {kind}/{name} is not a valid plugin"
# Return plugin Classe # Return plugin Classe
return plugin_cls.Plugin return plugin_cls.Plugin
class Manager():
class Manager:
"""Generic manager class"""
plugins_kind = [] plugins_kind = []
_schema_props_default = None _schema_props_default = None
_schema_props_new = None
_props_position = None
@classmethod @classmethod
def get_schema(cls, plugins_db): def get_schema(cls, plugins_db):
"""Retrieve configuration schema"""
# Properties # Properties
ret3 = {} ret3 = {}
@ -60,11 +56,13 @@ class Manager():
# ret[kind] = {} # ret[kind] = {}
plugin_kind = getattr(plugins_db, kind) plugin_kind = getattr(plugins_db, kind)
for plugin_name in [i for i in dir(plugin_kind) if not i.startswith('_')]: for plugin_name in [i for i in dir(plugin_kind) if not i.startswith("_")]:
plugin = getattr(plugin_kind, plugin_name) plugin = getattr(plugin_kind, plugin_name)
plugin_cls = getattr(plugin, 'Plugin', None) plugin_cls = getattr(plugin, "Plugin", None)
if plugin_cls: if plugin_cls:
schema_props = getattr(plugin_cls, '_schema_props_new', 'MISSING ITEM') schema_props = getattr(
plugin_cls, "_schema_props_new", "MISSING ITEM"
)
if schema_props: if schema_props:
# ret[kind][plugin_name] = schema_props # ret[kind][plugin_name] = schema_props
ret3.update(schema_props) ret3.update(schema_props)
@ -79,8 +77,9 @@ class Manager():
class BackendsManager(Manager): class BackendsManager(Manager):
"""Backend Manager"""
plugins_kind = ['engine', 'backend'] plugins_kind = ["engine", "backend"]
_schema_props_new = { _schema_props_new = {
"engine": { "engine": {
@ -89,14 +88,14 @@ class BackendsManager(Manager):
"optional": False, "optional": False,
}, },
"value": { "value": {
"default": 'UNSET', "default": "UNSET",
"optional": False, "optional": False,
}, },
} }
_props_position = 'oneOf/0/properties' _props_position = "oneOf/0/properties"
_schema_props_default = { _schema_props_default = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"default": "", "default": "",
# This does not work :( # This does not work :(
# "$def": { # "$def": {
@ -117,38 +116,37 @@ class BackendsManager(Manager):
"title": "string", "title": "string",
"description": "Enter a simple string configuration value for default engine", "description": "Enter a simple string configuration value for default engine",
}, },
] ],
} }
def _validate_item(self, item): def _validate_item(self, item):
"""Private method to validate sub class"""
if isinstance(item, str): if isinstance(item, str):
item = { item = {
"engine": self.config_main.default_engine, "engine": self.config_main.default_engine,
"value": item, "value": item,
} }
item = schema_validate(item, self._schema_props_default) item = schema_validate(item, self._schema_props_default)
assert (isinstance(item, dict)) assert isinstance(item, dict)
return item return item
def __init__(self, app): def __init__(self, app):
self.app = app self.app = app
self.config_app = app.conf2['config']['app'] self.config_app = app.conf2["config"]["app"]
self.config_main = app.conf2['config']['tree'] self.config_main = app.conf2["config"]["tree"]
self.config_items = list(app.conf2['tree']) self.config_items = list(app.conf2["tree"])
# THIS MAKE A BUG !!!! self.plugin_loader = LoadPlugin(AlberoPlugins) # THIS MAKE A BUG !!!! self.plugin_loader = LoadPlugin(AlberoPlugins)
self.plugins = [ self.plugins = [
'init', "init",
'loop', "loop",
'hier', "hier",
] ]
# Auto init # Auto init
self.backends = self.config_items self.backends = self.config_items
def query(self, key=None, scope=None, trace=False): def query(self, key=None, scope=None, trace=False):
backends = self.get_backends(key=key, scope=scope, trace=trace) backends = self.get_backends(key=key, scope=scope, trace=trace)
ret = self.get_results(backends, trace=trace) ret = self.get_results(backends, trace=trace)
@ -169,25 +167,22 @@ class BackendsManager(Manager):
log.debug(f"Backend preprocessing of {len(backends)} elements") log.debug(f"Backend preprocessing of {len(backends)} elements")
for plugin in self.plugins: for plugin in self.plugins:
# backend_cls = plugin_loader.load('backend', plugin) # backend_cls = plugin_loader.load('backend', plugin)
plugin = plugin_loader.load( plugin = plugin_loader.load("backend", plugin)()
'backend', plugin
)()
log.debug(f"Run {plugin}") log.debug(f"Run {plugin}")
new_backend, _run = plugin.process(backends, _run) new_backend, _run = plugin.process(backends, _run)
assert(isinstance(new_backend, list)), f"Got: {new_backend}" assert isinstance(new_backend, list), f"Got: {new_backend}"
assert(isinstance(_run, dict)), f"Got: {_run}" assert isinstance(_run, dict), f"Got: {_run}"
backends = new_backend backends = new_backend
# pprint (backends) # pprint (backends)
for i in backends: for i in backends:
assert (i.get('engine')), f"Got: {i}" assert i.get("engine"), f"Got: {i}"
log.debug(f"Backend preprocessing made {len(backends)} elements") log.debug(f"Backend preprocessing made {len(backends)} elements")
return backends return backends
def get_results(self, backends, trace=False): def get_results(self, backends, trace=False):
# Prepare plugins # Prepare plugins
@ -199,16 +194,14 @@ class BackendsManager(Manager):
# print ("BACKKENNDNNDNDNDND") # print ("BACKKENNDNNDNDNDND")
# pprint(backend) # pprint(backend)
engine = plugin_loader.load( engine = plugin_loader.load("engine", backend["engine"])(
'engine', backend['engine'] backend, parent=self, app=self.app
)( )
backend,
parent=self, app=self.app)
log.debug(f"Run engine: {engine}") log.debug(f"Run engine: {engine}")
new_result = engine.process() new_result = engine.process()
assert(isinstance(new_result, list)), f"Got: {new_result}" assert isinstance(new_result, list), f"Got: {new_result}"
new_results.extend(new_result) new_results.extend(new_result)
# Filter out? Not here !new_results = [i for i in new_results if i['found'] ] # Filter out? Not here !new_results = [i for i in new_results if i['found'] ]
@ -217,10 +210,9 @@ class BackendsManager(Manager):
return new_results return new_results
class RulesManager(Manager): class RulesManager(Manager):
plugins_kind = ['strategy'] plugins_kind = ["strategy"]
_schema_props_new = { _schema_props_new = {
"rule": { "rule": {
@ -242,7 +234,6 @@ class RulesManager(Manager):
"optional": True, "optional": True,
# "enum": ["first", "last", "merge"], # "enum": ["first", "last", "merge"],
}, },
"trace": { "trace": {
"type": "boolean", "type": "boolean",
"default": False, "default": False,
@ -253,29 +244,26 @@ class RulesManager(Manager):
}, },
} }
_props_position = 'oneOf/1/properties' _props_position = "oneOf/1/properties"
_schema_props_default = { _schema_props_default = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"default": "", "default": "",
"$def": { "$def": {
"items": {}, "items": {},
}, },
"oneOf": [ "oneOf": [
{ {"type": "string", "default": "BLAAAAHHH"},
"type": "string",
"default": "BLAAAAHHH"
},
{ {
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"default": {}, "default": {},
"properties": {"$ref": "#/$defs/name"}, "properties": {"$ref": "#/$defs/name"},
}, },
] ],
} }
OLD_rule_schema = { OLD_rule_schema = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
"properties": { "properties": {
@ -298,59 +286,56 @@ class RulesManager(Manager):
}, },
], ],
}, },
},
} }
}
def __init__(self, app): def __init__(self, app):
self.app = app self.app = app
self.config_app = app.conf2['config']['app'] self.config_app = app.conf2["config"]["app"]
self.config_main = app.conf2['config']['rules'] self.config_main = app.conf2["config"]["rules"]
self.config_items = list(app.conf2['rules']) self.config_items = list(app.conf2["rules"])
def get_result(self, candidates, key=None, scope=None, trace=False, explain=False): def get_result(self, candidates, key=None, scope=None, trace=False, explain=False):
# trace=False # trace=False
rules = self.config_items rules = self.config_items
key = key or '' key = key or ""
# Filter out invalid candidates # Filter out invalid candidates
matched_candidates = [i for i in candidates if i['found'] == True] matched_candidates = [i for i in candidates if i["found"] == True]
if len(matched_candidates) == 0: if len(matched_candidates) == 0:
log.debug("No matched candidates") log.debug("No matched candidates")
return None return None
# Look for matching key in rules defiunitions # Look for matching key in rules defiunitions
regex_support = False regex_support = False
matched_rule = {} matched_rule = {}
if regex_support: if regex_support:
raise Exception("Not Implemented") raise Exception("Not Implemented")
else:
rule = [ i for i in rules if i.get('rule') == key ] rule = [i for i in rules if i.get("rule") == key]
if len(rule) == 0: if len(rule) == 0:
log.debug(f"No matched rule for {key}, applying defaults") log.debug(f"No matched rule for %s, applying defaults", key)
else: else:
matched_rule = rule[0] matched_rule = rule[0]
log.debug(f"Matcher rule for {key}: {matched_rule}") log.debug(f"Matcher rule for {key}: {matched_rule}")
matched_rule['trace'] = trace matched_rule["trace"] = trace
matched_rule['explain'] = explain matched_rule["explain"] = explain
schema_validate(matched_rule, self._schema_props_default) schema_validate(matched_rule, self._schema_props_default)
# Prepare plugins # Prepare plugins
assert(isinstance(matched_candidates, list)), f"Got: {matched_candidates}" assert isinstance(matched_candidates, list), f"Got: {matched_candidates}"
assert(isinstance(matched_rule, dict)), f"Got: {matched_rule}" assert isinstance(matched_rule, dict), f"Got: {matched_rule}"
strategy = matched_rule.get('strategy', 'schema') strategy = matched_rule.get("strategy", "schema")
log.debug(f"Key '{key}' matched rule '{rule}' with '{strategy}' strategy") log.debug(f"Key '{key}' matched rule '{rule}' with '{strategy}' strategy")
# Load plugin # Load plugin
log.debug(f"Run strategy: {strategy}") log.debug(f"Run strategy: {strategy}")
plugin_loader = LoadPlugin(AlberoPlugins) plugin_loader = LoadPlugin(AlberoPlugins)
strategy = plugin_loader.load('strategy', strategy = plugin_loader.load(
"strategy",
strategy, strategy,
)(parent=self, app=self.app) )(parent=self, app=self.app)
new_result = strategy.process(matched_candidates, matched_rule) new_result = strategy.process(matched_candidates, matched_rule)

23
albero/plugin/backend/hier.py
View File

@ -1,6 +1,5 @@
import copy import copy
# from pathlib import Path # from pathlib import Path
# from albero.utils import render_template # from albero.utils import render_template
# from albero.plugin.common import PluginBackendClass # from albero.plugin.common import PluginBackendClass
@ -13,8 +12,10 @@ import copy
from albero.plugin.common import PluginBackendClass from albero.plugin.common import PluginBackendClass
from pprint import pprint from pprint import pprint
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class Plugin(PluginBackendClass): class Plugin(PluginBackendClass):
_plugin_name = "hier" _plugin_name = "hier"
@ -38,7 +39,7 @@ class Plugin(PluginBackendClass):
{"type": "null"}, {"type": "null"},
{"type": "string"}, {"type": "string"},
{"type": "array"}, {"type": "array"},
] ],
}, },
"var": { "var": {
"type": "string", "type": "string",
@ -57,12 +58,10 @@ class Plugin(PluginBackendClass):
}, },
}, },
}, },
] ],
} }
} }
def process(self, backends: list, ctx: dict) -> (list, dict): def process(self, backends: list, ctx: dict) -> (list, dict):
new_backends = [] new_backends = []
@ -79,13 +78,13 @@ class Plugin(PluginBackendClass):
hier_var = plugin_config.get("var", "item") hier_var = plugin_config.get("var", "item")
hier_sep = plugin_config.get("separator", "/") hier_sep = plugin_config.get("separator", "/")
if isinstance(hier_data, str): if isinstance(hier_data, str):
hier_data = cand['_run']['scope'].get(hier_data, None) hier_data = cand["_run"]["scope"].get(hier_data, None)
# Build a new list # Build a new list
if isinstance(hier_data, str): if isinstance(hier_data, str):
r = hier_data.split(hier_sep) r = hier_data.split(hier_sep)
assert (isinstance(r, list)), f"Got: {r}" assert isinstance(r, list), f"Got: {r}"
ret1 = [] ret1 = []
for index, part in enumerate(r): for index, part in enumerate(r):
@ -93,7 +92,7 @@ class Plugin(PluginBackendClass):
try: try:
prefix = ret1[index - 1] prefix = ret1[index - 1]
except IndexError: except IndexError:
prefix = f'{hier_sep}' prefix = f"{hier_sep}"
prefix = "" prefix = ""
item = f"{prefix}{part}{hier_sep}" item = f"{prefix}{part}{hier_sep}"
ret1.append(item) ret1.append(item)
@ -106,11 +105,9 @@ class Plugin(PluginBackendClass):
"hier_value": item, "hier_value": item,
"hier_var": hier_var, "hier_var": hier_var,
} }
_cand['_run']['hier'] = run _cand["_run"]["hier"] = run
_cand['_run']['scope'][hier_var] = item _cand["_run"]["scope"][hier_var] = item
ret2.append(_cand) ret2.append(_cand)
new_backends.extend(ret2) new_backends.extend(ret2)
return new_backends, ctx return new_backends, ctx

14
albero/plugin/backend/init.py
View File

@ -1,18 +1,18 @@
from albero.plugin.common import PluginBackendClass from albero.plugin.common import PluginBackendClass
from pprint import pprint from pprint import pprint
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
import copy import copy
class Plugin(PluginBackendClass): class Plugin(PluginBackendClass):
_plugin_name = "init" _plugin_name = "init"
_schema_props_new = None _schema_props_new = None
default_engine = 'jerakia' default_engine = "jerakia"
def process(self, backends: list, ctx: dict) -> (list, dict): def process(self, backends: list, ctx: dict) -> (list, dict):
@ -25,13 +25,11 @@ class Plugin(PluginBackendClass):
if not isinstance(item, dict): if not isinstance(item, dict):
item = default item = default
item['engine'] = item.get('engine', self.default_engine ) item["engine"] = item.get("engine", self.default_engine)
item['_run'] = copy.deepcopy(ctx) item["_run"] = copy.deepcopy(ctx)
item['_run']['backend'] = { item["_run"]["backend"] = {
"index": index, "index": index,
} }
new_backends.append(item) new_backends.append(item)
return new_backends, ctx return new_backends, ctx

26
albero/plugin/backend/loop.py
View File

@ -1,5 +1,3 @@
import copy import copy
from pathlib import Path from pathlib import Path
from albero.utils import render_template from albero.utils import render_template
@ -14,9 +12,11 @@ import textwrap
class Plugin(PluginBackendClass): class Plugin(PluginBackendClass):
_plugin_name = "loop" _plugin_name = "loop"
_plugin_help = """ _plugin_help = (
"""
This module helps to loop over a backend This module helps to loop over a backend
""", """,
)
_schema_props_new = { _schema_props_new = {
"loop": { "loop": {
"description": _plugin_help, "description": _plugin_help,
@ -47,8 +47,6 @@ class Plugin(PluginBackendClass):
"loop": None, "loop": None,
"comment": "Disable this module, no loop will operate", "comment": "Disable this module, no loop will operate",
}, },
# "loop": { # "loop": {
# "var": "my_var", # "var": "my_var",
# }, # },
@ -71,7 +69,6 @@ class Plugin(PluginBackendClass):
"default": {}, "default": {},
"title": "Complete config", "title": "Complete config",
"description": "", "description": "",
"properties": { "properties": {
"data": { "data": {
"default": None, "default": None,
@ -94,7 +91,7 @@ class Plugin(PluginBackendClass):
"title": "Hardcoded list", "title": "Hardcoded list",
"description": "Simply enter the list of value to be iterated to.", "description": "Simply enter the list of value to be iterated to.",
}, },
] ],
}, },
"var": { "var": {
"type": "string", "type": "string",
@ -115,14 +112,12 @@ class Plugin(PluginBackendClass):
"title": "Disable", "title": "Disable",
"description": "If set to null, it disable the module", "description": "If set to null, it disable the module",
}, },
] ],
} }
} }
def process(self, backends: list, ctx: dict) -> (list, dict): def process(self, backends: list, ctx: dict) -> (list, dict):
new_backends = [] new_backends = []
for cand in backends: for cand in backends:
cand = dict(cand) cand = dict(cand)
@ -137,8 +132,8 @@ class Plugin(PluginBackendClass):
# Retrieve config data # Retrieve config data
loop_var = loop_config.get("var", "item") loop_var = loop_config.get("var", "item")
if isinstance(loop_data, str): if isinstance(loop_data, str):
loop_data = cand['_run']['scope'].get(loop_data, None) loop_data = cand["_run"]["scope"].get(loop_data, None)
assert (isinstance(loop_data, list)), f"Got: {loop_data}" assert isinstance(loop_data, list), f"Got: {loop_data}"
# Build a new list # Build a new list
ret = [] ret = []
@ -149,14 +144,11 @@ class Plugin(PluginBackendClass):
"loop_value": item, "loop_value": item,
"loop_var": loop_var, "loop_var": loop_var,
} }
_cand['_run']['loop'] = run _cand["_run"]["loop"] = run
_cand['_run']['scope'][loop_var] = item _cand["_run"]["scope"][loop_var] = item
# _cand.scope[loop_var] = item # _cand.scope[loop_var] = item
ret.append(_cand) ret.append(_cand)
new_backends.extend(ret) new_backends.extend(ret)
return new_backends, ctx return new_backends, ctx

45
albero/plugin/common.py
View File

@ -1,4 +1,3 @@
# from box import Box # from box import Box
import textwrap import textwrap
from pprint import pprint from pprint import pprint
@ -9,6 +8,7 @@ import yaml
import json import json
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
from albero.utils import schema_validate from albero.utils import schema_validate
@ -16,7 +16,7 @@ import copy
# Candidate Classes # Candidate Classes
# ============================= # =============================
class Candidate(): class Candidate:
engine = None engine = None
found = False found = False
data = None data = None
@ -41,13 +41,9 @@ class Candidate():
return d return d
# Generic Classes # Generic Classes
# ============================= # =============================
class PluginClass(): class PluginClass:
_plugin_type = "none" _plugin_type = "none"
_plugin_value = None _plugin_value = None
@ -57,7 +53,7 @@ class PluginClass():
"engine": { "engine": {
"type": "string", "type": "string",
# TODO: Fix this ug # TODO: Fix this ug
"default": "jerakia" "default": "jerakia",
}, },
"value": {}, "value": {},
} }
@ -65,7 +61,7 @@ class PluginClass():
def __repr__(self): def __repr__(self):
kind = self._plugin_type kind = self._plugin_type
name = self._plugin_name name = self._plugin_name
value = getattr(self, 'value', 'NO VALUE') value = getattr(self, "value", "NO VALUE")
return f"{kind}.{name}:{value}" return f"{kind}.{name}:{value}"
def __init__(self, config=None, parent=None, app=None): def __init__(self, config=None, parent=None, app=None):
@ -79,21 +75,25 @@ class PluginClass():
def _init(self): def _init(self):
pass pass
def _validate(self): def _validate(self):
pass pass
class PluginBackendClass(PluginClass): class PluginBackendClass(PluginClass):
_plugin_type = "backend" _plugin_type = "backend"
def _init(self): def _init(self):
pass pass
class PluginStrategyClass(PluginClass): class PluginStrategyClass(PluginClass):
_plugin_type = "strategy" _plugin_type = "strategy"
def _init(self): def _init(self):
pass pass
class PluginEngineClass(PluginClass): class PluginEngineClass(PluginClass):
_plugin_type = "engine" _plugin_type = "engine"
@ -101,7 +101,6 @@ class PluginEngineClass(PluginClass):
"value": { "value": {
"default": "UNSET", "default": "UNSET",
}, },
#### SHOULD NOT BE HERE #### SHOULD NOT BE HERE
"hier": { "hier": {
"additionalProperties": True, "additionalProperties": True,
@ -118,7 +117,7 @@ class PluginEngineClass(PluginClass):
{"type": "null"}, {"type": "null"},
{"type": "string"}, {"type": "string"},
{"type": "array"}, {"type": "array"},
] ],
}, },
"separator": { "separator": {
"type": "string", "type": "string",
@ -130,10 +129,9 @@ class PluginEngineClass(PluginClass):
"default": False, "default": False,
"optional": True, "optional": True,
}, },
},
},
} }
}
}
# Default plugin API Methods # Default plugin API Methods
# ===================== # =====================
@ -143,13 +141,13 @@ class PluginEngineClass(PluginClass):
def _validate(self): def _validate(self):
# Build schema # Build schema
schema_keys = [a for a in dir(self) if a.startswith('_schema_props_')] schema_keys = [a for a in dir(self) if a.startswith("_schema_props_")]
props = {} props = {}
for key in schema_keys: for key in schema_keys:
schema = getattr(self, key) schema = getattr(self, key)
props.update(schema) props.update(schema)
self.schema = { self.schema = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"properties": props, "properties": props,
@ -159,7 +157,6 @@ class PluginEngineClass(PluginClass):
self.config = schema_validate(self.config, self.schema) self.config = schema_validate(self.config, self.schema)
return True return True
# Public Methods # Public Methods
# ===================== # =====================
def dump(self): def dump(self):
@ -181,7 +178,8 @@ class PluginEngineClass(PluginClass):
# File plugins Extensions # File plugins Extensions
# ============================= # =============================
class PluginFileGlob():
class PluginFileGlob:
_schema_props_glob = { _schema_props_glob = {
"glob": { "glob": {
@ -200,7 +198,7 @@ class PluginFileGlob():
"default": ["yml", "yaml"], "default": ["yml", "yaml"],
"optional": True, "optional": True,
}, },
} },
} }
} }
@ -208,13 +206,17 @@ class PluginFileGlob():
# DIRECT CALL TO APP< TOFIX # DIRECT CALL TO APP< TOFIX
app_config = self.app.conf2 app_config = self.app.conf2
root = app_config.get("default", {}).get("config", {}).get("root", f"{Path.cwd()}/tree") root = (
app_config.get("default", {})
.get("config", {})
.get("root", f"{Path.cwd()}/tree")
)
# root = self.app.conf2.config.app.root # root = self.app.conf2.config.app.root
# TOFIX print ("ITEM! %s" % type(root)) # TOFIX print ("ITEM! %s" % type(root))
# TOFIX print ("ITEM2 %s" % self.app.conf2.config.app.root) # TOFIX print ("ITEM2 %s" % self.app.conf2.config.app.root)
glob_config = self.config.get("glob", {}) glob_config = self.config.get("glob", {})
glob_file = glob_config['file'] glob_file = glob_config["file"]
# glob_ext = glob_config['ext'] # glob_ext = glob_config['ext']
item = Path(root) / Path(item) / Path(glob_file) item = Path(root) / Path(item) / Path(glob_file)
@ -227,4 +229,3 @@ class PluginFileGlob():
log.debug(f"Matched file for glob '{item}': {files}") log.debug(f"Matched file for glob '{item}': {files}")
return files return files

38
albero/plugin/engine/jerakia.py
View File

@ -1,4 +1,3 @@
from pathlib import Path from pathlib import Path
from albero.utils import render_template from albero.utils import render_template
from albero.plugin.common import PluginEngineClass, PluginFileGlob, Candidate from albero.plugin.common import PluginEngineClass, PluginFileGlob, Candidate
@ -10,6 +9,7 @@ import textwrap
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class FileCandidate(Candidate): class FileCandidate(Candidate):
path = None path = None
@ -24,10 +24,9 @@ class FileCandidate(Candidate):
return super()._report_data(data) return super()._report_data(data)
class Plugin(PluginEngineClass, PluginFileGlob): class Plugin(PluginEngineClass, PluginFileGlob):
_plugin_name = 'jerakia' _plugin_name = "jerakia"
### OLD ### OLD
_plugin_engine = "jerakia" _plugin_engine = "jerakia"
@ -42,23 +41,23 @@ class Plugin(PluginEngineClass, PluginFileGlob):
"type": "array", "type": "array",
"items": { "items": {
"type": "string", "type": "string",
} },
}, },
] ]
} }
} }
def _init(self): def _init(self):
paths = self.config.get('path', self.config.get('value')) paths = self.config.get("path", self.config.get("value"))
if isinstance(paths, str): if isinstance(paths, str):
paths = [paths] paths = [paths]
elif isinstance(paths, list): elif isinstance(paths, list):
pass pass
else: else:
raise Exception (f"Unsupported path value, expected str or dict, got: {paths} in {self.config}") raise Exception(
f"Unsupported path value, expected str or dict, got: {paths} in {self.config}"
)
self.paths = paths self.paths = paths
self.value = paths self.value = paths
@ -78,7 +77,6 @@ class Plugin(PluginEngineClass, PluginFileGlob):
return ret return ret
def _show_paths(self, scope): def _show_paths(self, scope):
parsed = self._preprocess(scope) parsed = self._preprocess(scope)
@ -93,14 +91,12 @@ class Plugin(PluginEngineClass, PluginFileGlob):
return ret3 return ret3
def process(self): def process(self):
# scope = self.scope # scope = self.scope
# pprint (self.config) # pprint (self.config)
scope = dict(self.config['_run']['scope']) scope = dict(self.config["_run"]["scope"])
key = self.config['_run']['key'] key = self.config["_run"]["key"]
assert isinstance(scope, dict), f"Got: {scope}" assert isinstance(scope, dict), f"Got: {scope}"
assert isinstance(key, (str, type(None))), f"Got: {key}" assert isinstance(key, (str, type(None))), f"Got: {key}"
@ -125,13 +121,13 @@ class Plugin(PluginEngineClass, PluginFileGlob):
# Build result object # Build result object
result = {} result = {}
result['run'] = { result["run"] = {
'path': path, "path": path,
'rel_path': str(Path(path).relative_to(Path.cwd())), "rel_path": str(Path(path).relative_to(Path.cwd())),
} }
result['parent'] = self.config result["parent"] = self.config
result['data'] = data result["data"] = data
result['found'] = found result["found"] = found
ret.append(result) ret.append(result)
@ -183,7 +179,3 @@ class Plugin(PluginEngineClass, PluginFileGlob):
# #log.debug(f"Found value: {ret_obj}") # #log.debug(f"Found value: {ret_obj}")
# ret_obj.found = found # ret_obj.found = found
# ret.append(ret_obj) # ret.append(ret_obj)

3
albero/plugin/strategy/last.py
View File

@ -1,9 +1,9 @@
import logging import logging
from albero.plugin.common import PluginStrategyClass from albero.plugin.common import PluginStrategyClass
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class Plugin(PluginStrategyClass): class Plugin(PluginStrategyClass):
_plugin_name = "last" _plugin_name = "last"
@ -12,4 +12,3 @@ class Plugin(PluginStrategyClass):
def process(self, candidates: list, rule=None) -> (list, dict): def process(self, candidates: list, rule=None) -> (list, dict):
return candidates[-1] return candidates[-1]

134
albero/plugin/strategy/schema.py
View File

@ -1,5 +1,3 @@
import logging import logging
from albero.plugin.common import PluginStrategyClass from albero.plugin.common import PluginStrategyClass
from albero.utils import schema_validate, str_ellipsis from albero.utils import schema_validate, str_ellipsis
@ -11,6 +9,7 @@ from pprint import pprint
from jsonmerge import Merger from jsonmerge import Merger
from prettytable import PrettyTable from prettytable import PrettyTable
class Plugin(PluginStrategyClass): class Plugin(PluginStrategyClass):
_plugin_name = "schema" _plugin_name = "schema"
@ -40,7 +39,7 @@ class Plugin(PluginStrategyClass):
{"type": "null"}, {"type": "null"},
{"type": "string"}, {"type": "string"},
{"type": "array"}, {"type": "array"},
] ],
}, },
"var": { "var": {
"type": "string", "type": "string",
@ -49,12 +48,12 @@ class Plugin(PluginStrategyClass):
}, },
}, },
}, },
] ],
} }
} }
default_merge_schema = { default_merge_schema = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"oneOf": [ "oneOf": [
{ {
"type": "array", "type": "array",
@ -88,76 +87,123 @@ class Plugin(PluginStrategyClass):
], ],
} }
def process(self, candidates: list, rule=None) -> (list, dict): def process(self, candidates: list, rule=None) -> (list, dict):
trace = rule['trace'] trace = rule["trace"]
explain = rule['explain'] explain = rule["explain"]
schema = rule.get('schema', None) or self.default_merge_schema schema = rule.get("schema", None) or self.default_merge_schema
merger = Merger(schema) merger = Merger(schema)
t = PrettyTable() t = PrettyTable()
t1 = PrettyTable() t1 = PrettyTable()
new_candidate = None new_candidate = None
for index, item in enumerate(candidates): for index, item in enumerate(candidates):
new_value = item['data'] new_value = item["data"]
result = merger.merge(new_candidate, new_value) result = merger.merge(new_candidate, new_value)
backend_info = dict(item['parent']) backend_info = dict(item["parent"])
backend_run = backend_info.pop("_run") backend_run = backend_info.pop("_run")
if explain: if explain:
t1.add_row([ t1.add_row(
[
index, index,
'\nBackendRun: ' + str_ellipsis(json.dumps( "\nBackendRun: "
+ str_ellipsis(
json.dumps(
backend_run, backend_run,
default=lambda o: '<not serializable>', indent=2), 70), default=lambda o: "<not serializable>",
'\nRuleRun: ' + str_ellipsis(json.dumps( indent=2,
item['run'], ),
default=lambda o: '<not serializable>', indent=2), 70), 70,
'---\nResult: ' + str_ellipsis(json.dumps( ),
result, "\nRuleRun: "
default=lambda o: '<not serializable>', indent=2), 70), + str_ellipsis(
]) json.dumps(
item["run"],
default=lambda o: "<not serializable>",
indent=2,
),
70,
),
"---\nResult: "
+ str_ellipsis(
json.dumps(
result, default=lambda o: "<not serializable>", indent=2
),
70,
),
]
)
if trace: if trace:
t.add_row([ t.add_row(
[
index, index,
'---\nBackendConfig: ' + str_ellipsis(json.dumps( "---\nBackendConfig: "
+ str_ellipsis(
json.dumps(
backend_info, backend_info,
default=lambda o: '<not serializable>', indent=2), 70) + default=lambda o: "<not serializable>",
'\nBackendRun: ' + str_ellipsis(json.dumps( indent=2,
),
70,
)
+ "\nBackendRun: "
+ str_ellipsis(
json.dumps(
backend_run, backend_run,
default=lambda o: '<not serializable>', indent=2), 70), default=lambda o: "<not serializable>",
indent=2,
'---\nRuleConfig: ' + str_ellipsis(json.dumps( ),
rule, 70,
default=lambda o: '<not serializable>', indent=2), 70) + ),
'\nRuleRun: ' + str_ellipsis(json.dumps( "---\nRuleConfig: "
item['run'], + str_ellipsis(
default=lambda o: '<not serializable>', indent=2), 70) + json.dumps(
rule, default=lambda o: "<not serializable>", indent=2
),
70,
)
+ "\nRuleRun: "
+ str_ellipsis(
json.dumps(
item["run"],
default=lambda o: "<not serializable>",
indent=2,
),
70,
)
+
#'\nSource: ' + str_ellipsis(json.dumps( #'\nSource: ' + str_ellipsis(json.dumps(
# new_candidate, # new_candidate,
# default=lambda o: '<not serializable>', indent=2), 70) + # default=lambda o: '<not serializable>', indent=2), 70) +
'\nNew data: ' + str_ellipsis(json.dumps( "\nNew data: "
+ str_ellipsis(
json.dumps(
new_value, new_value,
default=lambda o: '<not serializable>', indent=2), 70), default=lambda o: "<not serializable>",
indent=2,
'---\nResult: ' + str_ellipsis(json.dumps( ),
result, 70,
default=lambda o: '<not serializable>', indent=2), 70), ),
"---\nResult: "
+ str_ellipsis(
json.dumps(
result, default=lambda o: "<not serializable>", indent=2
),
70,
),
] ]
) )
new_candidate = result new_candidate = result
if trace: if trace:
t.field_names = ["Index", "Backend", "Rule", "Data"] t.field_names = ["Index", "Backend", "Rule", "Data"]
t.align = 'l' t.align = "l"
print(t) print(t)
if explain: if explain:
t1.field_names = ["Index", "Backend", "Rule", "Data"] t1.field_names = ["Index", "Backend", "Rule", "Data"]
t1.align = 'l' t1.align = "l"
print('Explain:\n' + repr(t1)) print("Explain:\n" + repr(t1))
return new_candidate return new_candidate

9
albero/query.py
View File

@ -12,19 +12,20 @@ from pprint import pprint
from albero.managers import BackendsManager, RulesManager from albero.managers import BackendsManager, RulesManager
from albero.utils import schema_validate from albero.utils import schema_validate
import anyconfig import anyconfig
# from box import Box # from box import Box
from pathlib import Path from pathlib import Path
import logging import logging
log = logging.getLogger(__name__)
log = logging.getLogger(__name__)
# Query # Query
########################################## ##########################################
class Query():
class Query:
def __init__(self, app): def __init__(self, app):
self.app = app self.app = app
@ -43,9 +44,7 @@ class Query():
ret = {} ret = {}
for i in dir(self): for i in dir(self):
if not i.startswith('_'): if not i.startswith("_"):
ret[i] = getattr(self, i) ret[i] = getattr(self, i)
pprint(ret) pprint(ret)

22
albero/utils.py
View File

@ -1,4 +1,3 @@
from pathlib import Path from pathlib import Path
from jinja2 import Template from jinja2 import Template
import yaml import yaml
@ -10,6 +9,7 @@ import collections
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -62,12 +62,14 @@ log = logging.getLogger(__name__)
# Utils Methods # Utils Methods
# ===================== # =====================
def render_template(path, params): def render_template(path, params):
"""Render template for a given string""" """Render template for a given string"""
assert (isinstance(params, dict)), f"Got: {params}" assert isinstance(params, dict), f"Got: {params}"
t = Template(path) t = Template(path)
return t.render(**params) return t.render(**params)
# def read_file(file): # def read_file(file):
# with open(file, 'r') as f: # with open(file, 'r') as f:
# data = f.read().replace('\n', '') # data = f.read().replace('\n', '')
@ -99,6 +101,7 @@ def render_template(path, params):
# Schema Methods # Schema Methods
# ===================== # =====================
def _extend_with_default(validator_class): def _extend_with_default(validator_class):
validate_properties = validator_class.VALIDATORS["properties"] validate_properties = validator_class.VALIDATORS["properties"]
@ -110,14 +113,18 @@ def _extend_with_default(validator_class):
try: try:
for error in validate_properties( for error in validate_properties(
validator, properties, instance, schema, validator,
properties,
instance,
schema,
): ):
continue continue
except Exception as e: except Exception as e:
print("CATCHED2222 ", e) print("CATCHED2222 ", e)
return validators.extend( return validators.extend(
validator_class, {"properties" : set_defaults}, validator_class,
{"properties": set_defaults},
) )
@ -130,18 +137,19 @@ def schema_validate(config, schema):
except Exception as e: except Exception as e:
print(e) print(e)
p = list(collections.deque(e.schema_path)) p = list(collections.deque(e.schema_path))
p = '/'.join([ str(i) for i in p ]) p = "/".join([str(i) for i in p])
p = f"schema/{p}" p = f"schema/{p}"
raise Exception( raise Exception(
f"Failed validating {p} for resource with content: {config} with !!!!!! schema: {schema}" f"Failed validating {p} for resource with content: {config} with !!!!!! schema: {schema}"
) )
return config return config
def str_ellipsis(txt, length=120): def str_ellipsis(txt, length=120):
txt = str(txt) txt = str(txt)
ret = [] ret = []
for string in txt.splitlines(): for string in txt.splitlines():
string = (string[:length - 4 ] + ' ...') if len(string) > length else string string = (string[: length - 4] + " ...") if len(string) > length else string
ret.append(string) ret.append(string)
ret = '\n'.join(ret) ret = "\n".join(ret)
return ret return ret