pro-barbu-foss/kheops
pro-barbu-foss/kheops
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Lint: Whole codebase with black

Browse Source
This commit is contained in:
mrjk 2022-01-15 22:54:13 -05:00
parent 3ab48fab3a
commit 76925684c7
12 changed files with 629 additions and 606 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
albero/app.py
View File

@ -12,26 +12,28 @@ from pprint import pprint
from albero.query import Query from albero.query import Query
from albero.utils import schema_validate from albero.utils import schema_validate
import anyconfig import anyconfig
# from box import Box # from box import Box
from pathlib import Path from pathlib import Path
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class App(): class App:
schema = { schema = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
"default": {}, "default": {},
"$def" :{ "$def": {
'backends_items': {}, "backends_items": {},
'backends_config': {}, "backends_config": {},
'rules_items': {}, "rules_items": {},
'rules_config': {}, "rules_config": {},
}, },
"patternProperties": { "patternProperties": {
".*": { ".*": {
"type": "object", "type": "object",
@ -39,53 +41,51 @@ class App():
"additionalProperties": False, "additionalProperties": False,
"properties": { "properties": {
"config": { "config": {
"type": "object", "type": "object",
"default": {}, "default": {},
"additionalProperties": False, "additionalProperties": False,
"properties": { "properties": {
"app": { "app": {
"type": "object",
"default": {},
"additionalProperties": False,
"properties": {
"root": {
"type": "string",
"default": None,
},
},
},
"tree": {
#"additionalProperties": False,
"type": "object",
"default": {},
},
"rules": {
"type": "object",
"default": {},
},
},
},
"tree": {
"type": "array",
"default": [],
"items": {
"type": "object", "type": "object",
"properties": { "$ref": "#/$defs/backends_items" }, "default": {},
"additionalProperties": False,
"properties": {
"root": {
"type": "string",
"default": None,
},
}, },
}, },
"rules": { "tree": {
"type": "array", # "additionalProperties": False,
"default": [], "type": "object",
# "arrayItem": { "$ref": "#/$defs/rules_items" }, "default": {},
},
"rules": {
"type": "object",
"default": {},
},
}, },
}, },
"tree": {
"type": "array",
"default": [],
"items": {
"type": "object",
"properties": {"$ref": "#/$defs/backends_items"},
},
},
"rules": {
"type": "array",
"default": [],
# "arrayItem": { "$ref": "#/$defs/rules_items" },
},
}, },
} },
} },
}
def __init__(self, config="albero.yml", namespace='default'): def __init__(self, config="albero.yml", namespace="default"):
conf2 = anyconfig.load(config) conf2 = anyconfig.load(config)
# Validate configuration # Validate configuration
@ -93,27 +93,26 @@ class App():
try: try:
conf2 = conf2[namespace] conf2 = conf2[namespace]
except KeyError: except KeyError:
log.error (f"Can't find namespace '{namespace}' in config '{config}'") log.error(f"Can't find namespace '{namespace}' in config '{config}'")
sys.exit(1) sys.exit(1)
# Init # Init
if not conf2['config']['app']['root']: if not conf2["config"]["app"]["root"]:
conf2['config']['app']['root'] = Path(config).parent conf2["config"]["app"]["root"] = Path(config).parent
else: else:
conf2['config']['app']['root'] = Path(conf2['config']['app']['root']) conf2["config"]["app"]["root"] = Path(conf2["config"]["app"]["root"])
# Finish # Finish
self.conf2 = dict(conf2) self.conf2 = dict(conf2)
def lookup(self, key=None, policy=None, scope=None, trace=False, explain=False): def lookup(self, key=None, policy=None, scope=None, trace=False, explain=False):
log.debug(f"Lookup key {key} with scope: {scope}") log.debug(f"Lookup key {key} with scope: {scope}")
q = Query(app = self) q = Query(app=self)
r = q.exec(key=key, scope=scope , policy=policy, trace=trace, explain=explain) r = q.exec(key=key, scope=scope, policy=policy, trace=trace, explain=explain)
print ("=== Query Result ===")
print(anyconfig.dumps(r, ac_parser='yaml'))
print ("=== Query Result ===")
print("=== Query Result ===")
print(anyconfig.dumps(r, ac_parser="yaml"))
print("=== Query Result ===")
def dump_schema(self): def dump_schema(self):
@ -125,10 +124,7 @@ class App():
r2 = RulesManager.get_schema(AlberoPlugins) r2 = RulesManager.get_schema(AlberoPlugins)
d = self.schema d = self.schema
d['patternProperties']['.*']['properties'] ['tree']['items']['properties'] = r1 d["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = r1
d['patternProperties']['.*']['properties'] ['tree']['items'] = r2 d["patternProperties"][".*"]["properties"]["tree"]["items"] = r2
print(json.dumps(d, indent=2)) print(json.dumps(d, indent=2))

40
albero/cli.py
View File

@ -18,6 +18,7 @@ sys.path.append("/home/jez/prj/bell/training/tiger-ansible/ext/ansible-tree")
import albero.app as Albero import albero.app as Albero
class CmdApp: class CmdApp:
"""Main CmdApp""" """Main CmdApp"""
@ -99,7 +100,9 @@ class CmdApp:
"""Prepare command line""" """Prepare command line"""
# Manage main parser # Manage main parser
parser = argparse.ArgumentParser(description="Albero, to lookup hierarchical data") parser = argparse.ArgumentParser(
description="Albero, to lookup hierarchical data"
)
parser.add_argument( parser.add_argument(
"-v", "--verbose", action="count", default=0, help="Increase verbosity" "-v", "--verbose", action="count", default=0, help="Increase verbosity"
) )
@ -112,13 +115,19 @@ class CmdApp:
# Manage command: demo # Manage command: demo
add_p = subparsers.add_parser("lookup") add_p = subparsers.add_parser("lookup")
add_p.add_argument("-n", "--namespace", help="Namespace name", default='default') add_p.add_argument(
add_p.add_argument("-f", "--file", help="File with params as dict. Can be stdin - .") "-n", "--namespace", help="Namespace name", default="default"
add_p.add_argument("-e", "--scope", dest="scope_param", action="append", default=[]) )
add_p.add_argument(
"-f", "--file", help="File with params as dict. Can be stdin - ."
)
add_p.add_argument(
"-e", "--scope", dest="scope_param", action="append", default=[]
)
add_p.add_argument("-p", "--policy") add_p.add_argument("-p", "--policy")
add_p.add_argument("-t", "--trace", action="store_true") add_p.add_argument("-t", "--trace", action="store_true")
add_p.add_argument("-x", "--explain", action="store_true") add_p.add_argument("-x", "--explain", action="store_true")
add_p.add_argument("key", default=None, nargs="*") add_p.add_argument("key", default=None, nargs="*")
# Manage command: demo # Manage command: demo
add_p = subparsers.add_parser("demo") add_p = subparsers.add_parser("demo")
@ -150,9 +159,9 @@ class CmdApp:
def cli_lookup(self): def cli_lookup(self):
"""Display how to use logging""" """Display how to use logging"""
config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml' config = "/home/jez/prj/bell/training/tiger-ansible/tree.yml"
# self.log.debug(f"Command line vars: {vars(self.args)}") # self.log.debug(f"Command line vars: {vars(self.args)}")
keys = self.args.key or [None] keys = self.args.key or [None]
# Parse payload from enf file: # Parse payload from enf file:
@ -162,7 +171,7 @@ class CmdApp:
# Parse cli params # Parse cli params
for i in self.args.scope_param: for i in self.args.scope_param:
r = i.split('=') r = i.split("=")
if len(r) != 2: if len(r) != 2:
raise Exception("Malformed params") raise Exception("Malformed params")
new_params[r[0]] = r[1] new_params[r[0]] = r[1]
@ -171,18 +180,19 @@ class CmdApp:
app = Albero.App(config=config, namespace=self.args.namespace) app = Albero.App(config=config, namespace=self.args.namespace)
for key in keys: for key in keys:
app.lookup(key=key, app.lookup(
scope=new_params, key=key,
trace=self.args.trace, scope=new_params,
explain=self.args.explain trace=self.args.trace,
) explain=self.args.explain,
)
def cli_schema(self): def cli_schema(self):
"""Display configuration schema""" """Display configuration schema"""
config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml' config = "/home/jez/prj/bell/training/tiger-ansible/tree.yml"
app = Albero.App(config=config) #, namespace=self.args.namespace) app = Albero.App(config=config) # , namespace=self.args.namespace)
app.dump_schema() app.dump_schema()

319
albero/managers.py
View File

@ -1,33 +1,22 @@
import dpath.util import dpath.util
import copy
import json
import textwrap
from prettytable import PrettyTable
from pathlib import Path
# from box import Box
from jsonmerge import Merger
import re
import logging import logging
from pprint import pprint from pprint import pprint
import collections
from albero.utils import schema_validate, str_ellipsis from albero.utils import schema_validate
import albero.plugin as AlberoPlugins import albero.plugin as AlberoPlugins
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class LoadPlugin(): class LoadPlugin:
def __init__(self, plugins): def __init__(self, plugins):
self.plugins = plugins self.plugins = plugins
def load(self, kind, name): def load(self, kind, name):
assert (isinstance(name, str)), f"Got: {name}" assert isinstance(name, str), f"Got: {name}"
# Get plugin kind # Get plugin kind
try: try:
@ -41,18 +30,25 @@ class LoadPlugin():
except Exception as e: except Exception as e:
raise Exception(f"Unknown module '{kind}.{name}': {e}") raise Exception(f"Unknown module '{kind}.{name}': {e}")
assert (hasattr(plugin_cls, 'Plugin')), f'Plugin {kind}/{name} is not a valid plugin' assert hasattr(
plugin_cls, "Plugin"
), f"Plugin {kind}/{name} is not a valid plugin"
# Return plugin Classe # Return plugin Classe
return plugin_cls.Plugin return plugin_cls.Plugin
class Manager():
class Manager:
"""Generic manager class"""
plugins_kind = [] plugins_kind = []
_schema_props_default = None _schema_props_default = None
_schema_props_new = None
_props_position = None
@classmethod @classmethod
def get_schema(cls, plugins_db): def get_schema(cls, plugins_db):
"""Retrieve configuration schema"""
# Properties # Properties
ret3 = {} ret3 = {}
@ -60,15 +56,17 @@ class Manager():
# ret[kind] = {} # ret[kind] = {}
plugin_kind = getattr(plugins_db, kind) plugin_kind = getattr(plugins_db, kind)
for plugin_name in [i for i in dir(plugin_kind) if not i.startswith('_')]: for plugin_name in [i for i in dir(plugin_kind) if not i.startswith("_")]:
plugin = getattr(plugin_kind, plugin_name) plugin = getattr(plugin_kind, plugin_name)
plugin_cls = getattr(plugin, 'Plugin', None) plugin_cls = getattr(plugin, "Plugin", None)
if plugin_cls: if plugin_cls:
schema_props = getattr(plugin_cls, '_schema_props_new', 'MISSING ITEM') schema_props = getattr(
plugin_cls, "_schema_props_new", "MISSING ITEM"
)
if schema_props: if schema_props:
# ret[kind][plugin_name] = schema_props # ret[kind][plugin_name] = schema_props
ret3.update( schema_props ) ret3.update(schema_props)
ret3.update( cls._schema_props_new ) ret3.update(cls._schema_props_new)
# Injection # Injection
ret1 = cls._schema_props_default ret1 = cls._schema_props_default
@ -79,76 +77,76 @@ class Manager():
class BackendsManager(Manager): class BackendsManager(Manager):
"""Backend Manager"""
plugins_kind = ['engine', 'backend'] plugins_kind = ["engine", "backend"]
_schema_props_new = { _schema_props_new = {
"engine": { "engine": {
"type": "string", "type": "string",
"default": "jerakia", "default": "jerakia",
"optional": False, "optional": False,
}, },
"value": { "value": {
"default": 'UNSET', "default": "UNSET",
"optional": False, "optional": False,
}, },
} }
_props_position = 'oneOf/0/properties' _props_position = "oneOf/0/properties"
_schema_props_default = { _schema_props_default = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"default": "", "default": "",
# This does not work :( # This does not work :(
#"$def": { # "$def": {
# "props": {}, # "props": {},
# }, # },
"oneOf": [ "oneOf": [
{ {
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"default": {}, "default": {},
"title": "object", "title": "object",
"properties": {}, "properties": {},
"description": "Object to configure a bacjend item", "description": "Object to configure a bacjend item",
}, },
{ {
"type": "string", "type": "string",
"default": "BLAAAAHHH", "default": "BLAAAAHHH",
"title": "string", "title": "string",
"description": "Enter a simple string configuration value for default engine", "description": "Enter a simple string configuration value for default engine",
}, },
] ],
} }
def _validate_item(self, item): def _validate_item(self, item):
"""Private method to validate sub class"""
if isinstance(item, str): if isinstance(item, str):
item = { item = {
"engine": self.config_main.default_engine, "engine": self.config_main.default_engine,
"value": item, "value": item,
} }
item = schema_validate(item, self._schema_props_default) item = schema_validate(item, self._schema_props_default)
assert (isinstance(item, dict)) assert isinstance(item, dict)
return item return item
def __init__(self, app): def __init__(self, app):
self.app = app self.app = app
self.config_app = app.conf2['config']['app'] self.config_app = app.conf2["config"]["app"]
self.config_main = app.conf2['config']['tree'] self.config_main = app.conf2["config"]["tree"]
self.config_items = list(app.conf2['tree']) self.config_items = list(app.conf2["tree"])
# THIS MAKE A BUG !!!! self.plugin_loader = LoadPlugin(AlberoPlugins) # THIS MAKE A BUG !!!! self.plugin_loader = LoadPlugin(AlberoPlugins)
self.plugins = [ self.plugins = [
'init', "init",
'loop', "loop",
'hier', "hier",
] ]
# Auto init # Auto init
self.backends = self.config_items self.backends = self.config_items
def query(self, key=None, scope=None, trace=False): def query(self, key=None, scope=None, trace=False):
backends = self.get_backends(key=key, scope=scope, trace=trace) backends = self.get_backends(key=key, scope=scope, trace=trace)
ret = self.get_results(backends, trace=trace) ret = self.get_results(backends, trace=trace)
@ -160,34 +158,31 @@ class BackendsManager(Manager):
# Prepare plugins # Prepare plugins
plugin_loader = LoadPlugin(AlberoPlugins) plugin_loader = LoadPlugin(AlberoPlugins)
_run = { _run = {
"key": key, "key": key,
"scope": scope, "scope": scope,
} }
# Preprocess backends plugins # Preprocess backends plugins
backends = self.config_items backends = self.config_items
log.debug(f"Backend preprocessing of {len(backends)} elements") log.debug(f"Backend preprocessing of {len(backends)} elements")
for plugin in self.plugins: for plugin in self.plugins:
#backend_cls = plugin_loader.load('backend', plugin) # backend_cls = plugin_loader.load('backend', plugin)
plugin = plugin_loader.load( plugin = plugin_loader.load("backend", plugin)()
'backend', plugin
)()
log.debug(f"Run {plugin}") log.debug(f"Run {plugin}")
new_backend, _run = plugin.process(backends, _run) new_backend, _run = plugin.process(backends, _run)
assert(isinstance(new_backend, list)), f"Got: {new_backend}" assert isinstance(new_backend, list), f"Got: {new_backend}"
assert(isinstance(_run, dict)), f"Got: {_run}" assert isinstance(_run, dict), f"Got: {_run}"
backends = new_backend backends = new_backend
# pprint (backends) # pprint (backends)
for i in backends: for i in backends:
assert (i.get('engine')), f"Got: {i}" assert i.get("engine"), f"Got: {i}"
log.debug(f"Backend preprocessing made {len(backends)} elements") log.debug(f"Backend preprocessing made {len(backends)} elements")
return backends return backends
def get_results(self, backends, trace=False): def get_results(self, backends, trace=False):
# Prepare plugins # Prepare plugins
@ -195,20 +190,18 @@ class BackendsManager(Manager):
new_results = [] new_results = []
for backend in backends: for backend in backends:
#result_cls = result_loader.load('result', result) # result_cls = result_loader.load('result', result)
# print ("BACKKENNDNNDNDNDND") # print ("BACKKENNDNNDNDNDND")
# pprint(backend) # pprint(backend)
engine = plugin_loader.load( engine = plugin_loader.load("engine", backend["engine"])(
'engine', backend['engine'] backend, parent=self, app=self.app
)( )
backend,
parent=self, app=self.app)
log.debug(f"Run engine: {engine}") log.debug(f"Run engine: {engine}")
new_result = engine.process() new_result = engine.process()
assert(isinstance(new_result, list)), f"Got: {new_result}" assert isinstance(new_result, list), f"Got: {new_result}"
new_results.extend(new_result) new_results.extend(new_result)
# Filter out? Not here !new_results = [i for i in new_results if i['found'] ] # Filter out? Not here !new_results = [i for i in new_results if i['found'] ]
@ -217,65 +210,60 @@ class BackendsManager(Manager):
return new_results return new_results
class RulesManager(Manager): class RulesManager(Manager):
plugins_kind = ['strategy'] plugins_kind = ["strategy"]
_schema_props_new = { _schema_props_new = {
"rule": { "rule": {
"default": ".*", "default": ".*",
"optional": True, "optional": True,
"oneOf": [
{
"type": "string",
},
{
"type": "null",
},
],
},
"strategy": {
"type": "string",
"default": "schema",
# "default": "last",
"optional": True,
# "enum": ["first", "last", "merge"],
},
"trace": {
"type": "boolean",
"default": False,
},
"explain": {
"type": "boolean",
"default": False,
},
}
_props_position = 'oneOf/1/properties'
_schema_props_default = {
"$schema": 'http://json-schema.org/draft-07/schema#',
"default": "",
"$def": {
"items": {},
},
"oneOf": [ "oneOf": [
{ {
"type": "string", "type": "string",
"default": "BLAAAAHHH"
}, },
{ {
"type": "object", "type": "null",
"additionalProperties": True,
"default": {},
"properties": { "$ref": "#/$defs/name" },
}, },
] ],
} },
"strategy": {
"type": "string",
"default": "schema",
# "default": "last",
"optional": True,
# "enum": ["first", "last", "merge"],
},
"trace": {
"type": "boolean",
"default": False,
},
"explain": {
"type": "boolean",
"default": False,
},
}
_props_position = "oneOf/1/properties"
_schema_props_default = {
"$schema": "http://json-schema.org/draft-07/schema#",
"default": "",
"$def": {
"items": {},
},
"oneOf": [
{"type": "string", "default": "BLAAAAHHH"},
{
"type": "object",
"additionalProperties": True,
"default": {},
"properties": {"$ref": "#/$defs/name"},
},
],
}
OLD_rule_schema = { OLD_rule_schema = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
"properties": { "properties": {
@ -286,73 +274,70 @@ class RulesManager(Manager):
"oneOf": [ "oneOf": [
{ {
"type": "null", "type": "null",
}, },
{ {
"type": "string", "type": "string",
}, },
{ {
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"default": {}, "default": {},
"properties": { "$ref": "#/$defs/name" }, "properties": {"$ref": "#/$defs/name"},
}, },
], ],
}, },
} },
} }
def __init__(self, app): def __init__(self, app):
self.app = app self.app = app
self.config_app = app.conf2['config']['app'] self.config_app = app.conf2["config"]["app"]
self.config_main = app.conf2['config']['rules'] self.config_main = app.conf2["config"]["rules"]
self.config_items = list(app.conf2['rules']) self.config_items = list(app.conf2["rules"])
def get_result(self, candidates, key=None, scope=None, trace=False, explain=False): def get_result(self, candidates, key=None, scope=None, trace=False, explain=False):
#trace=False # trace=False
rules = self.config_items rules = self.config_items
key = key or '' key = key or ""
# Filter out invalid candidates # Filter out invalid candidates
matched_candidates = [i for i in candidates if i['found'] == True] matched_candidates = [i for i in candidates if i["found"] == True]
if len(matched_candidates) == 0: if len(matched_candidates) == 0:
log.debug("No matched candidates") log.debug("No matched candidates")
return None return None
# Look for matching key in rules defiunitions # Look for matching key in rules defiunitions
regex_support = False regex_support = False
matched_rule = {} matched_rule = {}
if regex_support: if regex_support:
raise Exception("Not Implemented") raise Exception("Not Implemented")
else:
rule = [ i for i in rules if i.get('rule') == key ]
if len(rule) == 0:
log.debug(f"No matched rule for {key}, applying defaults")
else:
matched_rule = rule[0]
log.debug(f"Matcher rule for {key}: {matched_rule}")
matched_rule['trace'] = trace rule = [i for i in rules if i.get("rule") == key]
matched_rule['explain'] = explain if len(rule) == 0:
log.debug(f"No matched rule for %s, applying defaults", key)
else:
matched_rule = rule[0]
log.debug(f"Matcher rule for {key}: {matched_rule}")
matched_rule["trace"] = trace
matched_rule["explain"] = explain
schema_validate(matched_rule, self._schema_props_default) schema_validate(matched_rule, self._schema_props_default)
# Prepare plugins # Prepare plugins
assert(isinstance(matched_candidates, list)), f"Got: {matched_candidates}" assert isinstance(matched_candidates, list), f"Got: {matched_candidates}"
assert(isinstance(matched_rule, dict)), f"Got: {matched_rule}" assert isinstance(matched_rule, dict), f"Got: {matched_rule}"
strategy = matched_rule.get('strategy', 'schema') strategy = matched_rule.get("strategy", "schema")
log.debug(f"Key '{key}' matched rule '{rule}' with '{strategy}' strategy") log.debug(f"Key '{key}' matched rule '{rule}' with '{strategy}' strategy")
# Load plugin # Load plugin
log.debug(f"Run strategy: {strategy}") log.debug(f"Run strategy: {strategy}")
plugin_loader = LoadPlugin(AlberoPlugins) plugin_loader = LoadPlugin(AlberoPlugins)
strategy = plugin_loader.load('strategy', strategy = plugin_loader.load(
strategy, "strategy",
)(parent=self, app=self.app) strategy,
)(parent=self, app=self.app)
new_result = strategy.process(matched_candidates, matched_rule) new_result = strategy.process(matched_candidates, matched_rule)
return new_result return new_result

105
albero/plugin/backend/hier.py
View File

@ -1,11 +1,10 @@
import copy import copy
# from pathlib import Path # from pathlib import Path
# from albero.utils import render_template # from albero.utils import render_template
# from albero.plugin.common import PluginBackendClass # from albero.plugin.common import PluginBackendClass
# from pprint import pprint # from pprint import pprint
# #
# import logging # import logging
# import anyconfig # import anyconfig
# import textwrap # import textwrap
@ -13,55 +12,55 @@ import copy
from albero.plugin.common import PluginBackendClass from albero.plugin.common import PluginBackendClass
from pprint import pprint from pprint import pprint
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class Plugin(PluginBackendClass): class Plugin(PluginBackendClass):
_plugin_name = "hier" _plugin_name = "hier"
_schema_props_new = { _schema_props_new = {
"hier": { "hier": {
"default": None, "default": None,
"optional": True, "optional": True,
"oneOf": [ "oneOf": [
{ {
"type": "null", "type": "null",
}, },
{ {
"type": "string", "type": "string",
}, },
{ {
"additionalProperties": True, "additionalProperties": True,
"properties": { "properties": {
"data": { "data": {
"default": None, "default": None,
"anyOf": [ "anyOf": [
{ "type": "null" }, {"type": "null"},
{ "type": "string" }, {"type": "string"},
{ "type": "array" }, {"type": "array"},
] ],
}, },
"var": { "var": {
"type": "string", "type": "string",
"default": "hier_item", "default": "hier_item",
"optional": True, "optional": True,
}, },
"separator": { "separator": {
"type": "string", "type": "string",
"default": "/", "default": "/",
"optional": True, "optional": True,
}, },
"reversed": { "reversed": {
"type": "boolean", "type": "boolean",
"default": False, "default": False,
"optional": True, "optional": True,
},
}, },
}, },
] },
} ],
} }
}
def process(self, backends: list, ctx: dict) -> (list, dict): def process(self, backends: list, ctx: dict) -> (list, dict):
@ -79,13 +78,13 @@ class Plugin(PluginBackendClass):
hier_var = plugin_config.get("var", "item") hier_var = plugin_config.get("var", "item")
hier_sep = plugin_config.get("separator", "/") hier_sep = plugin_config.get("separator", "/")
if isinstance(hier_data, str): if isinstance(hier_data, str):
hier_data = cand['_run']['scope'].get(hier_data, None) hier_data = cand["_run"]["scope"].get(hier_data, None)
# Build a new list # Build a new list
if isinstance(hier_data, str): if isinstance(hier_data, str):
r = hier_data.split(hier_sep) r = hier_data.split(hier_sep)
assert (isinstance(r, list)), f"Got: {r}" assert isinstance(r, list), f"Got: {r}"
ret1 = [] ret1 = []
for index, part in enumerate(r): for index, part in enumerate(r):
@ -93,7 +92,7 @@ class Plugin(PluginBackendClass):
try: try:
prefix = ret1[index - 1] prefix = ret1[index - 1]
except IndexError: except IndexError:
prefix = f'{hier_sep}' prefix = f"{hier_sep}"
prefix = "" prefix = ""
item = f"{prefix}{part}{hier_sep}" item = f"{prefix}{part}{hier_sep}"
ret1.append(item) ret1.append(item)
@ -102,15 +101,13 @@ class Plugin(PluginBackendClass):
for item in ret1: for item in ret1:
_cand = copy.deepcopy(cand) _cand = copy.deepcopy(cand)
run = { run = {
"index": index, "index": index,
"hier_value": item, "hier_value": item,
"hier_var": hier_var, "hier_var": hier_var,
} }
_cand['_run']['hier'] = run _cand["_run"]["hier"] = run
_cand['_run']['scope'][hier_var] = item _cand["_run"]["scope"][hier_var] = item
ret2.append(_cand) ret2.append(_cand)
new_backends.extend(ret2) new_backends.extend(ret2)
return new_backends, ctx return new_backends, ctx

22
albero/plugin/backend/init.py
View File

@ -1,37 +1,35 @@
from albero.plugin.common import PluginBackendClass from albero.plugin.common import PluginBackendClass
from pprint import pprint from pprint import pprint
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
import copy import copy
class Plugin(PluginBackendClass): class Plugin(PluginBackendClass):
_plugin_name = "init" _plugin_name = "init"
_schema_props_new = None _schema_props_new = None
default_engine = 'jerakia' default_engine = "jerakia"
def process(self, backends: list, ctx: dict) -> (list, dict): def process(self, backends: list, ctx: dict) -> (list, dict):
new_backends = [] new_backends = []
for index, item in enumerate(backends): for index, item in enumerate(backends):
default = { default = {
"value": item, "value": item,
} }
if not isinstance(item, dict): if not isinstance(item, dict):
item = default item = default
item['engine'] = item.get('engine', self.default_engine ) item["engine"] = item.get("engine", self.default_engine)
item['_run'] = copy.deepcopy(ctx) item["_run"] = copy.deepcopy(ctx)
item['_run']['backend'] = { item["_run"]["backend"] = {
"index": index, "index": index,
} }
new_backends.append(item) new_backends.append(item)
return new_backends, ctx return new_backends, ctx

142
albero/plugin/backend/loop.py
View File

@ -1,5 +1,3 @@
import copy import copy
from pathlib import Path from pathlib import Path
from albero.utils import render_template from albero.utils import render_template
@ -14,56 +12,56 @@ import textwrap
class Plugin(PluginBackendClass): class Plugin(PluginBackendClass):
_plugin_name = "loop" _plugin_name = "loop"
_plugin_help = """ _plugin_help = (
"""
This module helps to loop over a backend This module helps to loop over a backend
""", """,
)
_schema_props_new = { _schema_props_new = {
"loop": { "loop": {
"description": _plugin_help, "description": _plugin_help,
"default": None, "default": None,
"optional": True, "optional": True,
"examples": [ "examples": [
{ {
"value": "site/{{ loop_env }}/config/{{ os }}", "value": "site/{{ loop_env }}/config/{{ os }}",
"loop": { "loop": {
"var": "loop_env", "var": "loop_env",
"data": [ "data": [
"dev", "dev",
"preprod", "preprod",
"prod", "prod",
], ],
},
"comment": "The module will loop three time over the value, and the variable `loop_env` will consecutely have `dev`, `preprod` and `prod` as value",
}, },
{ "comment": "The module will loop three time over the value, and the variable `loop_env` will consecutely have `dev`, `preprod` and `prod` as value",
"value": "site/{{ loop_env2 }}/config/{{ os }}", },
"loop": { {
"var": "loop_env2", "value": "site/{{ loop_env2 }}/config/{{ os }}",
"data": "my_scope_var", "loop": {
}, "var": "loop_env2",
"comment": "Like the previous example, but it will fetch the list from any scope variables", "data": "my_scope_var",
}, },
{ "comment": "Like the previous example, but it will fetch the list from any scope variables",
"loop": None, },
"comment": "Disable this module, no loop will operate", {
}, "loop": None,
"comment": "Disable this module, no loop will operate",
},
# "loop": { # "loop": {
# "var": "my_var", # "var": "my_var",
# }, # },
# }, # },
# "loop": { # "loop": {
# "var": "my_var", # "var": "my_var",
# }, # },
# "example": "", # "example": "",
# }, # },
# "loop": { # "loop": {
# "var": "my_var", # "var": "my_var",
# }, # },
# "example": "", # "example": "",
# }, # },
], ],
"oneOf": [ "oneOf": [
{ {
"type": "object", "type": "object",
@ -71,30 +69,29 @@ class Plugin(PluginBackendClass):
"default": {}, "default": {},
"title": "Complete config", "title": "Complete config",
"description": "", "description": "",
"properties": { "properties": {
"data": { "data": {
"default": None, "default": None,
"optional": False, "optional": False,
"title": "Module configuration", "title": "Module configuration",
"description": "Data list used for iterations. It only accept lists as type. It disable the module if set to `null`.", "description": "Data list used for iterations. It only accept lists as type. It disable the module if set to `null`.",
"anyOf":[ "anyOf": [
{ {
"type": "null", "type": "null",
"title": "Disable Module", "title": "Disable Module",
"description": "Disable the module", "description": "Disable the module",
}, },
{ {
"type": "string", "type": "string",
"title": "Scope variable", "title": "Scope variable",
"description": "Will look the value of the loop list from the scope. TOFIX: What if variablle does not exists?", "description": "Will look the value of the loop list from the scope. TOFIX: What if variablle does not exists?",
}, },
{ {
"type": "array", "type": "array",
"title": "Hardcoded list", "title": "Hardcoded list",
"description": "Simply enter the list of value to be iterated to.", "description": "Simply enter the list of value to be iterated to.",
}, },
] ],
}, },
"var": { "var": {
"type": "string", "type": "string",
@ -115,14 +112,12 @@ class Plugin(PluginBackendClass):
"title": "Disable", "title": "Disable",
"description": "If set to null, it disable the module", "description": "If set to null, it disable the module",
}, },
] ],
} }
} }
def process(self, backends: list, ctx: dict) -> (list, dict): def process(self, backends: list, ctx: dict) -> (list, dict):
new_backends = [] new_backends = []
for cand in backends: for cand in backends:
cand = dict(cand) cand = dict(cand)
@ -137,26 +132,23 @@ class Plugin(PluginBackendClass):
# Retrieve config data # Retrieve config data
loop_var = loop_config.get("var", "item") loop_var = loop_config.get("var", "item")
if isinstance(loop_data, str): if isinstance(loop_data, str):
loop_data = cand['_run']['scope'].get(loop_data, None) loop_data = cand["_run"]["scope"].get(loop_data, None)
assert (isinstance(loop_data, list)), f"Got: {loop_data}" assert isinstance(loop_data, list), f"Got: {loop_data}"
# Build a new list # Build a new list
ret = [] ret = []
for idx, item in enumerate(loop_data): for idx, item in enumerate(loop_data):
_cand = copy.deepcopy(cand) _cand = copy.deepcopy(cand)
run = { run = {
"loop_index": idx, "loop_index": idx,
"loop_value": item, "loop_value": item,
"loop_var": loop_var, "loop_var": loop_var,
} }
_cand['_run']['loop'] = run _cand["_run"]["loop"] = run
_cand['_run']['scope'][loop_var] = item _cand["_run"]["scope"][loop_var] = item
#_cand.scope[loop_var] = item # _cand.scope[loop_var] = item
ret.append(_cand) ret.append(_cand)
new_backends.extend(ret) new_backends.extend(ret)
return new_backends, ctx return new_backends, ctx

177
albero/plugin/common.py
View File

@ -1,4 +1,3 @@
# from box import Box # from box import Box
import textwrap import textwrap
from pprint import pprint from pprint import pprint
@ -9,6 +8,7 @@ import yaml
import json import json
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
from albero.utils import schema_validate from albero.utils import schema_validate
@ -16,7 +16,7 @@ import copy
# Candidate Classes # Candidate Classes
# ============================= # =============================
class Candidate(): class Candidate:
engine = None engine = None
found = False found = False
data = None data = None
@ -32,40 +32,36 @@ class Candidate():
def _report_data(self, data=None): def _report_data(self, data=None):
default_data = { default_data = {
#"rule": self.config, # "rule": self.config,
"value": self.engine._plugin_value, "value": self.engine._plugin_value,
"data": self.data, "data": self.data,
} }
data = data or default_data data = data or default_data
d = json.dumps(data, indent=2) #, sort_keys=True, ) d = json.dumps(data, indent=2) # , sort_keys=True, )
return d return d
# Generic Classes # Generic Classes
# ============================= # =============================
class PluginClass(): class PluginClass:
_plugin_type = "none" _plugin_type = "none"
_plugin_value = None _plugin_value = None
_schema_props_new = "UNSET PLUGIN PROPRIETIES" _schema_props_new = "UNSET PLUGIN PROPRIETIES"
_schema_props_plugin = { _schema_props_plugin = {
"engine": { "engine": {
"type": "string", "type": "string",
# TODO: Fix this ug # TODO: Fix this ug
"default": "jerakia" "default": "jerakia",
}, },
"value": {}, "value": {},
} }
def __repr__(self): def __repr__(self):
kind = self._plugin_type kind = self._plugin_type
name = self._plugin_name name = self._plugin_name
value = getattr(self, 'value', 'NO VALUE') value = getattr(self, "value", "NO VALUE")
return f"{kind}.{name}:{value}" return f"{kind}.{name}:{value}"
def __init__(self, config=None, parent=None, app=None): def __init__(self, config=None, parent=None, app=None):
@ -79,61 +75,63 @@ class PluginClass():
def _init(self): def _init(self):
pass pass
def _validate(self): def _validate(self):
pass pass
class PluginBackendClass(PluginClass): class PluginBackendClass(PluginClass):
_plugin_type = "backend" _plugin_type = "backend"
def _init(self): def _init(self):
pass pass
class PluginStrategyClass(PluginClass): class PluginStrategyClass(PluginClass):
_plugin_type = "strategy" _plugin_type = "strategy"
def _init(self): def _init(self):
pass pass
class PluginEngineClass(PluginClass): class PluginEngineClass(PluginClass):
_plugin_type = "engine" _plugin_type = "engine"
_schema_props_default = { _schema_props_default = {
"value": { "value": {
"default": "UNSET", "default": "UNSET",
},
#### SHOULD NOT BE HERE
"hier": {
"additionalProperties": True,
"optional": True,
"properties": {
"var": {
"type": "string",
"default": "item",
"optional": True,
}, },
"data": {
#### SHOULD NOT BE HERE "default": None,
"hier": { "anyOf": [
"additionalProperties": True, {"type": "null"},
"optional": True, {"type": "string"},
"properties": { {"type": "array"},
"var": { ],
"type": "string", },
"default": "item", "separator": {
"optional": True, "type": "string",
}, "default": "/",
"data": { "optional": True,
"default": None, },
"anyOf": [ "reversed": {
{ "type": "null" }, "type": "boolean",
{ "type": "string" }, "default": False,
{ "type": "array" }, "optional": True,
] },
}, },
"separator": { },
"type": "string", }
"default": "/",
"optional": True,
},
"reversed": {
"type": "boolean",
"default": False,
"optional": True,
},
}
}
}
# Default plugin API Methods # Default plugin API Methods
# ===================== # =====================
@ -143,13 +141,13 @@ class PluginEngineClass(PluginClass):
def _validate(self): def _validate(self):
# Build schema # Build schema
schema_keys = [a for a in dir(self) if a.startswith('_schema_props_')] schema_keys = [a for a in dir(self) if a.startswith("_schema_props_")]
props = {} props = {}
for key in schema_keys: for key in schema_keys:
schema = getattr(self, key) schema = getattr(self, key)
props.update(schema) props.update(schema)
self.schema = { self.schema = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"properties": props, "properties": props,
@ -159,72 +157,75 @@ class PluginEngineClass(PluginClass):
self.config = schema_validate(self.config, self.schema) self.config = schema_validate(self.config, self.schema)
return True return True
# Public Methods # Public Methods
# ===================== # =====================
def dump(self): def dump(self):
ret = { ret = {
"config": self.config, "config": self.config,
} }
return ret return ret
def lookup_candidates(self, key=None, scope=None): def lookup_candidates(self, key=None, scope=None):
raise Exception (f"Module does not implement this method :(") raise Exception(f"Module does not implement this method :(")
# It must always return a list of `Candidate` instances # It must always return a list of `Candidate` instances
return [] return []
def _example(self): def _example(self):
print (f"Module does not implement this method :(") print(f"Module does not implement this method :(")
return None return None
# File plugins Extensions # File plugins Extensions
# ============================= # =============================
class PluginFileGlob():
class PluginFileGlob:
_schema_props_glob = { _schema_props_glob = {
"glob": { "glob": {
"additionalProperties": False, "additionalProperties": False,
"default": { "default": {
"file": "ansible.yaml", "file": "ansible.yaml",
}, },
"properties": { "properties": {
"file": { "file": {
"type": "string", "type": "string",
"default": "ansible", "default": "ansible",
"optional": True, "optional": True,
}, },
"ext": { "ext": {
"type": "array", "type": "array",
"default": [ "yml", "yaml" ], "default": ["yml", "yaml"],
"optional": True, "optional": True,
}, },
} },
}
} }
}
def _glob(self, item): def _glob(self, item):
# DIRECT CALL TO APP< TOFIX # DIRECT CALL TO APP< TOFIX
app_config = self.app.conf2 app_config = self.app.conf2
root = app_config.get("default", {}).get("config", {}).get("root", f"{Path.cwd()}/tree") root = (
#root = self.app.conf2.config.app.root app_config.get("default", {})
.get("config", {})
.get("root", f"{Path.cwd()}/tree")
)
# root = self.app.conf2.config.app.root
# TOFIX print ("ITEM! %s" % type(root)) # TOFIX print ("ITEM! %s" % type(root))
# TOFIX print ("ITEM2 %s" % self.app.conf2.config.app.root) # TOFIX print ("ITEM2 %s" % self.app.conf2.config.app.root)
glob_config = self.config.get("glob", {}) glob_config = self.config.get("glob", {})
glob_file = glob_config['file'] glob_file = glob_config["file"]
#glob_ext = glob_config['ext'] # glob_ext = glob_config['ext']
item = Path(root) / Path(item) / Path(glob_file) item = Path(root) / Path(item) / Path(glob_file)
item = f"{item}" item = f"{item}"
#file = f"{glob_file}.{glob_ext}" # file = f"{glob_file}.{glob_ext}"
#print ("ITEM %s" % item) # print ("ITEM %s" % item)
files = glob.glob(item) files = glob.glob(item)
log.debug (f"Matched file for glob '{item}': {files}") log.debug(f"Matched file for glob '{item}': {files}")
return files return files

74
albero/plugin/engine/jerakia.py
View File

@ -1,4 +1,3 @@
from pathlib import Path from pathlib import Path
from albero.utils import render_template from albero.utils import render_template
from albero.plugin.common import PluginEngineClass, PluginFileGlob, Candidate from albero.plugin.common import PluginEngineClass, PluginFileGlob, Candidate
@ -10,55 +9,55 @@ import textwrap
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class FileCandidate(Candidate): class FileCandidate(Candidate):
path = None path = None
def _report_data(self): def _report_data(self):
data = { data = {
#"rule": self.config, # "rule": self.config,
"value": self.engine._plugin_value, "value": self.engine._plugin_value,
"data": self.data, "data": self.data,
"path": str(self.path.relative_to(Path.cwd())), "path": str(self.path.relative_to(Path.cwd())),
} }
data = dict(self.config) data = dict(self.config)
return super()._report_data(data) return super()._report_data(data)
class Plugin(PluginEngineClass, PluginFileGlob): class Plugin(PluginEngineClass, PluginFileGlob):
_plugin_name = 'jerakia' _plugin_name = "jerakia"
### OLD ### OLD
_plugin_engine = "jerakia" _plugin_engine = "jerakia"
# _schema_props_files = { # _schema_props_files = {
_schema_props_new = { _schema_props_new = {
"path": { "path": {
"anyOf": [ "anyOf": [
{ {
"type": "string",
},
{
"type": "array",
"items": {
"type": "string", "type": "string",
}, },
{ },
"type": "array", ]
"items": {
"type": "string",
}
},
]
}
} }
}
def _init(self): def _init(self):
paths = self.config.get('path', self.config.get('value')) paths = self.config.get("path", self.config.get("value"))
if isinstance(paths, str): if isinstance(paths, str):
paths = [paths] paths = [paths]
elif isinstance(paths, list): elif isinstance(paths, list):
pass pass
else: else:
raise Exception (f"Unsupported path value, expected str or dict, got: {paths} in {self.config}") raise Exception(
f"Unsupported path value, expected str or dict, got: {paths} in {self.config}"
)
self.paths = paths self.paths = paths
self.value = paths self.value = paths
@ -78,7 +77,6 @@ class Plugin(PluginEngineClass, PluginFileGlob):
return ret return ret
def _show_paths(self, scope): def _show_paths(self, scope):
parsed = self._preprocess(scope) parsed = self._preprocess(scope)
@ -93,14 +91,12 @@ class Plugin(PluginEngineClass, PluginFileGlob):
return ret3 return ret3
def process(self): def process(self):
# scope = self.scope
#scope = self.scope
# pprint (self.config) # pprint (self.config)
scope = dict(self.config['_run']['scope']) scope = dict(self.config["_run"]["scope"])
key = self.config['_run']['key'] key = self.config["_run"]["key"]
assert isinstance(scope, dict), f"Got: {scope}" assert isinstance(scope, dict), f"Got: {scope}"
assert isinstance(key, (str, type(None))), f"Got: {key}" assert isinstance(key, (str, type(None))), f"Got: {key}"
@ -125,13 +121,13 @@ class Plugin(PluginEngineClass, PluginFileGlob):
# Build result object # Build result object
result = {} result = {}
result['run'] = { result["run"] = {
'path': path, "path": path,
'rel_path': str(Path(path).relative_to(Path.cwd())), "rel_path": str(Path(path).relative_to(Path.cwd())),
} }
result['parent'] = self.config result["parent"] = self.config
result['data'] = data result["data"] = data
result['found'] = found result["found"] = found
ret.append(result) ret.append(result)
@ -141,7 +137,7 @@ class Plugin(PluginEngineClass, PluginFileGlob):
# # Read raw file content # # Read raw file content
# data = anyconfig.load(path, ac_parser="yaml") # data = anyconfig.load(path, ac_parser="yaml")
# #
# ret_obj2 ={ # ret_obj2 ={
# "_run": _run, # "_run": _run,
@ -183,7 +179,3 @@ class Plugin(PluginEngineClass, PluginFileGlob):
# #log.debug(f"Found value: {ret_obj}") # #log.debug(f"Found value: {ret_obj}")
# ret_obj.found = found # ret_obj.found = found
# ret.append(ret_obj) # ret.append(ret_obj)

3
albero/plugin/strategy/last.py
View File

@ -1,9 +1,9 @@
import logging import logging
from albero.plugin.common import PluginStrategyClass from albero.plugin.common import PluginStrategyClass
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class Plugin(PluginStrategyClass): class Plugin(PluginStrategyClass):
_plugin_name = "last" _plugin_name = "last"
@ -12,4 +12,3 @@ class Plugin(PluginStrategyClass):
def process(self, candidates: list, rule=None) -> (list, dict): def process(self, candidates: list, rule=None) -> (list, dict):
return candidates[-1] return candidates[-1]

166
albero/plugin/strategy/schema.py
View File

@ -1,5 +1,3 @@
import logging import logging
from albero.plugin.common import PluginStrategyClass from albero.plugin.common import PluginStrategyClass
from albero.utils import schema_validate, str_ellipsis from albero.utils import schema_validate, str_ellipsis
@ -11,6 +9,7 @@ from pprint import pprint
from jsonmerge import Merger from jsonmerge import Merger
from prettytable import PrettyTable from prettytable import PrettyTable
class Plugin(PluginStrategyClass): class Plugin(PluginStrategyClass):
_plugin_name = "schema" _plugin_name = "schema"
@ -36,11 +35,11 @@ class Plugin(PluginStrategyClass):
"data": { "data": {
"default": None, "default": None,
"optional": False, "optional": False,
"anyOf":[ "anyOf": [
{"type": "null"}, {"type": "null"},
{"type": "string"}, {"type": "string"},
{"type": "array"}, {"type": "array"},
] ],
}, },
"var": { "var": {
"type": "string", "type": "string",
@ -49,17 +48,17 @@ class Plugin(PluginStrategyClass):
}, },
}, },
}, },
] ],
} }
} }
default_merge_schema = { default_merge_schema = {
"$schema": 'http://json-schema.org/draft-07/schema#', "$schema": "http://json-schema.org/draft-07/schema#",
"oneOf": [ "oneOf": [
{ {
"type": "array", "type": "array",
"mergeStrategy": "append", "mergeStrategy": "append",
# "mergeStrategy": "arrayMergeById", # "mergeStrategy": "arrayMergeById",
}, },
{ {
"type": "object", "type": "object",
@ -88,76 +87,123 @@ class Plugin(PluginStrategyClass):
], ],
} }
def process(self, candidates: list, rule=None) -> (list, dict): def process(self, candidates: list, rule=None) -> (list, dict):
trace = rule['trace'] trace = rule["trace"]
explain = rule['explain'] explain = rule["explain"]
schema = rule.get('schema', None) or self.default_merge_schema schema = rule.get("schema", None) or self.default_merge_schema
merger = Merger(schema) merger = Merger(schema)
t = PrettyTable() t = PrettyTable()
t1 = PrettyTable() t1 = PrettyTable()
new_candidate = None new_candidate = None
for index, item in enumerate(candidates): for index, item in enumerate(candidates):
new_value = item['data'] new_value = item["data"]
result = merger.merge(new_candidate, new_value) result = merger.merge(new_candidate, new_value)
backend_info = dict(item['parent']) backend_info = dict(item["parent"])
backend_run = backend_info.pop("_run") backend_run = backend_info.pop("_run")
if explain: if explain:
t1.add_row([ t1.add_row(
index, [
'\nBackendRun: ' + str_ellipsis(json.dumps( index,
backend_run, "\nBackendRun: "
default=lambda o: '<not serializable>', indent=2), 70), + str_ellipsis(
'\nRuleRun: ' + str_ellipsis(json.dumps( json.dumps(
item['run'], backend_run,
default=lambda o: '<not serializable>', indent=2), 70), default=lambda o: "<not serializable>",
'---\nResult: ' + str_ellipsis(json.dumps( indent=2,
result, ),
default=lambda o: '<not serializable>', indent=2), 70), 70,
]) ),
"\nRuleRun: "
+ str_ellipsis(
json.dumps(
item["run"],
default=lambda o: "<not serializable>",
indent=2,
),
70,
),
"---\nResult: "
+ str_ellipsis(
json.dumps(
result, default=lambda o: "<not serializable>", indent=2
),
70,
),
]
)
if trace: if trace:
t.add_row([ t.add_row(
index, [
'---\nBackendConfig: ' + str_ellipsis(json.dumps( index,
backend_info, "---\nBackendConfig: "
default=lambda o: '<not serializable>', indent=2), 70) + + str_ellipsis(
'\nBackendRun: ' + str_ellipsis(json.dumps( json.dumps(
backend_run, backend_info,
default=lambda o: '<not serializable>', indent=2), 70), default=lambda o: "<not serializable>",
indent=2,
'---\nRuleConfig: ' + str_ellipsis(json.dumps( ),
rule, 70,
default=lambda o: '<not serializable>', indent=2), 70) +
'\nRuleRun: ' + str_ellipsis(json.dumps(
item['run'],
default=lambda o: '<not serializable>', indent=2), 70) +
#'\nSource: ' + str_ellipsis(json.dumps(
# new_candidate,
# default=lambda o: '<not serializable>', indent=2), 70) +
'\nNew data: ' + str_ellipsis(json.dumps(
new_value,
default=lambda o: '<not serializable>', indent=2), 70),
'---\nResult: ' + str_ellipsis(json.dumps(
result,
default=lambda o: '<not serializable>', indent=2), 70),
]
) )
+ "\nBackendRun: "
+ str_ellipsis(
json.dumps(
backend_run,
default=lambda o: "<not serializable>",
indent=2,
),
70,
),
"---\nRuleConfig: "
+ str_ellipsis(
json.dumps(
rule, default=lambda o: "<not serializable>", indent=2
),
70,
)
+ "\nRuleRun: "
+ str_ellipsis(
json.dumps(
item["run"],
default=lambda o: "<not serializable>",
indent=2,
),
70,
)
+
#'\nSource: ' + str_ellipsis(json.dumps(
# new_candidate,
# default=lambda o: '<not serializable>', indent=2), 70) +
"\nNew data: "
+ str_ellipsis(
json.dumps(
new_value,
default=lambda o: "<not serializable>",
indent=2,
),
70,
),
"---\nResult: "
+ str_ellipsis(
json.dumps(
result, default=lambda o: "<not serializable>", indent=2
),
70,
),
]
)
new_candidate = result new_candidate = result
if trace: if trace:
t.field_names = ["Index", "Backend", "Rule", "Data"] t.field_names = ["Index", "Backend", "Rule", "Data"]
t.align = 'l' t.align = "l"
print (t) print(t)
if explain: if explain:
t1.field_names = ["Index", "Backend", "Rule", "Data"] t1.field_names = ["Index", "Backend", "Rule", "Data"]
t1.align = 'l' t1.align = "l"
print('Explain:\n' + repr(t1)) print("Explain:\n" + repr(t1))
return new_candidate return new_candidate

11
albero/query.py
View File

@ -12,19 +12,20 @@ from pprint import pprint
from albero.managers import BackendsManager, RulesManager from albero.managers import BackendsManager, RulesManager
from albero.utils import schema_validate from albero.utils import schema_validate
import anyconfig import anyconfig
# from box import Box # from box import Box
from pathlib import Path from pathlib import Path
import logging import logging
log = logging.getLogger(__name__)
log = logging.getLogger(__name__)
# Query # Query
########################################## ##########################################
class Query():
class Query:
def __init__(self, app): def __init__(self, app):
self.app = app self.app = app
@ -43,9 +44,7 @@ class Query():
ret = {} ret = {}
for i in dir(self): for i in dir(self):
if not i.startswith('_'): if not i.startswith("_"):
ret[i] = getattr(self, i) ret[i] = getattr(self, i)
pprint (ret) pprint(ret)

52
albero/utils.py
View File

@ -1,4 +1,3 @@
from pathlib import Path from pathlib import Path
from jinja2 import Template from jinja2 import Template
import yaml import yaml
@ -10,6 +9,7 @@ import collections
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -62,19 +62,21 @@ log = logging.getLogger(__name__)
# Utils Methods # Utils Methods
# ===================== # =====================
def render_template(path, params): def render_template(path, params):
"""Render template for a given string""" """Render template for a given string"""
assert (isinstance(params, dict)), f"Got: {params}" assert isinstance(params, dict), f"Got: {params}"
t = Template(path) t = Template(path)
return t.render(**params) return t.render(**params)
#def read_file(file):
# def read_file(file):
# with open(file, 'r') as f: # with open(file, 'r') as f:
# data = f.read().replace('\n', '') # data = f.read().replace('\n', '')
# return data # return data
# #
# #
#def parse_file(file, fmt='auto'): # def parse_file(file, fmt='auto'):
# print ("DEPRECATED") # print ("DEPRECATED")
# raise Exception ("parse_file is deprecated") # raise Exception ("parse_file is deprecated")
# #
@ -99,6 +101,7 @@ def render_template(path, params):
# Schema Methods # Schema Methods
# ===================== # =====================
def _extend_with_default(validator_class): def _extend_with_default(validator_class):
validate_properties = validator_class.VALIDATORS["properties"] validate_properties = validator_class.VALIDATORS["properties"]
@ -110,38 +113,43 @@ def _extend_with_default(validator_class):
try: try:
for error in validate_properties( for error in validate_properties(
validator, properties, instance, schema, validator,
properties,
instance,
schema,
): ):
continue continue
except Exception as e: except Exception as e:
print ("CATCHED2222 ", e) print("CATCHED2222 ", e)
return validators.extend( return validators.extend(
validator_class, {"properties" : set_defaults}, validator_class,
{"properties": set_defaults},
) )
def schema_validate(config, schema): def schema_validate(config, schema):
# Validate the schema # Validate the schema
DefaultValidatingDraft7Validator = _extend_with_default(Draft7Validator) DefaultValidatingDraft7Validator = _extend_with_default(Draft7Validator)
try: try:
DefaultValidatingDraft7Validator(schema).validate(config) DefaultValidatingDraft7Validator(schema).validate(config)
except Exception as e: except Exception as e:
print (e) print(e)
p = list(collections.deque(e.schema_path)) p = list(collections.deque(e.schema_path))
p = '/'.join([ str(i) for i in p ]) p = "/".join([str(i) for i in p])
p = f"schema/{p}" p = f"schema/{p}"
raise Exception( raise Exception(
f"Failed validating {p} for resource with content: {config} with !!!!!! schema: {schema}" f"Failed validating {p} for resource with content: {config} with !!!!!! schema: {schema}"
) )
return config return config
def str_ellipsis(txt, length=120): def str_ellipsis(txt, length=120):
txt = str(txt) txt = str(txt)
ret = [] ret = []
for string in txt.splitlines(): for string in txt.splitlines():
string = (string[:length - 4 ] + ' ...') if len(string) > length else string string = (string[: length - 4] + " ...") if len(string) > length else string
ret.append(string) ret.append(string)
ret = '\n'.join(ret) ret = "\n".join(ret)
return ret return ret