diff --git a/kheops/app.py b/kheops/app.py index be4c5f5..35ee93f 100644 --- a/kheops/app.py +++ b/kheops/app.py @@ -89,7 +89,7 @@ CONF_SCHEMA = { }, { "type": "string", - "description": "Add a path prefix before all paths. This is quite useful to store your YAML data in a dedicated tree.", + "description": """Add a path prefix before all paths. This is quite useful to store your YAML data in a dedicated tree.""", }, ], }, @@ -149,13 +149,40 @@ CONF_SCHEMA = { class GenericInstance: + """ + GenericInstance class + + :var name: Name of the instace. + :vartype name: str or None + + :var run: Json compatible dict for instance runtime data. + :vartype run: dict + + + """ name = None run = {} class KheopsNamespace(GenericInstance, QueryProcessor): + """ + Kheops Namespace Class + + """ def __init__(self, app, name, config=None): + """ + Kheops Namespace Instance + + :param app: Parent Kheops Application. + :type app: Kheops + + :param name: Namespace name. + :type config: str + + :param config: Namespace configuration. + :type config: Any + """ self.name = name self.config = config or {} @@ -168,88 +195,19 @@ class KheopsNamespace(GenericInstance, QueryProcessor): self.run["path_ns"] = str(Path(app.run["config_src"]).parent.resolve()) -# def load_namespace(self, namespace="default"): -# # Validate configuration -# -# config = dict(self.raw_config) -# try: -# config = config[namespace] -# except KeyError: -# log.error("Can't find namespace '%s' in config '%s'", namespace, config) -# sys.exit(1) -# config = schema_validate(config, self.schema) -# -# pprint (config) -# -# self.run["path_cwd"] -# -# print ("OKKKKK") -# -# -# conf2 = config -# # Get application paths -# path_root = conf2["config"].get("app", {}).get("root", None) -# if path_root is None: -# path_root = Path(config).parent -# log.debug("Root path guessed from conf file location.") -# else: -# path_root = Path(conf2["config"]["app"]["root"]) -# log.debug("Root path is steup in config") -# -# -# -# path_root = str(path_root.resolve()) -# self.run["path_root"] = path_root -# -# -# # path_prefix = conf2["config"]["app"]["prefix"] -# # if not path_prefix: -# # path_prefix = '' -# # p = Path(path_prefix) -# # if not p.is_absolute(): -# # p = path_root / p -# # try: -# # p = p.resolve().relative_to(Path.cwd().resolve()) -# # except ValueError: -# # pass -# -# -# # Cache paths -# path_cache = Path(conf2["config"]["app"]["cache"]) -# if not path_cache.is_absolute(): -# path_cache = Path(path_root) / path_cache -# path_cache = str(path_cache) -# self.run["path_cache"] = path_cache -# self.cache = { -# 'files': Cache(path_cache), -# 'queries': Cache(path_cache), -# } -# -# # self.run['path_prefix'] = str(p.resolve()) -# log.debug("Working directory is %s, cwd is: %s", path_root, path_cwd) -# -# return config - - -# def query(self, key=None, scope=None): -# processor = QueryProcessor(app=self.app) -# result = processor.exec(key, scope) -# -# return result - class Kheops(GenericInstance): - """Main Kheops Application Instance""" + """ + Kheops Application Class + + """ def __init__(self, config="kheops.yml", namespace="default"): """ - init function + Kheops Application Instance - :param kind: Optional "kind" of ingredients. - :type kind: list[str] or None - :raise lumache.InvalidKindError: If the kind is invalid. - :return: The ingredients list. - :rtype: list[str] + :param config: Kheops configuration. If it's a string, it loads the config from file path. + :type config: str or dict """ # Init @@ -278,8 +236,7 @@ class Kheops(GenericInstance): :param config: Kheops configuration, can either be a file path or a dict. :type config: dict or str or None - :param namespace: Configuration namespace to use. - :type namespace: str + :return: The parsed configuration. :rtype: dict @@ -294,7 +251,7 @@ class Kheops(GenericInstance): source = "dict" return dict_conf - def lookup2( + def lookup( self, keys=None, policy=None, @@ -302,9 +259,17 @@ class Kheops(GenericInstance): trace=False, explain=False, validate_schema=False, - namespace="default", + namespace=None, ): - """Lookup a key in hierarchy""" + """ + Lookup a key in hierarchy + + :param keys: List of keys to query. + :type keys: list[str] + + :param scope: Scope key. + :type scope: dict + """ ret = {} # Loop over keys @@ -314,7 +279,7 @@ class Kheops(GenericInstance): # Identify namespace and key parts = key_def.split(":") - ns_name = self.ns_name + ns_name = namespace or self.ns_name if len(parts) > 1: ns_name = parts[0] key_name = parts[1] @@ -330,83 +295,92 @@ class Kheops(GenericInstance): # TODO: This may lead to inconsistant output format :/ # Return result - if len(keys) > 1: - log.debug("Append '%s' to results", key_name) - ret[key_name] = result - else: - log.debug("Return '%s' result", key_name) - return result + #if len(keys) > 1: + # log.debug("Append '%s' to results", key_name) + ret[key_name] = result + #else: + # log.debug("Return '%s' result", key_name) + # return result return ret - def lookup( - self, - keys=None, - policy=None, - scope=None, - trace=False, - explain=False, - validate_schema=False, - ): - """Lookup a key in hierarchy""" - log.debug("Lookup key %s with scope: %s", keys, scope) - assert isinstance(keys, list), f"Got {keys}" - query = Query(app=self) - ret = {} - for key in keys: - ret[key] = query.exec( - key=key, - scope=scope, - policy=policy, - trace=trace, - explain=explain, - validate_schema=validate_schema, - ) - return ret - def dump_schema(self): - """Dump configuration schema""" - ret1 = BackendsManager.get_schema(KheopsPlugins, mode="parts") - ret2 = RulesManager.get_schema(KheopsPlugins) - print(json.dumps(ret1, indent=2)) - return - # ret = self.schema - # ret["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = ret1 - # ret["patternProperties"][".*"]["properties"]["tree"]["items"] = ret2 - # print(json.dumps(ret, indent=2)) - def gen_docs(self): - """Generate documentation""" - print("WIP") - return None - # src = { - # "app": { - # "config_schema": None, - # "plugin_managers": { - # 'tree': None, - # 'rules': None, - # } - # } - # - # r1 = BackendsManager.get_schema(KheopsPlugins, mode='parts') - # print (json.dumps(r1, indent=2)) + # def DEPRECATED_lookup( + # self, + # keys=None, + # policy=None, + # scope=None, + # trace=False, + # explain=False, + # validate_schema=False, + # ): + # """Lookup a key in hierarchy""" + # log.debug("Lookup key %s with scope: %s", keys, scope) + # assert isinstance(keys, list), f"Got {keys}" - # ret = { - # - # } + # query = Query(app=self) + # ret = {} + # for key in keys: + # ret[key] = query.exec( + # key=key, + # scope=scope, + # policy=policy, + # trace=trace, + # explain=explain, + # validate_schema=validate_schema, + # ) + # return ret - # part_config = r1.get('config_schema', None) - # part_item = r1['items']['core_schema'] - # part_item_plugins = r1['items']['plugin'] + # def DEPRECATED_dump_schema(self): + # """Dump configuration schema""" - # for kind, plugins in part_item_plugins.items(): + # ret1 = BackendsManager.get_schema(KheopsPlugins, mode="parts") + # ret2 = RulesManager.get_schema(KheopsPlugins) + # print(json.dumps(ret1, indent=2)) + # return - # for plugin_name, schema in plugins.items(): - # part_item_ + # # ret = self.schema + # # ret["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = ret1 + # # ret["patternProperties"][".*"]["properties"]["tree"]["items"] = ret2 + + # # print(json.dumps(ret, indent=2)) + + # def DEPRECATED_gen_docs(self): + # """Generate documentation""" + + # print("WIP") + # return None + + # # src = { + # # "app": { + # # "config_schema": None, + # # "plugin_managers": { + # # 'tree': None, + # # 'rules': None, + # # } + # # } + # # + # # r1 = BackendsManager.get_schema(KheopsPlugins, mode='parts') + + # # print (json.dumps(r1, indent=2)) + + # # ret = { + # # + # # } + + # # part_config = r1.get('config_schema', None) + # # part_item = r1['items']['core_schema'] + # # part_item_plugins = r1['items']['plugin'] + + # # for kind, plugins in part_item_plugins.items(): + + # # for plugin_name, schema in plugins.items(): + # # part_item_ diff --git a/kheops/controllers.py b/kheops/controllers.py index e0dccf7..001cd3b 100644 --- a/kheops/controllers.py +++ b/kheops/controllers.py @@ -9,6 +9,8 @@ from prettytable import PrettyTable import kheops.plugin as KheopsPlugins from kheops.utils import render_template, render_template_python, str_ellipsis +from pprint import pprint + log = logging.getLogger(__name__) tracer = logging.getLogger(f"{__name__}.explain") @@ -73,7 +75,7 @@ class QueryProcessor: default_match_rule = { "key": None, "continue": False, - "strategy": "merge_deep", + "strategy": "merge_schema", } default_lookup_item = { @@ -100,6 +102,7 @@ class QueryProcessor: tracer.setLevel(logging.DEBUG) query = Query(key, scope) + log.info("Creating new query: %s", query.__dict__) # Match the KeyRule in keys (RULE CACHE) # Get the matching keys @@ -199,11 +202,11 @@ class QueryProcessor: [ "\nStatus:" + str_ellipsis(col1, 80), "\nRuntime:" + str_ellipsis(col2, 60), - "\nData:" + str_ellipsis(col3, 60), + "\nKey:" + str_ellipsis(col3, 60), ] ) - table.field_names = ["Status", "Runtime", "Data"] + table.field_names = ["Status", "Runtime", "Key Value"] table.align = "l" tracer.info("Explain candidates:\n" + str(table)) diff --git a/kheops/plugin/backend/file.py b/kheops/plugin/backend/file.py index f3b686c..d0bac91 100644 --- a/kheops/plugin/backend/file.py +++ b/kheops/plugin/backend/file.py @@ -93,7 +93,7 @@ class Plugin(BackendPlugin): status = "not_found" for ext, parser in self.extensions.items(): new_path = os.path.join(self.top_path, path + ext) - + log.debug("Looking into %s", new_path) if os.path.isfile(new_path): status = "found" try: diff --git a/kheops/plugin/common.py b/kheops/plugin/common.py index 6bacc43..d6ed66c 100644 --- a/kheops/plugin/common.py +++ b/kheops/plugin/common.py @@ -112,9 +112,10 @@ class ScopeExtLoop: } def loop_over( - self, lookups, conf, var_name="item", callback_context=None, callback=None + self, lookups, conf, module_name, var_name="item", callback_context=None, callback=None ): + var_name = conf.get("var", var_name) var_data_ref = conf.get("data", None) @@ -131,7 +132,7 @@ class ScopeExtLoop: var_data = lookup["_run"]["scope"][var_data] except KeyError: log.debug("Ignoring missing '%s' from scope", var_data) - pass + continue # Run callback if callback: @@ -139,14 +140,14 @@ class ScopeExtLoop: # Validate generated if not isinstance(var_data, list): - log.warning("Hier data must be a list, got: %s", var_data) - pass + log.warning("Loop data must be a list, got: %s", var_data) + continue # Create new object for index, var_value in enumerate(var_data): - if not "hier" in lookup["_run"]: - lookup["_run"]["hier"] = [] + if not module_name in lookup["_run"]: + lookup["_run"][module_name] = [] ctx = { "data_ref": var_data_ref, @@ -157,7 +158,7 @@ class ScopeExtLoop: new_item = copy.deepcopy(lookup) new_item["_run"]["scope"][var_name] = var_value - new_item["_run"]["hier"].append(ctx) + new_item["_run"][module_name].append(ctx) ret.append(new_item) diff --git a/kheops/plugin/scope/hier.py b/kheops/plugin/scope/hier.py index 2df81f4..43d634e 100644 --- a/kheops/plugin/scope/hier.py +++ b/kheops/plugin/scope/hier.py @@ -61,6 +61,11 @@ class Plugin(ScopePlugin, ScopeExtLoop): def _process_item(self, data, ctx): + # Validate data + if not isinstance(data, str): + log.debug("Hier data must be a list, got: %s", data) + return [] + return path_assemble_hier( data, sep=ctx["var_split"], @@ -78,6 +83,7 @@ class Plugin(ScopePlugin, ScopeExtLoop): lookups = self.loop_over( lookups, + module_name='hier', conf=conf, var_name="item_hier", callback=self._process_item, diff --git a/kheops/plugin/scope/loop.py b/kheops/plugin/scope/loop.py index 3d2b80a..2c1153f 100644 --- a/kheops/plugin/scope/loop.py +++ b/kheops/plugin/scope/loop.py @@ -63,6 +63,7 @@ class Plugin(ScopePlugin, ScopeExtLoop): lookups = self.loop_over( lookups, + module_name='loop', conf=conf, var_name="item_loop", ) diff --git a/kheops/plugin/strategy/__init__.py b/kheops/plugin/strategy/__init__.py index ceb86f1..339692a 100644 --- a/kheops/plugin/strategy/__init__.py +++ b/kheops/plugin/strategy/__init__.py @@ -2,3 +2,4 @@ from . import last from . import merge_deep +from . import merge_schema diff --git a/kheops/plugin/strategy/merge_deep.py b/kheops/plugin/strategy/merge_deep.py index 107d2e7..a0c8c19 100644 --- a/kheops/plugin/strategy/merge_deep.py +++ b/kheops/plugin/strategy/merge_deep.py @@ -1,4 +1,4 @@ -"""Last strategy Plugin""" +"""Merge Deep strategy Plugin""" import logging from mergedeep import merge, Strategy diff --git a/kheops/utils.py b/kheops/utils.py index 6802efb..788fa8b 100644 --- a/kheops/utils.py +++ b/kheops/utils.py @@ -6,6 +6,7 @@ from pathlib import Path from jinja2 import Template from jsonschema import Draft7Validator, validators +from pprint import pprint log = logging.getLogger(__name__) @@ -108,7 +109,7 @@ def _extend_with_default(validator_class): ): continue except Exception as err: - print("CATCHED2222 ", err) + log.debug("Jsonschema validation error: %s", err) return validators.extend( validator_class, @@ -124,7 +125,7 @@ def schema_validate(config, schema): try: DefaultValidatingDraft7Validator(schema).validate(config) except Exception as err: - print(err) + log.error(err) path = list(collections.deque(err.schema_path)) path = "/".join([str(i) for i in path]) path = f"schema/{path}"