From ffdbfc442b7e381cb44bf0c135b1270dfb5b2530 Mon Sep 17 00:00:00 2001 From: mrjk Date: Wed, 4 May 2022 19:39:21 -0400 Subject: [PATCH] Lint: some parts of the code --- kheops/app.py | 122 ++++++++++++++++------------------ kheops/controllers.py | 24 ++++--- kheops/plugin/backend/file.py | 62 ++++++++--------- kheops/plugin/common.py | 25 ++++--- kheops/utils.py | 10 ++- 5 files changed, 128 insertions(+), 115 deletions(-) diff --git a/kheops/app.py b/kheops/app.py index e7f85db..912fa41 100644 --- a/kheops/app.py +++ b/kheops/app.py @@ -18,26 +18,26 @@ from kheops.utils import schema_validate, dict_hash log = logging.getLogger(__name__) -CACHE_CONFIG_EXPIRE=15 +CACHE_CONFIG_EXPIRE = 15 CONF_SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": False, "default": {}, "required": ["config"], - #"$def": { + # "$def": { # "backends_items": {}, # "backends_config": {}, # "rules_items": {}, # "rules_config": {}, - #}, + # }, "properties": { "lookups": { "type": "array", "default": [], "items": { "type": "object", - #"properties": {"$ref": "#/$defs/backends_items"}, + # "properties": {"$ref": "#/$defs/backends_items"}, }, }, "rules": { @@ -45,12 +45,11 @@ CONF_SCHEMA = { "default": [], # "arrayItem": { "$ref": "#/$defs/rules_items" }, }, - "config": { "type": "object", "default": {}, "additionalProperties": True, - #"required": ["app"], + # "required": ["app"], "properties": { "app": { "type": "object", @@ -109,17 +108,18 @@ class KheopsNamespace(GenericInstance, QueryProcessor): Kheops Namespace Class """ + def __init__(self, app, name, config=None): """ Kheops Namespace Instance - :param app: Parent Kheops Application. + :param app: Parent Kheops Application. :type app: Kheops - :param name: Namespace name. + :param name: Namespace name. :type config: str - :param config: Namespace configuration. + :param config: Namespace configuration. :type config: Any """ @@ -130,11 +130,11 @@ class KheopsNamespace(GenericInstance, QueryProcessor): self.cache = app.cache # Init config (from cache) - config_hash = 'conf_ns_' + dict_hash(config) + config_hash = "conf_ns_" + dict_hash(config) try: config = self.cache[config_hash] log.debug("Loading namespace '%s' configuration from cache", self.name) - except KeyError as err: + except KeyError: config = schema_validate(config, CONF_SCHEMA) self.cache.set(config_hash, config, expire=CACHE_CONFIG_EXPIRE) super().__init__(config) @@ -152,7 +152,7 @@ class Kheops(GenericInstance): """ Kheops Application Instance - :param config: Kheops configuration. If it's a string, it loads the config from file path. + :param config: Kheops configuration. If it's a string, it loads the config from file path. :type config: str or dict """ @@ -179,21 +179,20 @@ class Kheops(GenericInstance): self.cache = cache or Cache("/tmp/kheops_cache/") self.raw_config = self.parse_conf(config) - #needle = 'conf_app_' + dict_hash(config) - #try: + # needle = 'conf_app_' + dict_hash(config) + # try: # self.raw_config = self.cache[needle] - #except KeyError: + # except KeyError: # self.raw_config = self.parse_conf(config) # self.cache.set(needle, config, expire=CACHE_CONFIG_EXPIRE) - def parse_conf(self, config="kheops.yml"): """ Parse Kheops configuration :param config: Kheops configuration, can either be a file path or a dict. :type config: dict or str or None - + :return: The parsed configuration. :rtype: dict @@ -204,17 +203,18 @@ class Kheops(GenericInstance): try: dict_conf = anyconfig.load(config) except Exception as err: - raise Exception ("Can't load kheops configuration, got: %s", err) + raise Exception("Can't load kheops configuration, got: %s", err) source = f"file:{config}" elif isinstance(config, dict): dict_conf = config source = "dict" + + self.run["conf_source"] = source return dict_conf def lookup( self, keys=None, - policy=None, scope=None, trace=False, explain=False, @@ -240,7 +240,9 @@ class Kheops(GenericInstance): for key_def in keys: key_def = key_def or "" - assert isinstance(key_def, str), f"Expected string as key, got {type(key_def)}: {key_def}" + assert isinstance( + key_def, str + ), f"Expected string as key, got {type(key_def)}: {key_def}" # Identify namespace and key parts = key_def.split("/") @@ -267,15 +269,15 @@ class Kheops(GenericInstance): # Prepare output _key = key_name - if namespace_prefix == True: + if namespace_prefix is True: _key = key_def ret[_key] = result # TODO: This may lead to inconsistant output format :/ # Return result - #if len(keys) > 1: + # if len(keys) > 1: # log.debug("Append '%s' to results", key_name) - #else: + # else: # log.debug("Return '%s' result", key_name) # return result @@ -285,56 +287,50 @@ class Kheops(GenericInstance): return ret - - - - - - # To clean/implement - # def DEPRECATED_dump_schema(self): - # """Dump configuration schema""" +# def DEPRECATED_dump_schema(self): +# """Dump configuration schema""" - # ret1 = BackendsManager.get_schema(KheopsPlugins, mode="parts") - # ret2 = RulesManager.get_schema(KheopsPlugins) - # print(json.dumps(ret1, indent=2)) - # return +# ret1 = BackendsManager.get_schema(KheopsPlugins, mode="parts") +# ret2 = RulesManager.get_schema(KheopsPlugins) +# print(json.dumps(ret1, indent=2)) +# return - # # ret = self.schema - # # ret["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = ret1 - # # ret["patternProperties"][".*"]["properties"]["tree"]["items"] = ret2 +# # ret = self.schema +# # ret["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = ret1 +# # ret["patternProperties"][".*"]["properties"]["tree"]["items"] = ret2 - # # print(json.dumps(ret, indent=2)) +# # print(json.dumps(ret, indent=2)) - # def DEPRECATED_gen_docs(self): - # """Generate documentation""" +# def DEPRECATED_gen_docs(self): +# """Generate documentation""" - # print("WIP") - # return None +# print("WIP") +# return None - # # src = { - # # "app": { - # # "config_schema": None, - # # "plugin_managers": { - # # 'tree': None, - # # 'rules': None, - # # } - # # } - # # - # # r1 = BackendsManager.get_schema(KheopsPlugins, mode='parts') +# # src = { +# # "app": { +# # "config_schema": None, +# # "plugin_managers": { +# # 'tree': None, +# # 'rules': None, +# # } +# # } +# # +# # r1 = BackendsManager.get_schema(KheopsPlugins, mode='parts') - # # print (json.dumps(r1, indent=2)) +# # print (json.dumps(r1, indent=2)) - # # ret = { - # # - # # } +# # ret = { +# # +# # } - # # part_config = r1.get('config_schema', None) - # # part_item = r1['items']['core_schema'] - # # part_item_plugins = r1['items']['plugin'] +# # part_config = r1.get('config_schema', None) +# # part_item = r1['items']['core_schema'] +# # part_item_plugins = r1['items']['plugin'] - # # for kind, plugins in part_item_plugins.items(): +# # for kind, plugins in part_item_plugins.items(): - # # for plugin_name, schema in plugins.items(): - # # part_item_ +# # for plugin_name, schema in plugins.items(): +# # part_item_ diff --git a/kheops/controllers.py b/kheops/controllers.py index 7a36a31..25c149a 100644 --- a/kheops/controllers.py +++ b/kheops/controllers.py @@ -24,6 +24,7 @@ CACHE_QUERY_EXPIRE = 10 # Helper classes # ------------------------ + class LoadPlugin: """Kheops plugins loader @@ -60,7 +61,8 @@ class LoadPlugin: # Return plugin Classe return plugin_cls.Plugin -class BackendCandidate(): + +class BackendCandidate: """Backend Candidate This object represents a backend candidate. It holds the value of the @@ -97,6 +99,7 @@ class Query: # Query Processor class # ------------------------ + class QueryProcessor: """QueryProcessor @@ -134,9 +137,7 @@ class QueryProcessor: # ------------------------ def query(self, key=None, scope=None, explain=False): - """Query key with scope - - """ + """Query key with scope""" # Look into cache query_hash = dict_hash([self.name, key, scope]) @@ -195,7 +196,6 @@ class QueryProcessor: self.cache.set(query_hash, result, expire=CACHE_QUERY_EXPIRE) return result - # Query parts methods # ------------------------ @@ -218,7 +218,6 @@ class QueryProcessor: return rule - def _exec_assemble_lookups(self, lookups, query): assert isinstance(lookups, list) @@ -227,10 +226,10 @@ class QueryProcessor: # Init the scope list new_lookups1 = [] for index, lookup_def in enumerate(lookups): - #shortform = False + # shortform = False if isinstance(lookup_def, str): - #shortform = True + # shortform = True lookup_def = { "path": lookup_def, } @@ -258,7 +257,9 @@ class QueryProcessor: plugin_name = plugin_def.get("module", None) if plugin_name: - plugin = self.plugin_loader.load("scope", plugin_name)(namespace=self) + plugin = self.plugin_loader.load("scope", plugin_name)( + namespace=self + ) ret = plugin.process_items(ret, plugin_def) new_lookups2.extend(ret) @@ -274,11 +275,12 @@ class QueryProcessor: lookup["path"] = new_path new_lookups3.append(lookup) else: - log.warning("Ignore lookup item because of missing scope vars: '%s'", path) + log.warning( + "Ignore lookup item because of missing scope vars: '%s'", path + ) return new_lookups3 - def _exec_backend_plugins(self, lookups, selector="matched"): selector = "matched" assert selector in ["last", "first", "all", "matched"] diff --git a/kheops/plugin/backend/file.py b/kheops/plugin/backend/file.py index b13f1cf..8f3c6a7 100644 --- a/kheops/plugin/backend/file.py +++ b/kheops/plugin/backend/file.py @@ -2,6 +2,7 @@ import os import logging + # from pprint import pprint import anyconfig @@ -9,7 +10,8 @@ from anyconfig.common.errors import BaseError as AnyConfigBaseError from kheops.plugin.common import BackendPlugin, BackendCandidate log = logging.getLogger(__name__) -CACHE_FILE_EXPIRE=5 +CACHE_FILE_EXPIRE = 5 + class Plugin(BackendPlugin): """File Backend Plugin @@ -20,12 +22,12 @@ class Plugin(BackendPlugin): plugin_name = "file" extensions = { - ".yml": "yaml", - ".yaml": "yaml", - #".toml": "toml", - #".ini": "ini", - #".json": "json", - } + ".yml": "yaml", + ".yaml": "yaml", + # ".toml": "toml", + # ".ini": "ini", + # ".json": "json", + } _schema_config = { "backend_file": { @@ -34,43 +36,41 @@ class Plugin(BackendPlugin): "type": "object", "properties": { "extensions": { - "title": "File formats", - "description": """ + "title": "File formats", + "description": """ This object describe which parser is assigned to which extension. Adding more format will have a performance impact because it will try to find all of the specified format. It is better to keep this list as small as possible. """, - - "type": "object", - "default": extensions, - "additionalProperties": { - "title": "Name of the extension with parser", - "type": "string" - } + "type": "object", + "default": extensions, + "additionalProperties": { + "title": "Name of the extension with parser", + "type": "string", }, + }, "path_prefix": { - "title": "Prefix string to append to final path", - "description": """ + "title": "Prefix string to append to final path", + "description": """ String to be added at the end of the resolved path. This is useful to change the place of the root hierarchy. """, - "type": "string" - }, + "type": "string", + }, "path_suffix": { - "title": "Suffix string to prepend to final path", - "description": """ + "title": "Suffix string to prepend to final path", + "description": """ String to be added at the end of the resolved path. This is useful to provide Hiera or Jerakia support.""", - "type": "string", - "examples": [ - { "path_suffix": "/ansible" }, - ] - }, - } - } + "type": "string", + "examples": [ + {"path_suffix": "/ansible"}, + ], + }, + }, } - + } _schema_props_new = { "path": { @@ -136,7 +136,7 @@ class Plugin(BackendPlugin): try: raw_data = cache[cache_key] status = "found" - #log.info("Found cached: %s with %s", new_path, raw_data) + # log.info("Found cached: %s with %s", new_path, raw_data) break except KeyError: if os.path.isfile(new_path): diff --git a/kheops/plugin/common.py b/kheops/plugin/common.py index 9c5e47f..78475fb 100644 --- a/kheops/plugin/common.py +++ b/kheops/plugin/common.py @@ -22,20 +22,23 @@ NoneType = type(None) # Generic Plugin classes # ------------------------- + class KheopsPlugin: plugin_name = None plugin_type = None plugin_kind = None def __init__(self): - assert isinstance(self.plugin_name, str), f"Missing name attribute in plugin: {self.__class__}" + assert isinstance( + self.plugin_name, str + ), f"Missing name attribute in plugin: {self.__class__}" assert isinstance(self.plugin_kind, str) config_key = f"{self.plugin_kind}_{self.plugin_name}" self.config = self.ns.config["config"].get(config_key, {}) self.config_key = config_key - #if self.config: + # if self.config: # log.debug("Load plugin configuration in config with key '%s', got: %s", config_key, self.config) self._init() @@ -61,6 +64,7 @@ class KheopsItemPlugin(KheopsPlugin): # Plugin classes # ------------------------- + class BackendPlugin(KheopsItemPlugin): plugin_kind = "backend" @@ -140,11 +144,14 @@ class ScopePlugin(KheopsListPlugin): self.ns = namespace super().__init__() + # Helper classes # ------------------------- -class BackendCandidate(): + +class BackendCandidate: """Represent a backend candidate""" + def __init__(self, path=None, data=None, run=None, status=None): assert isinstance(run, dict) self.path = path @@ -156,7 +163,6 @@ class BackendCandidate(): return f"Status: {self.status}, Path: {self.path} => {self.data}" - class ScopeExtLoop: """This Scope Extension allow to loop over a lookup""" @@ -179,10 +185,15 @@ class ScopeExtLoop: } def loop_over( - self, lookups, conf, module_name, var_name="item", callback_context=None, callback=None + self, + lookups, + conf, + module_name, + var_name="item", + callback_context=None, + callback=None, ): - var_name = conf.get("var", var_name) var_data_ref = conf.get("data", None) @@ -235,8 +246,6 @@ class ScopeExtLoop: return ret - - # To clean/implement diff --git a/kheops/utils.py b/kheops/utils.py index 89ad75e..9231005 100644 --- a/kheops/utils.py +++ b/kheops/utils.py @@ -77,6 +77,7 @@ class Default(dict): def __missing__(self, key): return "" + # Source: https://www.doc.ic.ac.uk/~nuric/coding/how-to-hash-a-dictionary-in-python.html def dict_hash(dictionary: Dict[str, Any]) -> str: """MD5 hash of a dictionary.""" @@ -104,6 +105,7 @@ def render_template_python(text, params, ignore_missing=True): # Schema Methods # ===================== + def _extend_with_default(validator_class): validate_properties = validator_class.VALIDATORS["properties"] @@ -113,12 +115,16 @@ def _extend_with_default(validator_class): instance.setdefault(property, subschema["default"]) for error in validate_properties( - validator, properties, instance, schema, + validator, + properties, + instance, + schema, ): yield error return validators.extend( - validator_class, {"properties" : set_defaults}, + validator_class, + {"properties": set_defaults}, )