kheops/app.py

@@ -13,12 +13,11 @@ from diskcache import Cache
 
 import kheops.plugin as KheopsPlugins
 from kheops.controllers import QueryProcessor
-from kheops.utils import schema_validate, dict_hash
+from kheops.utils import schema_validate
 
 
 log = logging.getLogger(__name__)
 
-CACHE_CONFIG_EXPIRE = 15
 CONF_SCHEMA = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
@@ -45,6 +44,7 @@ CONF_SCHEMA = {
             "default": [],
             # "arrayItem":  { "$ref": "#/$defs/rules_items" },
         },
+
         "config": {
             "type": "object",
             "default": {},
@@ -108,7 +108,6 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
     Kheops Namespace Class
 
     """
-
     def __init__(self, app, name, config=None):
         """
         Kheops Namespace Instance
@@ -123,22 +122,14 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
         :type config: Any
         """
 
-        # Init object
+        config = schema_validate(config, CONF_SCHEMA)
+        super().__init__(config)
+
         self.name = name
         self.app = app
         self.run = dict(app.run)
-        self.cache = app.cache
-
-        # Init config (from cache)
-        config_hash = "conf_ns_" + dict_hash(config)
-        try:
-            config = self.cache[config_hash]
-            log.debug("Loading namespace '%s' configuration from cache", self.name)
-        except KeyError:
-            config = schema_validate(config, CONF_SCHEMA)
-            self.cache.set(config_hash, config, expire=CACHE_CONFIG_EXPIRE)
-        super().__init__(config)
 
+        # Validate configuration
         self.run["path_ns"] = str(Path(app.run["config_src"]).parent.resolve())
 
 
@@ -148,7 +139,7 @@ class Kheops(GenericInstance):
 
     """
 
-    def __init__(self, config="kheops.yml", namespace="default", cache=None):
+    def __init__(self, config="kheops.yml", namespace="default"):
         """
         Kheops Application Instance
 
@@ -175,17 +166,8 @@ class Kheops(GenericInstance):
 
         self.ns_name = namespace
         self.namespaces = {}
-
-        self.cache = cache or Cache("/tmp/kheops_cache/")
         self.raw_config = self.parse_conf(config)
 
-        # needle = 'conf_app_' + dict_hash(config)
-        # try:
-        #    self.raw_config = self.cache[needle]
-        # except KeyError:
-        #    self.raw_config = self.parse_conf(config)
-        #    self.cache.set(needle, config, expire=CACHE_CONFIG_EXPIRE)
-
     def parse_conf(self, config="kheops.yml"):
         """
         Parse Kheops configuration
@@ -208,13 +190,12 @@ class Kheops(GenericInstance):
         elif isinstance(config, dict):
             dict_conf = config
             source = "dict"
-
-        self.run["conf_source"] = source
         return dict_conf
 
     def lookup(
         self,
         keys=None,
+        policy=None,
         scope=None,
         trace=False,
         explain=False,
@@ -235,14 +216,11 @@ class Kheops(GenericInstance):
         :type scope: dict
         """
 
-        # Loop over keys
         ret = {}
+        # Loop over keys
         for key_def in keys:
 
             key_def = key_def or ""
-            assert isinstance(
-                key_def, str
-            ), f"Expected string as key, got {type(key_def)}: {key_def}"
 
             # Identify namespace and key
             parts = key_def.split("/")
@@ -269,7 +247,7 @@ class Kheops(GenericInstance):
 
             # Prepare output
             _key = key_name
-            if namespace_prefix is True:
+            if namespace_prefix == True:
                 _key = key_def
             ret[_key] = result
 
@@ -281,12 +259,15 @@ class Kheops(GenericInstance):
             #    log.debug("Return '%s' result", key_name)
             #    return result
 
-        if explain:
-            # This is never a really good idea to show direct data ...
-            log.debug("Returned result: %s", ret)
         return ret
 
 
+
+
+
+
+
+
 # To clean/implement
 
     # def DEPRECATED_dump_schema(self):

kheops/controllers.py

@@ -13,18 +13,17 @@ from pathlib import Path
 from prettytable import PrettyTable
 
 import kheops.plugin as KheopsPlugins
-from kheops.utils import render_template_python, str_ellipsis, dict_hash
+from kheops.utils import render_template_python, str_ellipsis
 
 
 log = logging.getLogger(__name__)
 tracer = logging.getLogger(f"{__name__}.explain")
-CACHE_QUERY_EXPIRE = 10
+
 
 
 # Helper classes
 # ------------------------
 
-
 class LoadPlugin:
     """Kheops plugins loader
 
@@ -61,8 +60,7 @@ class LoadPlugin:
         # Return plugin Classe
         return plugin_cls.Plugin
 
-
+class BackendCandidate():
-class BackendCandidate:
     """Backend Candidate
 
     This object represents a backend candidate. It holds the value of the
@@ -99,7 +97,6 @@ class Query:
 # Query Processor class
 # ------------------------
 
-
 class QueryProcessor:
     """QueryProcessor
 
@@ -137,20 +134,15 @@ class QueryProcessor:
     # ------------------------
 
     def query(self, key=None, scope=None, explain=False):
-        """Query key with scope"""
+        """Query key with scope
 
-        # Look into cache
+        """
-        query_hash = dict_hash([self.name, key, scope])
-        if query_hash in self.cache:
-            log.debug("Result fetched from cache")
-            self.cache.touch(query_hash, expire=CACHE_QUERY_EXPIRE)
-            return self.cache[query_hash]
 
         if explain:
             tracer.setLevel(logging.DEBUG)
 
         query = Query(key, scope)
-        log.info("Creating new query for namespace '%s': %s", self.name, query.__dict__)
+        log.info("Creating new query: %s", query.__dict__)
 
         # Match the KeyRule in keys (RULE CACHE)
         # Get the matching keys
@@ -168,7 +160,7 @@ class QueryProcessor:
 
         # Generate explain report
         if explain:
-            self._explain_lookups(parsed_lookups, query)
+            self._explain_lookups(parsed_lookups)
 
         # Fetch the module
         # Retrieve the module instance
@@ -193,9 +185,9 @@ class QueryProcessor:
         # TODO: Apply output plugins
         # result = self._exec_output_plugins(result)
 
-        self.cache.set(query_hash, result, expire=CACHE_QUERY_EXPIRE)
         return result
 
+
     # Query parts methods
     # ------------------------
 
@@ -218,6 +210,7 @@ class QueryProcessor:
 
         return rule
 
+
     def _exec_assemble_lookups(self, lookups, query):
 
         assert isinstance(lookups, list)
@@ -257,9 +250,7 @@ class QueryProcessor:
                 plugin_name = plugin_def.get("module", None)
 
                 if plugin_name:
-                    plugin = self.plugin_loader.load("scope", plugin_name)(
+                    plugin = self.plugin_loader.load("scope", plugin_name)(namespace=self)
-                        namespace=self
-                    )
                     ret = plugin.process_items(ret, plugin_def)
 
             new_lookups2.extend(ret)
@@ -275,12 +266,11 @@ class QueryProcessor:
                 lookup["path"] = new_path
                 new_lookups3.append(lookup)
             else:
-                log.warning(
+                log.warning("Ignore lookup item because of missing scope vars: '%s'", path)
-                    "Ignore lookup item because of missing scope vars: '%s'", path
-                )
 
         return new_lookups3
 
+
     def _exec_backend_plugins(self, lookups, selector="matched"):
         selector = "matched"
         assert selector in ["last", "first", "all", "matched"]
@@ -314,7 +304,7 @@ class QueryProcessor:
     # Explain methods
     # ------------------------
 
-    def _explain_lookups(self, parsed_lookups, query):
+    def _explain_lookups(self, parsed_lookups):
         """Explain list of lookups"""
 
         table = PrettyTable()
@@ -335,7 +325,7 @@ class QueryProcessor:
             )
         table.field_names = ["Config", "Runtime"]
         table.align = "l"
-        tracer.info("Explain lookups: %s/%s\n%s", self.name, query.key, str(table))
+        tracer.info("Explain lookups:\n%s", str(table))
 
     def _explain_candidates(self, candidates, query):
         """Explain list of candidates"""
@@ -371,4 +361,4 @@ class QueryProcessor:
 
         table.field_names = ["Status", "Runtime", "Key Value"]
         table.align = "l"
-        tracer.info("Explain candidates: %s/%s\n%s", self.name, query.key, str(table))
+        tracer.info("Explain candidates:\n%s", str(table))

kheops/plugin/backend/file.py

@@ -2,7 +2,6 @@
 
 import os
 import logging
-
 # from pprint import pprint
 
 import anyconfig
@@ -10,8 +9,6 @@ from anyconfig.common.errors import BaseError as AnyConfigBaseError
 from kheops.plugin.common import BackendPlugin, BackendCandidate
 
 log = logging.getLogger(__name__)
-CACHE_FILE_EXPIRE = 5
-
 
 class Plugin(BackendPlugin):
     """File Backend Plugin
@@ -43,12 +40,13 @@ class Plugin(BackendPlugin):
                         to find all of the specified format. It is better to keep this list as small
                         as possible.
                         """,
+
                         "type": "object",
                         "default": extensions,
                         "additionalProperties": {
                                 "title": "Name of the extension with parser",
-                        "type": "string",
+                                "type": "string"
-                    },
+                            }
                     },
                 "path_prefix": {
                         "title": "Prefix string to append to final path",
@@ -56,7 +54,7 @@ class Plugin(BackendPlugin):
                         String to be added at the end of the resolved path. This is useful to change
                         the place of the root hierarchy.
                         """,
-                    "type": "string",
+                        "type": "string"
                     },
                 "path_suffix": {
                         "title": "Suffix string to prepend to final path",
@@ -66,11 +64,12 @@ class Plugin(BackendPlugin):
                         "type": "string",
                         "examples": [
                             { "path_suffix": "/ansible" },
-                    ],
+                            ]
-                },
                     },
                 }
             }
+        }
+
 
     _schema_props_new = {
         "path": {
@@ -119,8 +118,6 @@ class Plugin(BackendPlugin):
 
     def fetch_data(self, config) -> list:
 
-        cache = self.ns.cache
-
         path = config.get("path")
         if self.path_suffix:
             path = f"{path}{self.path_suffix}"
@@ -130,21 +127,11 @@ class Plugin(BackendPlugin):
         extensions = self.config.get("extensions", self.extensions)
         for ext, parser in extensions.items():
             new_path = os.path.join(self.top_path, path + ext)
-            cache_key = "file_content_" + new_path
-
-            # Check first if content exists in cache
-            try:
-                raw_data = cache[cache_key]
-                status = "found"
-                # log.info("Found cached: %s with %s", new_path, raw_data)
-                break
-            except KeyError:
             if os.path.isfile(new_path):
                 status = "found"
                 try:
                     log.info("Found file: %s", new_path)
                     raw_data = anyconfig.load(new_path, ac_parser=parser)
-                        cache.set(cache_key, raw_data, expire=CACHE_FILE_EXPIRE)
                 except AnyConfigBaseError as err:
                     status = "broken"
                     raw_data = None

kheops/plugin/common.py

@@ -6,7 +6,7 @@ from kheops.utils import schema_validate
 from pprint import pprint
 
 log = logging.getLogger(__name__)
-NoneType = type(None)
+
 
 # Vocabulary:
 # Key Rules
@@ -22,16 +22,13 @@ NoneType = type(None)
 # Generic Plugin classes
 # -------------------------
 
-
 class KheopsPlugin:
     plugin_name = None
     plugin_type = None
     plugin_kind = None
 
     def __init__(self):
-        assert isinstance(
+        assert isinstance(self.plugin_name, str), f"Missing name attribute in plugin: {self.__class__}"
-            self.plugin_name, str
-        ), f"Missing name attribute in plugin: {self.__class__}"
         assert isinstance(self.plugin_kind, str)
 
         config_key = f"{self.plugin_kind}_{self.plugin_name}"
@@ -64,7 +61,6 @@ class KheopsItemPlugin(KheopsPlugin):
 # Plugin classes
 # -------------------------
 
-
 class BackendPlugin(KheopsItemPlugin):
     plugin_kind = "backend"
 
@@ -144,14 +140,11 @@ class ScopePlugin(KheopsListPlugin):
         self.ns = namespace
         super().__init__()
 
-
 # Helper classes
 # -------------------------
 
-
+class BackendCandidate():
-class BackendCandidate:
     """Represent a backend candidate"""
-
     def __init__(self, path=None, data=None, run=None, status=None):
         assert isinstance(run, dict)
         self.path = path
@@ -163,6 +156,7 @@ class BackendCandidate:
         return f"Status: {self.status}, Path: {self.path} => {self.data}"
 
 
+
 class ScopeExtLoop:
     """This Scope Extension allow to loop over a lookup"""
 
@@ -185,15 +179,10 @@ class ScopeExtLoop:
     }
 
     def loop_over(
-        self,
+        self, lookups, conf, module_name, var_name="item", callback_context=None, callback=None
-        lookups,
-        conf,
-        module_name,
-        var_name="item",
-        callback_context=None,
-        callback=None,
     ):
 
+
         var_name = conf.get("var", var_name)
         var_data_ref = conf.get("data", None)
 
@@ -218,8 +207,7 @@ class ScopeExtLoop:
 
             # Validate generated
             if not isinstance(var_data, list):
-                if not isinstance(var_data, NoneType):
+                log.warning("Loop data must be a list, got: %s", var_data)
-                    log.warning("Loop data must be a list, got: '%s'", var_data)
                 continue
 
             # Create new object
@@ -235,8 +223,6 @@ class ScopeExtLoop:
                     "variable": var_name,
                 }
 
-                # Note: This implie a performance penalty to do so, but
-                # we really need a full copy of the dict. copy.copy or dict() are not enough
                 new_item = copy.deepcopy(lookup)
                 new_item["_run"]["scope"][var_name] = var_value
                 new_item["_run"][module_name].append(ctx)
@@ -246,6 +232,8 @@ class ScopeExtLoop:
         return ret
 
 
+
+
 # To clean/implement
 
 

kheops/utils.py

@@ -4,10 +4,6 @@ import collections
 import logging
 from pathlib import Path
 
-import hashlib
-import json
-from typing import Dict, Any
-
 from jinja2 import Template
 from jsonschema import Draft7Validator, validators
 from pprint import pprint
@@ -78,17 +74,6 @@ class Default(dict):
         return ""
 
 
-# Source: https://www.doc.ic.ac.uk/~nuric/coding/how-to-hash-a-dictionary-in-python.html
-def dict_hash(dictionary: Dict[str, Any]) -> str:
-    """MD5 hash of a dictionary."""
-    dhash = hashlib.md5()
-    # We need to sort arguments so {'a': 1, 'b': 2} is
-    # the same as {'b': 2, 'a': 1}
-    encoded = json.dumps(dictionary, sort_keys=True).encode()
-    dhash.update(encoded)
-    return dhash.hexdigest()
-
-
 def render_template_python(text, params, ignore_missing=True):
     """Render template for a given string"""
     assert isinstance(params, dict), f"Got: {params}"
@@ -105,7 +90,6 @@ def render_template_python(text, params, ignore_missing=True):
 # Schema Methods
 # =====================
 
-
 def _extend_with_default(validator_class):
     validate_properties = validator_class.VALIDATORS["properties"]
 
@@ -115,16 +99,12 @@ def _extend_with_default(validator_class):
                 instance.setdefault(property, subschema["default"])
 
         for error in validate_properties(
-            validator,
+            validator, properties, instance, schema,
-            properties,
-            instance,
-            schema,
         ):
             yield error
 
     return validators.extend(
-        validator_class,
+        validator_class, {"properties" : set_defaults},
-        {"properties": set_defaults},
     )