Add: Caching mechanism with diskcache

kheops/app.py

@@ -13,11 +13,12 @@ from diskcache import Cache
 
 import kheops.plugin as KheopsPlugins
 from kheops.controllers import QueryProcessor
-from kheops.utils import schema_validate
+from kheops.utils import schema_validate, dict_hash
 
 
 log = logging.getLogger(__name__)
 
+CACHE_CONFIG_EXPIRE=15
 CONF_SCHEMA = {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
@@ -122,14 +123,22 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
         :type config: Any
         """
 
-        config = schema_validate(config, CONF_SCHEMA)
-        super().__init__(config)
-
+        # Init object
         self.name = name
         self.app = app
         self.run = dict(app.run)
+        self.cache = app.cache
+
+        # Init config (from cache)
+        config_hash = 'conf_ns_' + dict_hash(config)
+        try:
+            config = self.cache[config_hash]
+            log.debug("Loading namespace '%s' configuration from cache", self.name)
+        except KeyError as err:
+            config = schema_validate(config, CONF_SCHEMA)
+            self.cache.set(config_hash, config, expire=CACHE_CONFIG_EXPIRE)
+        super().__init__(config)
 
-        # Validate configuration
         self.run["path_ns"] = str(Path(app.run["config_src"]).parent.resolve())
 
 
@@ -139,7 +148,7 @@ class Kheops(GenericInstance):
 
     """
 
-    def __init__(self, config="kheops.yml", namespace="default"):
+    def __init__(self, config="kheops.yml", namespace="default", cache=None):
         """
         Kheops Application Instance
 
@@ -166,8 +175,18 @@ class Kheops(GenericInstance):
 
         self.ns_name = namespace
         self.namespaces = {}
+
+        self.cache = cache or Cache("/tmp/kheops_cache/")
         self.raw_config = self.parse_conf(config)
 
+        #needle = 'conf_app_' + dict_hash(config)
+        #try:
+        #    self.raw_config = self.cache[needle]
+        #except KeyError:
+        #    self.raw_config = self.parse_conf(config)
+        #    self.cache.set(needle, config, expire=CACHE_CONFIG_EXPIRE)
+
+
     def parse_conf(self, config="kheops.yml"):
         """
         Parse Kheops configuration
@@ -216,11 +235,12 @@ class Kheops(GenericInstance):
         :type scope: dict
         """
 
-        ret = {}
         # Loop over keys
+        ret = {}
         for key_def in keys:
 
             key_def = key_def or ""
+            assert isinstance(key_def, str), f"Expected string as key, got {type(key_def)}: {key_def}"
 
             # Identify namespace and key
             parts = key_def.split("/")
@@ -259,6 +279,9 @@ class Kheops(GenericInstance):
             #    log.debug("Return '%s' result", key_name)
             #    return result
 
+        if explain:
+            # This is never a really good idea to show direct data ...
+            log.debug("Returned result: %s", ret)
         return ret
 
 

kheops/controllers.py

@@ -13,12 +13,12 @@ from pathlib import Path
 from prettytable import PrettyTable
 
 import kheops.plugin as KheopsPlugins
-from kheops.utils import render_template_python, str_ellipsis
+from kheops.utils import render_template_python, str_ellipsis, dict_hash
 
 
 log = logging.getLogger(__name__)
 tracer = logging.getLogger(f"{__name__}.explain")
-
+CACHE_QUERY_EXPIRE = 10
 
 
 # Helper classes
@@ -138,6 +138,13 @@ class QueryProcessor:
 
         """
 
+        # Look into cache
+        query_hash = dict_hash([self.name, key, scope])
+        if query_hash in self.cache:
+            log.debug("Result fetched from cache")
+            self.cache.touch(query_hash, expire=CACHE_QUERY_EXPIRE)
+            return self.cache[query_hash]
+
         if explain:
             tracer.setLevel(logging.DEBUG)
 
@@ -185,6 +192,7 @@ class QueryProcessor:
         # TODO: Apply output plugins
         # result = self._exec_output_plugins(result)
 
+        self.cache.set(query_hash, result, expire=CACHE_QUERY_EXPIRE)
         return result
 
 

kheops/plugin/backend/file.py

@@ -9,6 +9,7 @@ from anyconfig.common.errors import BaseError as AnyConfigBaseError
 from kheops.plugin.common import BackendPlugin, BackendCandidate
 
 log = logging.getLogger(__name__)
+CACHE_FILE_EXPIRE=5
 
 class Plugin(BackendPlugin):
     """File Backend Plugin
@@ -118,6 +119,8 @@ class Plugin(BackendPlugin):
 
     def fetch_data(self, config) -> list:
 
+        cache = self.ns.cache
+
         path = config.get("path")
         if self.path_suffix:
             path = f"{path}{self.path_suffix}"
@@ -127,11 +130,21 @@ class Plugin(BackendPlugin):
         extensions = self.config.get("extensions", self.extensions)
         for ext, parser in extensions.items():
             new_path = os.path.join(self.top_path, path + ext)
+            cache_key = "file_content_" + new_path
+
+            # Check first if content exists in cache
+            try:
+                raw_data = cache[cache_key]
+                status = "found"
+                #log.info("Found cached: %s with %s", new_path, raw_data)
+                break
+            except KeyError:
                 if os.path.isfile(new_path):
                     status = "found"
                     try:
                         log.info("Found file: %s", new_path)
                         raw_data = anyconfig.load(new_path, ac_parser=parser)
+                        cache.set(cache_key, raw_data, expire=CACHE_FILE_EXPIRE)
                     except AnyConfigBaseError as err:
                         status = "broken"
                         raw_data = None