From 2707e3ae767e55e711579fece7ebe5c5b705b9a4 Mon Sep 17 00:00:00 2001 From: mrjk Date: Wed, 4 May 2022 19:31:09 -0400 Subject: [PATCH] Add: Caching mechanism with diskcache --- kheops/app.py | 37 ++++++++++++++++++++++++++++------- kheops/controllers.py | 12 ++++++++++-- kheops/plugin/backend/file.py | 33 +++++++++++++++++++++---------- 3 files changed, 63 insertions(+), 19 deletions(-) diff --git a/kheops/app.py b/kheops/app.py index d035597..e7f85db 100644 --- a/kheops/app.py +++ b/kheops/app.py @@ -13,11 +13,12 @@ from diskcache import Cache import kheops.plugin as KheopsPlugins from kheops.controllers import QueryProcessor -from kheops.utils import schema_validate +from kheops.utils import schema_validate, dict_hash log = logging.getLogger(__name__) +CACHE_CONFIG_EXPIRE=15 CONF_SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", @@ -122,14 +123,22 @@ class KheopsNamespace(GenericInstance, QueryProcessor): :type config: Any """ - config = schema_validate(config, CONF_SCHEMA) - super().__init__(config) - + # Init object self.name = name self.app = app self.run = dict(app.run) + self.cache = app.cache + + # Init config (from cache) + config_hash = 'conf_ns_' + dict_hash(config) + try: + config = self.cache[config_hash] + log.debug("Loading namespace '%s' configuration from cache", self.name) + except KeyError as err: + config = schema_validate(config, CONF_SCHEMA) + self.cache.set(config_hash, config, expire=CACHE_CONFIG_EXPIRE) + super().__init__(config) - # Validate configuration self.run["path_ns"] = str(Path(app.run["config_src"]).parent.resolve()) @@ -139,7 +148,7 @@ class Kheops(GenericInstance): """ - def __init__(self, config="kheops.yml", namespace="default"): + def __init__(self, config="kheops.yml", namespace="default", cache=None): """ Kheops Application Instance @@ -166,8 +175,18 @@ class Kheops(GenericInstance): self.ns_name = namespace self.namespaces = {} + + self.cache = cache or Cache("/tmp/kheops_cache/") self.raw_config = self.parse_conf(config) + #needle = 'conf_app_' + dict_hash(config) + #try: + # self.raw_config = self.cache[needle] + #except KeyError: + # self.raw_config = self.parse_conf(config) + # self.cache.set(needle, config, expire=CACHE_CONFIG_EXPIRE) + + def parse_conf(self, config="kheops.yml"): """ Parse Kheops configuration @@ -216,11 +235,12 @@ class Kheops(GenericInstance): :type scope: dict """ - ret = {} # Loop over keys + ret = {} for key_def in keys: key_def = key_def or "" + assert isinstance(key_def, str), f"Expected string as key, got {type(key_def)}: {key_def}" # Identify namespace and key parts = key_def.split("/") @@ -259,6 +279,9 @@ class Kheops(GenericInstance): # log.debug("Return '%s' result", key_name) # return result + if explain: + # This is never a really good idea to show direct data ... + log.debug("Returned result: %s", ret) return ret diff --git a/kheops/controllers.py b/kheops/controllers.py index 479d48f..7a36a31 100644 --- a/kheops/controllers.py +++ b/kheops/controllers.py @@ -13,12 +13,12 @@ from pathlib import Path from prettytable import PrettyTable import kheops.plugin as KheopsPlugins -from kheops.utils import render_template_python, str_ellipsis +from kheops.utils import render_template_python, str_ellipsis, dict_hash log = logging.getLogger(__name__) tracer = logging.getLogger(f"{__name__}.explain") - +CACHE_QUERY_EXPIRE = 10 # Helper classes @@ -138,6 +138,13 @@ class QueryProcessor: """ + # Look into cache + query_hash = dict_hash([self.name, key, scope]) + if query_hash in self.cache: + log.debug("Result fetched from cache") + self.cache.touch(query_hash, expire=CACHE_QUERY_EXPIRE) + return self.cache[query_hash] + if explain: tracer.setLevel(logging.DEBUG) @@ -185,6 +192,7 @@ class QueryProcessor: # TODO: Apply output plugins # result = self._exec_output_plugins(result) + self.cache.set(query_hash, result, expire=CACHE_QUERY_EXPIRE) return result diff --git a/kheops/plugin/backend/file.py b/kheops/plugin/backend/file.py index c251a98..b13f1cf 100644 --- a/kheops/plugin/backend/file.py +++ b/kheops/plugin/backend/file.py @@ -9,6 +9,7 @@ from anyconfig.common.errors import BaseError as AnyConfigBaseError from kheops.plugin.common import BackendPlugin, BackendCandidate log = logging.getLogger(__name__) +CACHE_FILE_EXPIRE=5 class Plugin(BackendPlugin): """File Backend Plugin @@ -118,6 +119,8 @@ class Plugin(BackendPlugin): def fetch_data(self, config) -> list: + cache = self.ns.cache + path = config.get("path") if self.path_suffix: path = f"{path}{self.path_suffix}" @@ -127,18 +130,28 @@ class Plugin(BackendPlugin): extensions = self.config.get("extensions", self.extensions) for ext, parser in extensions.items(): new_path = os.path.join(self.top_path, path + ext) - if os.path.isfile(new_path): - status = "found" - try: - log.info("Found file: %s", new_path) - raw_data = anyconfig.load(new_path, ac_parser=parser) - except AnyConfigBaseError as err: - status = "broken" - raw_data = None - log.warning("Could not parse file %s: %s", new_path, err) + cache_key = "file_content_" + new_path - # Stop the loop extension if we found a result. + # Check first if content exists in cache + try: + raw_data = cache[cache_key] + status = "found" + #log.info("Found cached: %s with %s", new_path, raw_data) break + except KeyError: + if os.path.isfile(new_path): + status = "found" + try: + log.info("Found file: %s", new_path) + raw_data = anyconfig.load(new_path, ac_parser=parser) + cache.set(cache_key, raw_data, expire=CACHE_FILE_EXPIRE) + except AnyConfigBaseError as err: + status = "broken" + raw_data = None + log.warning("Could not parse file %s: %s", new_path, err) + + # Stop the loop extension if we found a result. + break log.debug("Skip absent file: %s", new_path)