pro-barbu-foss/kheops
pro-barbu-foss/kheops
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: pro-barbu-foss:develop
Branches Tags
pro-barbu-foss:main
pro-barbu-foss:old-dev
pro-barbu-foss:wip
pro-barbu-foss:develop
..
compare: pro-barbu-foss:main
Branches Tags
pro-barbu-foss:old-dev
pro-barbu-foss:wip
pro-barbu-foss:develop
pro-barbu-foss:main

No commits in common. "develop" and "main" have entirely different histories.
develop ... main

5 changed files with 130 additions and 204 deletions
Show Stats Expand all files Collapse all files

51
kheops/app.py
View File

@ -13,12 +13,11 @@ from diskcache import Cache
import kheops.plugin as KheopsPlugins
from kheops.controllers import QueryProcessor
from kheops.utils import schema_validate, dict_hash
from kheops.utils import schema_validate
log = logging.getLogger(__name__)
CACHE_CONFIG_EXPIRE = 15
CONF_SCHEMA = {
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
@ -45,6 +44,7 @@ CONF_SCHEMA = {
"default": [],
# "arrayItem": { "$ref": "#/$defs/rules_items" },
},
"config": {
"type": "object",
"default": {},
@ -108,7 +108,6 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
Kheops Namespace Class
"""
def __init__(self, app, name, config=None):
"""
Kheops Namespace Instance
@ -123,22 +122,14 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
:type config: Any
"""
# Init object
config = schema_validate(config, CONF_SCHEMA)
super().__init__(config)
self.name = name
self.app = app
self.run = dict(app.run)
self.cache = app.cache
# Init config (from cache)
config_hash = "conf_ns_" + dict_hash(config)
try:
config = self.cache[config_hash]
log.debug("Loading namespace '%s' configuration from cache", self.name)
except KeyError:
config = schema_validate(config, CONF_SCHEMA)
self.cache.set(config_hash, config, expire=CACHE_CONFIG_EXPIRE)
super().__init__(config)
# Validate configuration
self.run["path_ns"] = str(Path(app.run["config_src"]).parent.resolve())
@ -148,7 +139,7 @@ class Kheops(GenericInstance):
"""
def __init__(self, config="kheops.yml", namespace="default", cache=None):
def __init__(self, config="kheops.yml", namespace="default"):
"""
Kheops Application Instance
@ -175,17 +166,8 @@ class Kheops(GenericInstance):
self.ns_name = namespace
self.namespaces = {}
self.cache = cache or Cache("/tmp/kheops_cache/")
self.raw_config = self.parse_conf(config)
# needle = 'conf_app_' + dict_hash(config)
# try:
# self.raw_config = self.cache[needle]
# except KeyError:
# self.raw_config = self.parse_conf(config)
# self.cache.set(needle, config, expire=CACHE_CONFIG_EXPIRE)
def parse_conf(self, config="kheops.yml"):
"""
Parse Kheops configuration
@ -208,13 +190,12 @@ class Kheops(GenericInstance):
elif isinstance(config, dict):
dict_conf = config
source = "dict"
self.run["conf_source"] = source
return dict_conf
def lookup(
self,
keys=None,
policy=None,
scope=None,
trace=False,
explain=False,
@ -235,14 +216,11 @@ class Kheops(GenericInstance):
:type scope: dict
"""
# Loop over keys
ret = {}
# Loop over keys
for key_def in keys:
key_def = key_def or ""
assert isinstance(
key_def, str
), f"Expected string as key, got {type(key_def)}: {key_def}"
# Identify namespace and key
parts = key_def.split("/")
@ -269,7 +247,7 @@ class Kheops(GenericInstance):
# Prepare output
_key = key_name
if namespace_prefix is True:
if namespace_prefix == True:
_key = key_def
ret[_key] = result
@ -281,12 +259,15 @@ class Kheops(GenericInstance):
# log.debug("Return '%s' result", key_name)
# return result
if explain:
# This is never a really good idea to show direct data ...
log.debug("Returned result: %s", ret)
return ret
# To clean/implement
# def DEPRECATED_dump_schema(self):

40
kheops/controllers.py
View File

@ -13,18 +13,17 @@ from pathlib import Path
from prettytable import PrettyTable
import kheops.plugin as KheopsPlugins
from kheops.utils import render_template_python, str_ellipsis, dict_hash
from kheops.utils import render_template_python, str_ellipsis
log = logging.getLogger(__name__)
tracer = logging.getLogger(f"{__name__}.explain")
CACHE_QUERY_EXPIRE = 10
# Helper classes
# ------------------------
class LoadPlugin:
"""Kheops plugins loader
@ -61,8 +60,7 @@ class LoadPlugin:
# Return plugin Classe
return plugin_cls.Plugin
class BackendCandidate:
class BackendCandidate():
"""Backend Candidate
This object represents a backend candidate. It holds the value of the
@ -99,7 +97,6 @@ class Query:
# Query Processor class
# ------------------------
class QueryProcessor:
"""QueryProcessor
@ -137,20 +134,15 @@ class QueryProcessor:
# ------------------------
def query(self, key=None, scope=None, explain=False):
"""Query key with scope"""
"""Query key with scope
# Look into cache
query_hash = dict_hash([self.name, key, scope])
if query_hash in self.cache:
log.debug("Result fetched from cache")
self.cache.touch(query_hash, expire=CACHE_QUERY_EXPIRE)
return self.cache[query_hash]
"""
if explain:
tracer.setLevel(logging.DEBUG)
query = Query(key, scope)
log.info("Creating new query for namespace '%s': %s", self.name, query.__dict__)
log.info("Creating new query: %s", query.__dict__)
# Match the KeyRule in keys (RULE CACHE)
# Get the matching keys
@ -168,7 +160,7 @@ class QueryProcessor:
# Generate explain report
if explain:
self._explain_lookups(parsed_lookups, query)
self._explain_lookups(parsed_lookups)
# Fetch the module
# Retrieve the module instance
@ -193,9 +185,9 @@ class QueryProcessor:
# TODO: Apply output plugins
# result = self._exec_output_plugins(result)
self.cache.set(query_hash, result, expire=CACHE_QUERY_EXPIRE)
return result
# Query parts methods
# ------------------------
@ -218,6 +210,7 @@ class QueryProcessor:
return rule
def _exec_assemble_lookups(self, lookups, query):
assert isinstance(lookups, list)
@ -257,9 +250,7 @@ class QueryProcessor:
plugin_name = plugin_def.get("module", None)
if plugin_name:
plugin = self.plugin_loader.load("scope", plugin_name)(
namespace=self
)
plugin = self.plugin_loader.load("scope", plugin_name)(namespace=self)
ret = plugin.process_items(ret, plugin_def)
new_lookups2.extend(ret)
@ -275,12 +266,11 @@ class QueryProcessor:
lookup["path"] = new_path
new_lookups3.append(lookup)
else:
log.warning(
"Ignore lookup item because of missing scope vars: '%s'", path
)
log.warning("Ignore lookup item because of missing scope vars: '%s'", path)
return new_lookups3
def _exec_backend_plugins(self, lookups, selector="matched"):
selector = "matched"
assert selector in ["last", "first", "all", "matched"]
@ -314,7 +304,7 @@ class QueryProcessor:
# Explain methods
# ------------------------
def _explain_lookups(self, parsed_lookups, query):
def _explain_lookups(self, parsed_lookups):
"""Explain list of lookups"""
table = PrettyTable()
@ -335,7 +325,7 @@ class QueryProcessor:
)
table.field_names = ["Config", "Runtime"]
table.align = "l"
tracer.info("Explain lookups: %s/%s\n%s", self.name, query.key, str(table))
tracer.info("Explain lookups:\n%s", str(table))
def _explain_candidates(self, candidates, query):
"""Explain list of candidates"""
@ -371,4 +361,4 @@ class QueryProcessor:
table.field_names = ["Status", "Runtime", "Key Value"]
table.align = "l"
tracer.info("Explain candidates: %s/%s\n%s", self.name, query.key, str(table))
tracer.info("Explain candidates:\n%s", str(table))

27
kheops/plugin/backend/file.py
View File

@ -2,7 +2,6 @@
import os
import logging
# from pprint import pprint
import anyconfig
@ -10,8 +9,6 @@ from anyconfig.common.errors import BaseError as AnyConfigBaseError
from kheops.plugin.common import BackendPlugin, BackendCandidate
log = logging.getLogger(__name__)
CACHE_FILE_EXPIRE = 5
class Plugin(BackendPlugin):
"""File Backend Plugin
@ -43,12 +40,13 @@ class Plugin(BackendPlugin):
to find all of the specified format. It is better to keep this list as small
as possible.
""",
"type": "object",
"default": extensions,
"additionalProperties": {
"title": "Name of the extension with parser",
"type": "string",
},
"type": "string"
}
},
"path_prefix": {
"title": "Prefix string to append to final path",
@ -56,7 +54,7 @@ class Plugin(BackendPlugin):
String to be added at the end of the resolved path. This is useful to change
the place of the root hierarchy.
""",
"type": "string",
"type": "string"
},
"path_suffix": {
"title": "Suffix string to prepend to final path",
@ -66,11 +64,12 @@ class Plugin(BackendPlugin):
"type": "string",
"examples": [
{ "path_suffix": "/ansible" },
],
},
]
},
}
}
}
_schema_props_new = {
"path": {
@ -119,8 +118,6 @@ class Plugin(BackendPlugin):
def fetch_data(self, config) -> list:
cache = self.ns.cache
path = config.get("path")
if self.path_suffix:
path = f"{path}{self.path_suffix}"
@ -130,21 +127,11 @@ class Plugin(BackendPlugin):
extensions = self.config.get("extensions", self.extensions)
for ext, parser in extensions.items():
new_path = os.path.join(self.top_path, path + ext)
cache_key = "file_content_" + new_path
# Check first if content exists in cache
try:
raw_data = cache[cache_key]
status = "found"
# log.info("Found cached: %s with %s", new_path, raw_data)
break
except KeyError:
if os.path.isfile(new_path):
status = "found"
try:
log.info("Found file: %s", new_path)
raw_data = anyconfig.load(new_path, ac_parser=parser)
cache.set(cache_key, raw_data, expire=CACHE_FILE_EXPIRE)
except AnyConfigBaseError as err:
status = "broken"
raw_data = None

30
kheops/plugin/common.py
View File

@ -6,7 +6,7 @@ from kheops.utils import schema_validate
from pprint import pprint
log = logging.getLogger(__name__)
NoneType = type(None)
# Vocabulary:
# Key Rules
@ -22,16 +22,13 @@ NoneType = type(None)
# Generic Plugin classes
# -------------------------
class KheopsPlugin:
plugin_name = None
plugin_type = None
plugin_kind = None
def __init__(self):
assert isinstance(
self.plugin_name, str
), f"Missing name attribute in plugin: {self.__class__}"
assert isinstance(self.plugin_name, str), f"Missing name attribute in plugin: {self.__class__}"
assert isinstance(self.plugin_kind, str)
config_key = f"{self.plugin_kind}_{self.plugin_name}"
@ -64,7 +61,6 @@ class KheopsItemPlugin(KheopsPlugin):
# Plugin classes
# -------------------------
class BackendPlugin(KheopsItemPlugin):
plugin_kind = "backend"
@ -144,14 +140,11 @@ class ScopePlugin(KheopsListPlugin):
self.ns = namespace
super().__init__()
# Helper classes
# -------------------------
class BackendCandidate:
class BackendCandidate():
"""Represent a backend candidate"""
def __init__(self, path=None, data=None, run=None, status=None):
assert isinstance(run, dict)
self.path = path
@ -163,6 +156,7 @@ class BackendCandidate:
return f"Status: {self.status}, Path: {self.path} => {self.data}"
class ScopeExtLoop:
"""This Scope Extension allow to loop over a lookup"""
@ -185,15 +179,10 @@ class ScopeExtLoop:
}
def loop_over(
self,
lookups,
conf,
module_name,
var_name="item",
callback_context=None,
callback=None,
self, lookups, conf, module_name, var_name="item", callback_context=None, callback=None
):
var_name = conf.get("var", var_name)
var_data_ref = conf.get("data", None)
@ -218,8 +207,7 @@ class ScopeExtLoop:
# Validate generated
if not isinstance(var_data, list):
if not isinstance(var_data, NoneType):
log.warning("Loop data must be a list, got: '%s'", var_data)
log.warning("Loop data must be a list, got: %s", var_data)
continue
# Create new object
@ -235,8 +223,6 @@ class ScopeExtLoop:
"variable": var_name,
}
# Note: This implie a performance penalty to do so, but
# we really need a full copy of the dict. copy.copy or dict() are not enough
new_item = copy.deepcopy(lookup)
new_item["_run"]["scope"][var_name] = var_value
new_item["_run"][module_name].append(ctx)
@ -246,6 +232,8 @@ class ScopeExtLoop:
return ret
# To clean/implement

24
kheops/utils.py
View File

@ -4,10 +4,6 @@ import collections
import logging
from pathlib import Path
import hashlib
import json
from typing import Dict, Any
from jinja2 import Template
from jsonschema import Draft7Validator, validators
from pprint import pprint
@ -78,17 +74,6 @@ class Default(dict):
return ""
# Source: https://www.doc.ic.ac.uk/~nuric/coding/how-to-hash-a-dictionary-in-python.html
def dict_hash(dictionary: Dict[str, Any]) -> str:
"""MD5 hash of a dictionary."""
dhash = hashlib.md5()
# We need to sort arguments so {'a': 1, 'b': 2} is
# the same as {'b': 2, 'a': 1}
encoded = json.dumps(dictionary, sort_keys=True).encode()
dhash.update(encoded)
return dhash.hexdigest()
def render_template_python(text, params, ignore_missing=True):
"""Render template for a given string"""
assert isinstance(params, dict), f"Got: {params}"
@ -105,7 +90,6 @@ def render_template_python(text, params, ignore_missing=True):
# Schema Methods
# =====================
def _extend_with_default(validator_class):
validate_properties = validator_class.VALIDATORS["properties"]
@ -115,16 +99,12 @@ def _extend_with_default(validator_class):
instance.setdefault(property, subschema["default"])
for error in validate_properties(
validator,
properties,
instance,
schema,
validator, properties, instance, schema,
):
yield error
return validators.extend(
validator_class,
{"properties": set_defaults},
validator_class, {"properties" : set_defaults},
)