pro-barbu-foss/kheops
pro-barbu-foss/kheops
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Lint: some parts of the code

Browse Source
This commit is contained in:
mrjk 2022-05-04 19:39:21 -04:00
parent 2707e3ae76
commit ffdbfc442b
5 changed files with 128 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
kheops/app.py
View File

@ -18,26 +18,26 @@ from kheops.utils import schema_validate, dict_hash
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
CACHE_CONFIG_EXPIRE=15 CACHE_CONFIG_EXPIRE = 15
CONF_SCHEMA = { CONF_SCHEMA = {
"$schema": "http://json-schema.org/draft-07/schema#", "$schema": "http://json-schema.org/draft-07/schema#",
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
"default": {}, "default": {},
"required": ["config"], "required": ["config"],
#"$def": { # "$def": {
# "backends_items": {}, # "backends_items": {},
# "backends_config": {}, # "backends_config": {},
# "rules_items": {}, # "rules_items": {},
# "rules_config": {}, # "rules_config": {},
#}, # },
"properties": { "properties": {
"lookups": { "lookups": {
"type": "array", "type": "array",
"default": [], "default": [],
"items": { "items": {
"type": "object", "type": "object",
#"properties": {"$ref": "#/$defs/backends_items"}, # "properties": {"$ref": "#/$defs/backends_items"},
}, },
}, },
"rules": { "rules": {
@ -45,12 +45,11 @@ CONF_SCHEMA = {
"default": [], "default": [],
# "arrayItem": { "$ref": "#/$defs/rules_items" }, # "arrayItem": { "$ref": "#/$defs/rules_items" },
}, },
"config": { "config": {
"type": "object", "type": "object",
"default": {}, "default": {},
"additionalProperties": True, "additionalProperties": True,
#"required": ["app"], # "required": ["app"],
"properties": { "properties": {
"app": { "app": {
"type": "object", "type": "object",
@ -109,6 +108,7 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
Kheops Namespace Class Kheops Namespace Class
""" """
def __init__(self, app, name, config=None): def __init__(self, app, name, config=None):
""" """
Kheops Namespace Instance Kheops Namespace Instance
@ -130,11 +130,11 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
self.cache = app.cache self.cache = app.cache
# Init config (from cache) # Init config (from cache)
config_hash = 'conf_ns_' + dict_hash(config) config_hash = "conf_ns_" + dict_hash(config)
try: try:
config = self.cache[config_hash] config = self.cache[config_hash]
log.debug("Loading namespace '%s' configuration from cache", self.name) log.debug("Loading namespace '%s' configuration from cache", self.name)
except KeyError as err: except KeyError:
config = schema_validate(config, CONF_SCHEMA) config = schema_validate(config, CONF_SCHEMA)
self.cache.set(config_hash, config, expire=CACHE_CONFIG_EXPIRE) self.cache.set(config_hash, config, expire=CACHE_CONFIG_EXPIRE)
super().__init__(config) super().__init__(config)
@ -179,14 +179,13 @@ class Kheops(GenericInstance):
self.cache = cache or Cache("/tmp/kheops_cache/") self.cache = cache or Cache("/tmp/kheops_cache/")
self.raw_config = self.parse_conf(config) self.raw_config = self.parse_conf(config)
#needle = 'conf_app_' + dict_hash(config) # needle = 'conf_app_' + dict_hash(config)
#try: # try:
# self.raw_config = self.cache[needle] # self.raw_config = self.cache[needle]
#except KeyError: # except KeyError:
# self.raw_config = self.parse_conf(config) # self.raw_config = self.parse_conf(config)
# self.cache.set(needle, config, expire=CACHE_CONFIG_EXPIRE) # self.cache.set(needle, config, expire=CACHE_CONFIG_EXPIRE)
def parse_conf(self, config="kheops.yml"): def parse_conf(self, config="kheops.yml"):
""" """
Parse Kheops configuration Parse Kheops configuration
@ -204,17 +203,18 @@ class Kheops(GenericInstance):
try: try:
dict_conf = anyconfig.load(config) dict_conf = anyconfig.load(config)
except Exception as err: except Exception as err:
raise Exception ("Can't load kheops configuration, got: %s", err) raise Exception("Can't load kheops configuration, got: %s", err)
source = f"file:{config}" source = f"file:{config}"
elif isinstance(config, dict): elif isinstance(config, dict):
dict_conf = config dict_conf = config
source = "dict" source = "dict"
self.run["conf_source"] = source
return dict_conf return dict_conf
def lookup( def lookup(
self, self,
keys=None, keys=None,
policy=None,
scope=None, scope=None,
trace=False, trace=False,
explain=False, explain=False,
@ -240,7 +240,9 @@ class Kheops(GenericInstance):
for key_def in keys: for key_def in keys:
key_def = key_def or "" key_def = key_def or ""
assert isinstance(key_def, str), f"Expected string as key, got {type(key_def)}: {key_def}" assert isinstance(
key_def, str
), f"Expected string as key, got {type(key_def)}: {key_def}"
# Identify namespace and key # Identify namespace and key
parts = key_def.split("/") parts = key_def.split("/")
@ -267,15 +269,15 @@ class Kheops(GenericInstance):
# Prepare output # Prepare output
_key = key_name _key = key_name
if namespace_prefix == True: if namespace_prefix is True:
_key = key_def _key = key_def
ret[_key] = result ret[_key] = result
# TODO: This may lead to inconsistant output format :/ # TODO: This may lead to inconsistant output format :/
# Return result # Return result
#if len(keys) > 1: # if len(keys) > 1:
# log.debug("Append '%s' to results", key_name) # log.debug("Append '%s' to results", key_name)
#else: # else:
# log.debug("Return '%s' result", key_name) # log.debug("Return '%s' result", key_name)
# return result # return result
@ -285,56 +287,50 @@ class Kheops(GenericInstance):
return ret return ret
# To clean/implement # To clean/implement
# def DEPRECATED_dump_schema(self): # def DEPRECATED_dump_schema(self):
# """Dump configuration schema""" # """Dump configuration schema"""
# ret1 = BackendsManager.get_schema(KheopsPlugins, mode="parts") # ret1 = BackendsManager.get_schema(KheopsPlugins, mode="parts")
# ret2 = RulesManager.get_schema(KheopsPlugins) # ret2 = RulesManager.get_schema(KheopsPlugins)
# print(json.dumps(ret1, indent=2)) # print(json.dumps(ret1, indent=2))
# return # return
# # ret = self.schema # # ret = self.schema
# # ret["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = ret1 # # ret["patternProperties"][".*"]["properties"]["tree"]["items"]["properties"] = ret1
# # ret["patternProperties"][".*"]["properties"]["tree"]["items"] = ret2 # # ret["patternProperties"][".*"]["properties"]["tree"]["items"] = ret2
# # print(json.dumps(ret, indent=2)) # # print(json.dumps(ret, indent=2))
# def DEPRECATED_gen_docs(self): # def DEPRECATED_gen_docs(self):
# """Generate documentation""" # """Generate documentation"""
# print("WIP") # print("WIP")
# return None # return None
# # src = { # # src = {
# # "app": { # # "app": {
# # "config_schema": None, # # "config_schema": None,
# # "plugin_managers": { # # "plugin_managers": {
# # 'tree': None, # # 'tree': None,
# # 'rules': None, # # 'rules': None,
# # } # # }
# # } # # }
# # # #
# # r1 = BackendsManager.get_schema(KheopsPlugins, mode='parts') # # r1 = BackendsManager.get_schema(KheopsPlugins, mode='parts')
# # print (json.dumps(r1, indent=2)) # # print (json.dumps(r1, indent=2))
# # ret = { # # ret = {
# # # #
# # } # # }
# # part_config = r1.get('config_schema', None) # # part_config = r1.get('config_schema', None)
# # part_item = r1['items']['core_schema'] # # part_item = r1['items']['core_schema']
# # part_item_plugins = r1['items']['plugin'] # # part_item_plugins = r1['items']['plugin']
# # for kind, plugins in part_item_plugins.items(): # # for kind, plugins in part_item_plugins.items():
# # for plugin_name, schema in plugins.items(): # # for plugin_name, schema in plugins.items():
# # part_item_ # # part_item_

24
kheops/controllers.py
View File

@ -24,6 +24,7 @@ CACHE_QUERY_EXPIRE = 10
# Helper classes # Helper classes
# ------------------------ # ------------------------
class LoadPlugin: class LoadPlugin:
"""Kheops plugins loader """Kheops plugins loader
@ -60,7 +61,8 @@ class LoadPlugin:
# Return plugin Classe # Return plugin Classe
return plugin_cls.Plugin return plugin_cls.Plugin
class BackendCandidate():
class BackendCandidate:
"""Backend Candidate """Backend Candidate
This object represents a backend candidate. It holds the value of the This object represents a backend candidate. It holds the value of the
@ -97,6 +99,7 @@ class Query:
# Query Processor class # Query Processor class
# ------------------------ # ------------------------
class QueryProcessor: class QueryProcessor:
"""QueryProcessor """QueryProcessor
@ -134,9 +137,7 @@ class QueryProcessor:
# ------------------------ # ------------------------
def query(self, key=None, scope=None, explain=False): def query(self, key=None, scope=None, explain=False):
"""Query key with scope """Query key with scope"""
"""
# Look into cache # Look into cache
query_hash = dict_hash([self.name, key, scope]) query_hash = dict_hash([self.name, key, scope])
@ -195,7 +196,6 @@ class QueryProcessor:
self.cache.set(query_hash, result, expire=CACHE_QUERY_EXPIRE) self.cache.set(query_hash, result, expire=CACHE_QUERY_EXPIRE)
return result return result
# Query parts methods # Query parts methods
# ------------------------ # ------------------------
@ -218,7 +218,6 @@ class QueryProcessor:
return rule return rule
def _exec_assemble_lookups(self, lookups, query): def _exec_assemble_lookups(self, lookups, query):
assert isinstance(lookups, list) assert isinstance(lookups, list)
@ -227,10 +226,10 @@ class QueryProcessor:
# Init the scope list # Init the scope list
new_lookups1 = [] new_lookups1 = []
for index, lookup_def in enumerate(lookups): for index, lookup_def in enumerate(lookups):
#shortform = False # shortform = False
if isinstance(lookup_def, str): if isinstance(lookup_def, str):
#shortform = True # shortform = True
lookup_def = { lookup_def = {
"path": lookup_def, "path": lookup_def,
} }
@ -258,7 +257,9 @@ class QueryProcessor:
plugin_name = plugin_def.get("module", None) plugin_name = plugin_def.get("module", None)
if plugin_name: if plugin_name:
plugin = self.plugin_loader.load("scope", plugin_name)(namespace=self) plugin = self.plugin_loader.load("scope", plugin_name)(
namespace=self
)
ret = plugin.process_items(ret, plugin_def) ret = plugin.process_items(ret, plugin_def)
new_lookups2.extend(ret) new_lookups2.extend(ret)
@ -274,11 +275,12 @@ class QueryProcessor:
lookup["path"] = new_path lookup["path"] = new_path
new_lookups3.append(lookup) new_lookups3.append(lookup)
else: else:
log.warning("Ignore lookup item because of missing scope vars: '%s'", path) log.warning(
"Ignore lookup item because of missing scope vars: '%s'", path
)
return new_lookups3 return new_lookups3
def _exec_backend_plugins(self, lookups, selector="matched"): def _exec_backend_plugins(self, lookups, selector="matched"):
selector = "matched" selector = "matched"
assert selector in ["last", "first", "all", "matched"] assert selector in ["last", "first", "all", "matched"]

62
kheops/plugin/backend/file.py
View File

@ -2,6 +2,7 @@
import os import os
import logging import logging
# from pprint import pprint # from pprint import pprint
import anyconfig import anyconfig
@ -9,7 +10,8 @@ from anyconfig.common.errors import BaseError as AnyConfigBaseError
from kheops.plugin.common import BackendPlugin, BackendCandidate from kheops.plugin.common import BackendPlugin, BackendCandidate
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
CACHE_FILE_EXPIRE=5 CACHE_FILE_EXPIRE = 5
class Plugin(BackendPlugin): class Plugin(BackendPlugin):
"""File Backend Plugin """File Backend Plugin
@ -20,12 +22,12 @@ class Plugin(BackendPlugin):
plugin_name = "file" plugin_name = "file"
extensions = { extensions = {
".yml": "yaml", ".yml": "yaml",
".yaml": "yaml", ".yaml": "yaml",
#".toml": "toml", # ".toml": "toml",
#".ini": "ini", # ".ini": "ini",
#".json": "json", # ".json": "json",
} }
_schema_config = { _schema_config = {
"backend_file": { "backend_file": {
@ -34,43 +36,41 @@ class Plugin(BackendPlugin):
"type": "object", "type": "object",
"properties": { "properties": {
"extensions": { "extensions": {
"title": "File formats", "title": "File formats",
"description": """ "description": """
This object describe which parser is assigned to which extension. This object describe which parser is assigned to which extension.
Adding more format will have a performance impact because it will try Adding more format will have a performance impact because it will try
to find all of the specified format. It is better to keep this list as small to find all of the specified format. It is better to keep this list as small
as possible. as possible.
""", """,
"type": "object",
"type": "object", "default": extensions,
"default": extensions, "additionalProperties": {
"additionalProperties": { "title": "Name of the extension with parser",
"title": "Name of the extension with parser", "type": "string",
"type": "string"
}
}, },
},
"path_prefix": { "path_prefix": {
"title": "Prefix string to append to final path", "title": "Prefix string to append to final path",
"description": """ "description": """
String to be added at the end of the resolved path. This is useful to change String to be added at the end of the resolved path. This is useful to change
the place of the root hierarchy. the place of the root hierarchy.
""", """,
"type": "string" "type": "string",
}, },
"path_suffix": { "path_suffix": {
"title": "Suffix string to prepend to final path", "title": "Suffix string to prepend to final path",
"description": """ "description": """
String to be added at the end of the resolved path. This is useful to String to be added at the end of the resolved path. This is useful to
provide Hiera or Jerakia support.""", provide Hiera or Jerakia support.""",
"type": "string", "type": "string",
"examples": [ "examples": [
{ "path_suffix": "/ansible" }, {"path_suffix": "/ansible"},
] ],
}, },
} },
}
} }
}
_schema_props_new = { _schema_props_new = {
"path": { "path": {
@ -136,7 +136,7 @@ class Plugin(BackendPlugin):
try: try:
raw_data = cache[cache_key] raw_data = cache[cache_key]
status = "found" status = "found"
#log.info("Found cached: %s with %s", new_path, raw_data) # log.info("Found cached: %s with %s", new_path, raw_data)
break break
except KeyError: except KeyError:
if os.path.isfile(new_path): if os.path.isfile(new_path):

25
kheops/plugin/common.py
View File

@ -22,20 +22,23 @@ NoneType = type(None)
# Generic Plugin classes # Generic Plugin classes
# ------------------------- # -------------------------
class KheopsPlugin: class KheopsPlugin:
plugin_name = None plugin_name = None
plugin_type = None plugin_type = None
plugin_kind = None plugin_kind = None
def __init__(self): def __init__(self):
assert isinstance(self.plugin_name, str), f"Missing name attribute in plugin: {self.__class__}" assert isinstance(
self.plugin_name, str
), f"Missing name attribute in plugin: {self.__class__}"
assert isinstance(self.plugin_kind, str) assert isinstance(self.plugin_kind, str)
config_key = f"{self.plugin_kind}_{self.plugin_name}" config_key = f"{self.plugin_kind}_{self.plugin_name}"
self.config = self.ns.config["config"].get(config_key, {}) self.config = self.ns.config["config"].get(config_key, {})
self.config_key = config_key self.config_key = config_key
#if self.config: # if self.config:
# log.debug("Load plugin configuration in config with key '%s', got: %s", config_key, self.config) # log.debug("Load plugin configuration in config with key '%s', got: %s", config_key, self.config)
self._init() self._init()
@ -61,6 +64,7 @@ class KheopsItemPlugin(KheopsPlugin):
# Plugin classes # Plugin classes
# ------------------------- # -------------------------
class BackendPlugin(KheopsItemPlugin): class BackendPlugin(KheopsItemPlugin):
plugin_kind = "backend" plugin_kind = "backend"
@ -140,11 +144,14 @@ class ScopePlugin(KheopsListPlugin):
self.ns = namespace self.ns = namespace
super().__init__() super().__init__()
# Helper classes # Helper classes
# ------------------------- # -------------------------
class BackendCandidate():
class BackendCandidate:
"""Represent a backend candidate""" """Represent a backend candidate"""
def __init__(self, path=None, data=None, run=None, status=None): def __init__(self, path=None, data=None, run=None, status=None):
assert isinstance(run, dict) assert isinstance(run, dict)
self.path = path self.path = path
@ -156,7 +163,6 @@ class BackendCandidate():
return f"Status: {self.status}, Path: {self.path} => {self.data}" return f"Status: {self.status}, Path: {self.path} => {self.data}"
class ScopeExtLoop: class ScopeExtLoop:
"""This Scope Extension allow to loop over a lookup""" """This Scope Extension allow to loop over a lookup"""
@ -179,10 +185,15 @@ class ScopeExtLoop:
} }
def loop_over( def loop_over(
self, lookups, conf, module_name, var_name="item", callback_context=None, callback=None self,
lookups,
conf,
module_name,
var_name="item",
callback_context=None,
callback=None,
): ):
var_name = conf.get("var", var_name) var_name = conf.get("var", var_name)
var_data_ref = conf.get("data", None) var_data_ref = conf.get("data", None)
@ -235,8 +246,6 @@ class ScopeExtLoop:
return ret return ret
# To clean/implement # To clean/implement

10
kheops/utils.py
View File

@ -77,6 +77,7 @@ class Default(dict):
def __missing__(self, key): def __missing__(self, key):
return "" return ""
# Source: https://www.doc.ic.ac.uk/~nuric/coding/how-to-hash-a-dictionary-in-python.html # Source: https://www.doc.ic.ac.uk/~nuric/coding/how-to-hash-a-dictionary-in-python.html
def dict_hash(dictionary: Dict[str, Any]) -> str: def dict_hash(dictionary: Dict[str, Any]) -> str:
"""MD5 hash of a dictionary.""" """MD5 hash of a dictionary."""
@ -104,6 +105,7 @@ def render_template_python(text, params, ignore_missing=True):
# Schema Methods # Schema Methods
# ===================== # =====================
def _extend_with_default(validator_class): def _extend_with_default(validator_class):
validate_properties = validator_class.VALIDATORS["properties"] validate_properties = validator_class.VALIDATORS["properties"]
@ -113,12 +115,16 @@ def _extend_with_default(validator_class):
instance.setdefault(property, subschema["default"]) instance.setdefault(property, subschema["default"])
for error in validate_properties( for error in validate_properties(
validator, properties, instance, schema, validator,
properties,
instance,
schema,
): ):
yield error yield error
return validators.extend( return validators.extend(
validator_class, {"properties" : set_defaults}, validator_class,
{"properties": set_defaults},
) )