pro-barbu-foss/kheops
pro-barbu-foss/kheops
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix: linting with black

Browse Source
This commit is contained in:
mrjk 2022-02-02 00:25:54 -05:00
parent 327a0e1eb2
commit e7c66d8771
10 changed files with 385 additions and 489 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

348
kheops/app.py
View File

@ -18,151 +18,143 @@ from kheops.utils import schema_validate
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
CONF_SCHEMA = { CONF_SCHEMA = {
"$schema": "http://json-schema.org/draft-07/schema#", "$schema": "http://json-schema.org/draft-07/schema#",
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
"default": {}, "default": {},
"$def": { "$def": {
"backends_items": {}, "backends_items": {},
"backends_config": {}, "backends_config": {},
"rules_items": {}, "rules_items": {},
"rules_config": {}, "rules_config": {},
}, },
#"patternProperties": { # "patternProperties": {
# ".*": { # ".*": {
# "type": "object", # "type": "object",
# "optional": True, # "optional": True,
# "additionalProperties": False, # "additionalProperties": False,
"properties": { "properties": {
"config": { "config": {
"type": "object", "type": "object",
"default": {}, "default": {},
"additionalProperties": True, "additionalProperties": True,
"properties": { "properties": {
"app": { "app": {
"type": "object", "type": "object",
"default": {}, "default": {},
"additionalProperties": False, "additionalProperties": False,
"properties": { "properties": {
"root": { "root": {
"default": None, "default": None,
"oneOf": [ "oneOf": [
{ {
"type": "null", "type": "null",
"description": "Application current working directory is the `kheops.yml` directory", "description": "Application current working directory is the `kheops.yml` directory",
},
{
"type": "string",
"description": "Application working directory. If a relative path is used, it will be depending on `kheops.yml` directory",
},
],
},
"cache": {
"default": "kheops_cache",
"oneOf": [
{
"type": "null",
"description": "Disable cache",
},
{
"type": "string",
"description": "Path of the cache directory",
},
],
},
}, },
}, {
"type": "string",
# OLD "description": "Application working directory. If a relative path is used, it will be depending on `kheops.yml` directory",
"tree": {
# "additionalProperties": False,
"type": "object",
"default": {},
"deprecated": True,
"properties": {
"prefix": {
"default": None,
"oneOf": [
{
"type": "null",
"description": "Disable prefix, all files are lookup up from the app root dir.",
},
{
"type": "string",
"description": "Add a path prefix before all paths. This is quite useful to store your YAML data in a dedicated tree.",
},
],
},
}, },
}, ],
"lookups": { },
# "additionalProperties": False, "cache": {
"type": "object", "default": "kheops_cache",
"default": {}, "oneOf": [
"properties": { {
"prefix": { "type": "null",
"default": None, "description": "Disable cache",
"oneOf": [
{
"type": "null",
"description": "Disable prefix, all files are lookup up from the app root dir.",
},
{
"type": "string",
"description": "Add a path prefix before all paths. This is quite useful to store your YAML data in a dedicated tree.",
},
],
},
}, },
}, {
"rules": { "type": "string",
"type": "object", "description": "Path of the cache directory",
"default": {}, },
}, ],
}, },
}, },
"tree": {
"type": "array",
"default": [],
"items": {
"type": "object",
"properties": {"$ref": "#/$defs/backends_items"},
},
},
"lookups": {
"type": "array",
"default": [],
"items": {
"type": "object",
"properties": {"$ref": "#/$defs/backends_items"},
},
},
"rules": {
"type": "array",
"default": [],
# "arrayItem": { "$ref": "#/$defs/rules_items" },
},
}, },
# }, # OLD
# }, "tree": {
} # "additionalProperties": False,
"type": "object",
"default": {},
"deprecated": True,
"properties": {
"prefix": {
"default": None,
"oneOf": [
{
"type": "null",
"description": "Disable prefix, all files are lookup up from the app root dir.",
},
{
"type": "string",
"description": "Add a path prefix before all paths. This is quite useful to store your YAML data in a dedicated tree.",
},
],
},
},
},
"lookups": {
# "additionalProperties": False,
"type": "object",
"default": {},
"properties": {
"prefix": {
"default": None,
"oneOf": [
{
"type": "null",
"description": "Disable prefix, all files are lookup up from the app root dir.",
},
{
"type": "string",
"description": "Add a path prefix before all paths. This is quite useful to store your YAML data in a dedicated tree.",
},
],
},
},
},
"rules": {
"type": "object",
"default": {},
},
},
},
"tree": {
"type": "array",
"default": [],
"items": {
"type": "object",
"properties": {"$ref": "#/$defs/backends_items"},
},
},
"lookups": {
"type": "array",
"default": [],
"items": {
"type": "object",
"properties": {"$ref": "#/$defs/backends_items"},
},
},
"rules": {
"type": "array",
"default": [],
# "arrayItem": { "$ref": "#/$defs/rules_items" },
},
},
# },
# },
}
class GenericInstance:
class GenericInstance():
name = None name = None
run = {} run = {}
class KheopsNamespace(GenericInstance, QueryProcessor): class KheopsNamespace(GenericInstance, QueryProcessor):
def __init__(self, app, name, config=None): def __init__(self, app, name, config=None):
self.name = name self.name = name
@ -170,14 +162,10 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
self.app = app self.app = app
self.run = dict(app.run) self.run = dict(app.run)
# Validate configuration # Validate configuration
self.config = schema_validate(self.config, CONF_SCHEMA) self.config = schema_validate(self.config, CONF_SCHEMA)
self.run["path_ns"] = str(Path(app.run['config_src']).parent.resolve()) self.run["path_ns"] = str(Path(app.run["config_src"]).parent.resolve())
# def load_namespace(self, namespace="default"): # def load_namespace(self, namespace="default"):
@ -243,19 +231,15 @@ class KheopsNamespace(GenericInstance, QueryProcessor):
# return config # return config
# def query(self, key=None, scope=None):
#def query(self, key=None, scope=None): # processor = QueryProcessor(app=self.app)
# processor = QueryProcessor(app=self.app) # result = processor.exec(key, scope)
# result = processor.exec(key, scope)
# #
# return result # return result
class Kheops(GenericInstance): class Kheops(GenericInstance):
"""Main Kheops Application Instance """Main Kheops Application Instance"""
"""
def __init__(self, config="kheops.yml", namespace="default"): def __init__(self, config="kheops.yml", namespace="default"):
""" """
@ -277,20 +261,17 @@ class Kheops(GenericInstance):
self.run["config_src"] = config self.run["config_src"] = config
if isinstance(config, str): if isinstance(config, str):
self.run["config_type"] = 'file' self.run["config_type"] = "file"
self.run["path_config"] = str(Path(config).parent.resolve()) self.run["path_config"] = str(Path(config).parent.resolve())
elif isinstance(config, dict): elif isinstance(config, dict):
self.run["config_type"] = 'dict' self.run["config_type"] = "dict"
self.run["path_config"] = str(path_cwd) self.run["path_config"] = str(path_cwd)
else: else:
raise Exception("Need a valid config") raise Exception("Need a valid config")
self.ns_name = namespace self.ns_name = namespace
self.raw_config = self.parse_conf(config) self.raw_config = self.parse_conf(config)
def parse_conf(self, config="kheops.yml"): def parse_conf(self, config="kheops.yml"):
""" """
Parse Kheops configuration Parse Kheops configuration
@ -307,31 +288,32 @@ class Kheops(GenericInstance):
# Load config # Load config
if isinstance(config, str): if isinstance(config, str):
dict_conf = anyconfig.load(config) dict_conf = anyconfig.load(config)
source = f'file:{config}' source = f"file:{config}"
elif isinstance(config, dict): elif isinstance(config, dict):
dict_conf = config dict_conf = config
source = 'dict' source = "dict"
return dict_conf return dict_conf
def lookup2(
self,
keys=None,
policy=None,
def lookup2(self, keys=None, policy=None, scope=None, scope=None,
trace=False, explain=False, validate_schema=False, trace=False,
namespace='default' , explain=False,
): validate_schema=False,
namespace="default",
):
"""Lookup a key in hierarchy""" """Lookup a key in hierarchy"""
ret = {} ret = {}
# Loop over keys # Loop over keys
for key_def in keys: for key_def in keys:
key_def = key_def or '' key_def = key_def or ""
# Identify namespace and key # Identify namespace and key
parts = key_def.split(':') parts = key_def.split(":")
ns_name = self.ns_name ns_name = self.ns_name
if len(parts) > 1: if len(parts) > 1:
ns_name = parts[0] ns_name = parts[0]
@ -357,29 +339,15 @@ class Kheops(GenericInstance):
return ret return ret
def lookup(
self,
keys=None,
policy=None,
scope=None,
trace=False,
explain=False,
validate_schema=False,
):
def lookup(self, keys=None, policy=None, scope=None, trace=False, explain=False, validate_schema=False):
"""Lookup a key in hierarchy""" """Lookup a key in hierarchy"""
log.debug("Lookup key %s with scope: %s", keys, scope) log.debug("Lookup key %s with scope: %s", keys, scope)
assert isinstance(keys, list), f"Got {keys}" assert isinstance(keys, list), f"Got {keys}"
@ -387,7 +355,14 @@ class Kheops(GenericInstance):
query = Query(app=self) query = Query(app=self)
ret = {} ret = {}
for key in keys: for key in keys:
ret[key] = query.exec(key=key, scope=scope, policy=policy, trace=trace, explain=explain, validate_schema=validate_schema) ret[key] = query.exec(
key=key,
scope=scope,
policy=policy,
trace=trace,
explain=explain,
validate_schema=validate_schema,
)
return ret return ret
def dump_schema(self): def dump_schema(self):
@ -405,8 +380,7 @@ class Kheops(GenericInstance):
# print(json.dumps(ret, indent=2)) # print(json.dumps(ret, indent=2))
def gen_docs(self): def gen_docs(self):
""" Generate documentation""" """Generate documentation"""
print("WIP") print("WIP")
return None return None

17
kheops/cli.py
View File

@ -122,11 +122,13 @@ class CmdApp:
add_p = subparsers.add_parser("schema") add_p = subparsers.add_parser("schema")
add_p = subparsers.add_parser("gen_doc") add_p = subparsers.add_parser("gen_doc")
# Manage command: lookup2 # Manage command: lookup2
add_p = subparsers.add_parser("lookup") add_p = subparsers.add_parser("lookup")
add_p.add_argument( add_p.add_argument(
"-n", "--namespace", help="Namespace name (KHEOPS_NAMESPACE)", default=os.environ.get("KHEOPS_NAMESPACE", "default") "-n",
"--namespace",
help="Namespace name (KHEOPS_NAMESPACE)",
default=os.environ.get("KHEOPS_NAMESPACE", "default"),
) )
add_p.add_argument( add_p.add_argument(
"-f", "--file", help="File with params as dict. Can be stdin - ." "-f", "--file", help="File with params as dict. Can be stdin - ."
@ -178,7 +180,6 @@ class CmdApp:
keys = self.args.keys or [None] keys = self.args.keys or [None]
new_params = {} new_params = {}
if self.args.file: if self.args.file:
new_params = anyconfig.load(self.args.file, ac_parser="yaml") new_params = anyconfig.load(self.args.file, ac_parser="yaml")
@ -192,25 +193,17 @@ class CmdApp:
self.log.info("CLI: %s with env: %s", keys, new_params) self.log.info("CLI: %s with env: %s", keys, new_params)
app = Kheops.Kheops(config=self.args.config, namespace=self.args.namespace) app = Kheops.Kheops(config=self.args.config, namespace=self.args.namespace)
ret = app.lookup2( ret = app.lookup2(
namespace=self.args.namespace, namespace=self.args.namespace,
keys=keys, keys=keys,
scope=new_params, scope=new_params,
trace=self.args.trace, trace=self.args.trace,
explain=self.args.explain, explain=self.args.explain,
validate_schema=True, validate_schema=True,
) )
print(anyconfig.dumps(ret, ac_parser=self.args.format)) print(anyconfig.dumps(ret, ac_parser=self.args.format))
def cli_lookup_OLD(self): def cli_lookup_OLD(self):
"""Display how to use logging""" """Display how to use logging"""
@ -232,7 +225,7 @@ class CmdApp:
self.log.info("CLI: %s with env: %s", keys, new_params) self.log.info("CLI: %s with env: %s", keys, new_params)
app = Kheops.App(config=self.args.config, namespace=self.args.namespace) app = Kheops.App(config=self.args.config, namespace=self.args.namespace)
#for key in keys: # for key in keys:
ret = app.lookup( ret = app.lookup(
keys=keys, keys=keys,
scope=new_params, scope=new_params,

201
kheops/controllers.py
View File

@ -1,7 +1,7 @@
import json import json
import logging import logging
#from pprint import pprint
# from pprint import pprint
from pathlib import Path from pathlib import Path
from prettytable import PrettyTable from prettytable import PrettyTable
@ -10,7 +10,7 @@ import kheops.plugin as KheopsPlugins
from kheops.utils import render_template, render_template_python, str_ellipsis from kheops.utils import render_template, render_template_python, str_ellipsis
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
tracer = logging.getLogger(f'{__name__}.explain') tracer = logging.getLogger(f"{__name__}.explain")
class LoadPlugin: class LoadPlugin:
@ -46,8 +46,7 @@ class LoadPlugin:
plugin_loader = LoadPlugin(KheopsPlugins) plugin_loader = LoadPlugin(KheopsPlugins)
class Query:
class Query():
key = None key = None
scope = None scope = None
@ -62,41 +61,39 @@ class Query():
# class QueryController(): # class QueryController():
#def exec(self, key=None, scope=None): # def exec(self, key=None, scope=None):
# query = Query(key, scope) # query = Query(key, scope)
# result = self.processor.exec(query) # result = self.processor.exec(query)
# return result # return result
class QueryProcessor(): class QueryProcessor:
"""QueryProcessor class provides all the methods to be able to make queries""" """QueryProcessor class provides all the methods to be able to make queries"""
default_match_rule = { default_match_rule = {
"key": None, "key": None,
"continue": False, "continue": False,
"strategy": "merge_deep", "strategy": "merge_deep",
} }
default_lookup_item = { default_lookup_item = {
"path": None, "path": None,
"backend": "file", "backend": "file",
"continue": True, "continue": True,
} }
#def __init__(self, app): # def __init__(self, app):
# self.app = app # self.app = app
#self.config = app.conf2['config'] or {} # self.config = app.conf2['config'] or {}
#self.lookups = app.conf2['lookups'] or [] # self.lookups = app.conf2['lookups'] or []
#self.rules = app.conf2['rules'] or [] # self.rules = app.conf2['rules'] or []
def CHILDREN_INIT(self, config): def CHILDREN_INIT(self, config):
self.config = config self.config = config
pass pass
# def exec(self, key=None, scope=None):
#def exec(self, key=None, scope=None):
def query(self, key=None, scope=None, explain=False): def query(self, key=None, scope=None, explain=False):
if explain: if explain:
@ -105,19 +102,18 @@ class QueryProcessor():
query = Query(key, scope) query = Query(key, scope)
# Match the KeyRule in keys (RULE CACHE) # Match the KeyRule in keys (RULE CACHE)
# Get the matching keys # Get the matching keys
# Assemble if more than one and merge when continue. # Assemble if more than one and merge when continue.
# Got the Matched rule (RULE CACHE) # Got the Matched rule (RULE CACHE)
# We'll need strategy, and it's selector field: matched/first/last/all # We'll need strategy, and it's selector field: matched/first/last/all
#key_rule = self._get_key_rule(key) or {} # key_rule = self._get_key_rule(key) or {}
#key_strategy = key_rule.get('strategy', None) # key_strategy = key_rule.get('strategy', None)
key_rule = self._exec_get_rule(query) key_rule = self._exec_get_rule(query)
log.info("Matched rule for key '%s': %s", query.key, key_rule) log.info("Matched rule for key '%s': %s", query.key, key_rule)
# Build the lookups [] => [] # Build the lookups [] => []
# Fetch static config from app (for include and NS:includes ...) # Fetch static config from app (for include and NS:includes ...)
# Loop over lookups and process each lookup with ScopePlugins # Loop over lookups and process each lookup with ScopePlugins
lookups = self.config.get("lookups", {}).copy() lookups = self.config.get("lookups", {}).copy()
parsed_lookups = self._exec_assemble_lookups(lookups, query) parsed_lookups = self._exec_assemble_lookups(lookups, query)
@ -126,51 +122,55 @@ class QueryProcessor():
self._explain_lookups(parsed_lookups) self._explain_lookups(parsed_lookups)
# FEtch the module # FEtch the module
# Retrieve the module instance # Retrieve the module instance
# Get it's match policy # Get it's match policy
# TODO # TODO
plugin_name = key_rule.get('strategy', None) plugin_name = key_rule.get("strategy", None)
strategy_plugin = plugin_loader.load('strategy', plugin_name)(self) strategy_plugin = plugin_loader.load("strategy", plugin_name)(self)
# Get the data (strategy.selector) # Get the data (strategy.selector)
# For each entry, ask the backend to return the data: file, http, consul ... # For each entry, ask the backend to return the data: file, http, consul ...
# Return zero, one or more results depending the strategy.selector # Return zero, one or more results depending the strategy.selector
#result = get_backends_results(strategy, lookups) # result = get_backends_results(strategy, lookups)
candidates = self._exec_backend_plugins(parsed_lookups, selector=strategy_plugin.selector) candidates = self._exec_backend_plugins(
parsed_lookups, selector=strategy_plugin.selector
)
# Generate explain report # Generate explain report
if explain: if explain:
self._explain_candidates(candidates, query) self._explain_candidates(candidates, query)
# Apply the merge strategy, recall strategy # Apply the merge strategy, recall strategy
result = strategy_plugin.merge_results(candidates, key_rule, query) result = strategy_plugin.merge_results(candidates, key_rule, query)
# TODO: Apply output plugins # TODO: Apply output plugins
# result = self._exec_output_plugins(result) # result = self._exec_output_plugins(result)
return result return result
def _explain_lookups(self, parsed_lookups): def _explain_lookups(self, parsed_lookups):
"""Explain list of lookups""" """Explain list of lookups"""
table = PrettyTable() table = PrettyTable()
for item in parsed_lookups: for item in parsed_lookups:
col1 = json.dumps({ k:v for k, v in item.items() if k not in ['_run'] }, default=lambda o: "<not serializable>", indent=2) col1 = json.dumps(
col2 = json.dumps(item['_run'], default=lambda o: "<not serializable>", indent=2) {k: v for k, v in item.items() if k not in ["_run"]},
table.add_row([ default=lambda o: "<not serializable>",
"\nConfig:"+ str_ellipsis(col1, 60), indent=2,
"\nRuntime:"+ str_ellipsis(col2, 60), )
]) col2 = json.dumps(
item["_run"], default=lambda o: "<not serializable>", indent=2
)
table.add_row(
[
"\nConfig:" + str_ellipsis(col1, 60),
"\nRuntime:" + str_ellipsis(col2, 60),
]
)
table.field_names = ["Config", "Runtime"] table.field_names = ["Config", "Runtime"]
table.align = "l" table.align = "l"
tracer.info("Explain lookups:\n" + str(table)) tracer.info("Explain lookups:\n" + str(table))
def _explain_candidates(self, candidates, query): def _explain_candidates(self, candidates, query):
"""Explain list of candidates""" """Explain list of candidates"""
@ -179,59 +179,66 @@ class QueryProcessor():
table = PrettyTable() table = PrettyTable()
for item_obj in candidates: for item_obj in candidates:
item = item_obj.__dict__ item = item_obj.__dict__
item["rel_path"] = str(Path(item['path']).relative_to(Path.cwd())) item["rel_path"] = str(Path(item["path"]).relative_to(Path.cwd()))
col1 = json.dumps({ k:v for k, v in item.items() if k not in ['run', 'data'] }, default=lambda o: "<not serializable>", indent=2) col1 = json.dumps(
col2 = json.dumps(item['run']['_run'], default=lambda o: "<not serializable>", indent=2) {k: v for k, v in item.items() if k not in ["run", "data"]},
col3 = item_obj.data.get(query.key, "NOT FOUND") if query.key is not None else item_obj.data default=lambda o: "<not serializable>",
indent=2,
)
col2 = json.dumps(
item["run"]["_run"], default=lambda o: "<not serializable>", indent=2
)
col3 = (
item_obj.data.get(query.key, "NOT FOUND")
if query.key is not None
else item_obj.data
)
col3 = json.dumps(col3, default=lambda o: "<not serializable>", indent=2) col3 = json.dumps(col3, default=lambda o: "<not serializable>", indent=2)
table.add_row([ table.add_row(
"\nStatus:"+ str_ellipsis(col1, 80), [
"\nRuntime:"+ str_ellipsis(col2, 60), "\nStatus:" + str_ellipsis(col1, 80),
"\nData:"+ str_ellipsis(col3, 60), "\nRuntime:" + str_ellipsis(col2, 60),
]) "\nData:" + str_ellipsis(col3, 60),
]
)
table.field_names = ["Status", "Runtime", "Data"] table.field_names = ["Status", "Runtime", "Data"]
table.align = "l" table.align = "l"
tracer.info("Explain candidates:\n" + str(table)) tracer.info("Explain candidates:\n" + str(table))
def _exec_backend_plugins(self, lookups, selector="matched"): def _exec_backend_plugins(self, lookups, selector="matched"):
selector = 'matched' selector = "matched"
assert (selector in ['last', 'first', 'all', 'matched']) assert selector in ["last", "first", "all", "matched"]
assert isinstance(lookups, list) assert isinstance(lookups, list)
#lookups = self.config.get("lookups", {}).copy() # lookups = self.config.get("lookups", {}).copy()
plugins = {} plugins = {}
ret = [] ret = []
for index, lookup_def in enumerate(lookups): for index, lookup_def in enumerate(lookups):
# Update object # Update object
lookup_def['_run']['backend_index'] = index lookup_def["_run"]["backend_index"] = index
# Load plugin # Load plugin
plugin_name = lookup_def["backend"] plugin_name = lookup_def["backend"]
if plugin_name in plugins: if plugin_name in plugins:
plugin = plugins[plugin_name] plugin = plugins[plugin_name]
else: else:
plugin = plugin_loader.load('backend', plugin_name)(namespace=self) plugin = plugin_loader.load("backend", plugin_name)(namespace=self)
# Get candidates # Get candidates
candidates = plugin.fetch_data(lookup_def) candidates = plugin.fetch_data(lookup_def)
# Apply selector # Apply selector
for candidate in candidates: for candidate in candidates:
if candidate.status == 'found' or selector == 'all': if candidate.status == "found" or selector == "all":
ret.append(candidate) ret.append(candidate)
return ret return ret
def _exec_assemble_lookups(self, lookups, query): def _exec_assemble_lookups(self, lookups, query):
assert isinstance(lookups, list) assert isinstance(lookups, list)
assert len(lookups) > 0 assert len(lookups) > 0
@ -243,20 +250,20 @@ class QueryProcessor():
if isinstance(lookup_def, str): if isinstance(lookup_def, str):
shortform = True shortform = True
lookup_def = { lookup_def = {
'path': lookup_def, "path": lookup_def,
} }
assert isinstance(lookup_def, dict) assert isinstance(lookup_def, dict)
new_lookup = dict(self.default_lookup_item) new_lookup = dict(self.default_lookup_item)
new_lookup.update(lookup_def) new_lookup.update(lookup_def)
new_lookup['_run'] = { new_lookup["_run"] = {
'scope': query.scope, "scope": query.scope,
'key': query.key, "key": query.key,
'conf': { "conf": {
'index': index, "index": index,
} }
# 'shortform': shortform, # 'shortform': shortform,
} }
new_lookups1.append(new_lookup) new_lookups1.append(new_lookup)
# Apply lookups modules # Apply lookups modules
@ -269,36 +276,34 @@ class QueryProcessor():
plugin_name = plugin_def.get("module", None) plugin_name = plugin_def.get("module", None)
if plugin_name: if plugin_name:
plugin = plugin_loader.load('scope', plugin_name)(namespace=self) plugin = plugin_loader.load("scope", plugin_name)(namespace=self)
ret = plugin.process_items(ret, plugin_def) ret = plugin.process_items(ret, plugin_def)
new_lookups2.extend(ret) new_lookups2.extend(ret)
# Parse the `path` value with scope variables # Parse the `path` value with scope variables
new_lookups3 = [] new_lookups3 = []
for lookup in new_lookups2: for lookup in new_lookups2:
path = lookup['path'] path = lookup["path"]
scope = lookup['_run']['scope'] scope = lookup["_run"]["scope"]
new_path = render_template_python(path, scope, ignore_missing=False) new_path = render_template_python(path, scope, ignore_missing=False)
if new_path: if new_path:
lookup['_run']['raw_path'] = path lookup["_run"]["raw_path"] = path
lookup['path'] = new_path lookup["path"] = new_path
new_lookups3.append(lookup) new_lookups3.append(lookup)
else: else:
log.info("Ignore because of missing scope vars: '%s'", path) log.info("Ignore because of missing scope vars: '%s'", path)
return new_lookups3 return new_lookups3
def _exec_get_rule(self, query, mode="match"):
def _exec_get_rule(self, query, mode='match'):
key = query.key key = query.key
rules = self.config['rules'] or {} rules = self.config["rules"] or {}
if mode == "match": if mode == "match":
rule = dict(self.default_match_rule) rule = dict(self.default_match_rule)
rules = [i for i in rules if i.get('key', None) == key ] rules = [i for i in rules if i.get("key", None) == key]
if len(rules) > 0: if len(rules) > 0:
match = rules[0] match = rules[0]
rule.update(match) rule.update(match)
@ -306,12 +311,6 @@ class QueryProcessor():
log.debug("Applying default rule for key '%s'", key) log.debug("Applying default rule for key '%s'", key)
rule = self.default_match_rule rule = self.default_match_rule
else: else:
raise Exception (f"Mode '{mode}' is not implemented") raise Exception(f"Mode '{mode}' is not implemented")
return rule return rule

33
kheops/plugin/backend/file.py
View File

@ -10,6 +10,7 @@ from kheops.utils import render_template, glob_files, render_template_python
from kheops.plugin.common import BackendPlugin, BackendCandidate from kheops.plugin.common import BackendPlugin, BackendCandidate
from pprint import pprint from pprint import pprint
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -27,7 +28,7 @@ log = logging.getLogger(__name__)
# return super()._report_data(data) # return super()._report_data(data)
#class Plugin(PluginEngineClass, PluginFileGlob): # class Plugin(PluginEngineClass, PluginFileGlob):
class Plugin(BackendPlugin): class Plugin(BackendPlugin):
"""Generic Plugin Class""" """Generic Plugin Class"""
@ -65,45 +66,40 @@ class Plugin(BackendPlugin):
}, },
} }
extensions = { extensions = {".yml": "yaml", ".yaml": "yaml"}
'.yml': 'yaml',
'.yaml': 'yaml'
}
def _init(self): def _init(self):
# Guess top path # Guess top path
top_path = self.ns.run['path_config'] top_path = self.ns.run["path_config"]
path_prefix = self.ns.config['config'].get('file_path_prefix', None) path_prefix = self.ns.config["config"].get("file_path_prefix", None)
if path_prefix: if path_prefix:
top_path = os.path.join(top_path, path_prefix) top_path = os.path.join(top_path, path_prefix)
self.top_path = top_path self.top_path = top_path
# Fetch module config # Fetch module config
path_suffix = self.ns.config['config'].get('file_path_suffix', "auto") path_suffix = self.ns.config["config"].get("file_path_suffix", "auto")
if path_suffix == 'auto': if path_suffix == "auto":
path_suffix = f"/{self.ns.name}" path_suffix = f"/{self.ns.name}"
self.path_suffix = path_suffix self.path_suffix = path_suffix
def fetch_data(self, config) -> list: def fetch_data(self, config) -> list:
path = config.get('path') path = config.get("path")
if self.path_suffix: if self.path_suffix:
path = f"{path}{self.path_suffix}" path = f"{path}{self.path_suffix}"
raw_data = None raw_data = None
status = 'not_found' status = "not_found"
for ext, parser in self.extensions.items(): for ext, parser in self.extensions.items():
new_path = os.path.join(self.top_path, path + ext ) new_path = os.path.join(self.top_path, path + ext)
if os.path.isfile(new_path): if os.path.isfile(new_path):
status = 'found' status = "found"
try: try:
raw_data = anyconfig.load(new_path, ac_parser=parser) raw_data = anyconfig.load(new_path, ac_parser=parser)
except Exception: except Exception:
status = 'broken' status = "broken"
raw_data = None raw_data = None
break break
@ -111,8 +107,7 @@ class Plugin(BackendPlugin):
path=new_path, path=new_path,
status=status, status=status,
run=config, run=config,
data= raw_data, data=raw_data,
) )
return [ret] return [ret]

172
kheops/plugin/common.py
View File

@ -8,16 +8,15 @@ from pprint import pprint
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
# Vocabulary: # Vocabulary:
# Key Rules # Key Rules
# ConfPlugin[1] # ConfPlugin[1]
# StrategyPlugin[1] # StrategyPlugin[1]
# OutPlugin[N] # OutPlugin[N]
# Lookup Hierarchy # Lookup Hierarchy
# ConfPlugin[1] # ConfPlugin[1]
# ScopePlugin[N] # ScopePlugin[N]
# BackendPlugin[1] # BackendPlugin[1]
# Generic classes # Generic classes
@ -26,7 +25,6 @@ class KheopsPlugin:
plugin_type = None plugin_type = None
plugin_kind = None plugin_kind = None
def __init__(self): def __init__(self):
self._init() self._init()
@ -36,20 +34,21 @@ class KheopsPlugin:
class KheopsListPlugin(KheopsPlugin): class KheopsListPlugin(KheopsPlugin):
plugin_type = 'list' plugin_type = "list"
def process_list(self, item_list) -> list: def process_list(self, item_list) -> list:
pass pass
class KheopsItemPlugin(KheopsPlugin): class KheopsItemPlugin(KheopsPlugin):
plugin_type = 'item' plugin_type = "item"
def process_item(self, item) -> list: def process_item(self, item) -> list:
pass pass
# Other classes # Other classes
class BackendCandidate(): class BackendCandidate:
def __init__(self, path=None, data=None, run=None, status=None): def __init__(self, path=None, data=None, run=None, status=None):
assert isinstance(run, dict) assert isinstance(run, dict)
self.path = path self.path = path
@ -60,24 +59,27 @@ class BackendCandidate():
def __repr__(self): def __repr__(self):
return f"Status: {self.status}, Path: {self.path} => {self.data}" return f"Status: {self.status}, Path: {self.path} => {self.data}"
# Specific classes # Specific classes
class ConfPlugin(KheopsListPlugin): class ConfPlugin(KheopsListPlugin):
plugin_kind = "conf" plugin_kind = "conf"
schema_prop = { schema_prop = {
"include": {}, # Direct config, DICT "include": {}, # Direct config, DICT
} }
def process_list(self, item_list) -> list: def process_list(self, item_list) -> list:
pass pass
class ScopePlugin(KheopsListPlugin): class ScopePlugin(KheopsListPlugin):
plugin_kind = "scope" plugin_kind = "scope"
schema_prop = { schema_prop = {
"_scope": [], # List of scope modification to apply "_scope": [], # List of scope modification to apply
"init": {}, "init": {},
"loop_N": {}, "loop_N": {},
"hier_N": {}, "hier_N": {},
} }
def process_item(self, item_list) -> list: def process_item(self, item_list) -> list:
@ -88,9 +90,8 @@ class ScopePlugin(KheopsListPlugin):
super().__init__() super().__init__()
class ScopeExtLoop:
class ScopeExtLoop(): """This Scope Extension allow to loop over a lookup"""
'''This Scope Extension allow to loop over a lookup'''
schema_props = { schema_props = {
"properties": { "properties": {
@ -110,11 +111,12 @@ class ScopeExtLoop():
}, },
} }
def loop_over(
self, lookups, conf, var_name="item", callback_context=None, callback=None
):
def loop_over(self, lookups, conf, var_name='item', callback_context=None, callback=None): var_name = conf.get("var", var_name)
var_data_ref = conf.get("data", None)
var_name = conf.get('var', var_name)
var_data_ref = conf.get('data', None)
if not var_data_ref: if not var_data_ref:
log.debug("No data to loop over for: %s", var_data_ref) log.debug("No data to loop over for: %s", var_data_ref)
@ -126,9 +128,9 @@ class ScopeExtLoop():
var_data = var_data_ref var_data = var_data_ref
if isinstance(var_data_ref, str): if isinstance(var_data_ref, str):
try: try:
var_data = lookup['_run']['scope'][var_data] var_data = lookup["_run"]["scope"][var_data]
except KeyError: except KeyError:
log.debug ("Ignoring missing '%s' from scope", var_data) log.debug("Ignoring missing '%s' from scope", var_data)
pass pass
# Run callback # Run callback
@ -143,59 +145,56 @@ class ScopeExtLoop():
# Create new object # Create new object
for index, var_value in enumerate(var_data): for index, var_value in enumerate(var_data):
if not 'hier' in lookup['_run']: if not "hier" in lookup["_run"]:
lookup['_run']['hier'] = [] lookup["_run"]["hier"] = []
ctx = { ctx = {
'data_ref': var_data_ref, "data_ref": var_data_ref,
'index': index, "index": index,
'value': var_value, "value": var_value,
'variable': var_name, "variable": var_name,
} }
new_item = copy.deepcopy(lookup) new_item = copy.deepcopy(lookup)
new_item['_run']['scope'][var_name] = var_value new_item["_run"]["scope"][var_name] = var_value
new_item['_run']['hier'].append(ctx) new_item["_run"]["hier"].append(ctx)
ret.append(new_item) ret.append(new_item)
return ret return ret
class BackendPlugin(KheopsItemPlugin): class BackendPlugin(KheopsItemPlugin):
plugin_kind = "backend" plugin_kind = "backend"
schema_prop = { schema_prop = {
"backend": {}, # GENERIC, String "backend": {}, # GENERIC, String
"file": {}, "file": {},
"glob": {}, "glob": {},
"http": {}, "http": {},
"consul": {}, "consul": {},
"vault": {}, "vault": {},
} }
def fetch_data(self, lookups) -> list:
raise Exception('Not implemented')
def fetch_data(self, lookups) -> list:
raise Exception("Not implemented")
def __init__(self, namespace): def __init__(self, namespace):
self.ns = namespace self.ns = namespace
super().__init__() super().__init__()
class StrategyPlugin(KheopsItemPlugin): class StrategyPlugin(KheopsItemPlugin):
plugin_kind = "strategy" plugin_kind = "strategy"
schema_prop = { schema_prop = {
"_strategy": {}, # GENERIC, String "_strategy": {}, # GENERIC, String
"merge": {}, "merge": {},
"first": {}, "first": {},
"last": {}, "last": {},
"smart": {}, "smart": {},
"schema": {}, "schema": {},
} }
def merge_results(self, candidates, rule) -> list: def merge_results(self, candidates, rule) -> list:
pass pass
@ -207,64 +206,15 @@ class StrategyPlugin(KheopsItemPlugin):
class OutPlugin(KheopsItemPlugin): class OutPlugin(KheopsItemPlugin):
plugin_kind = "out" plugin_kind = "out"
schema_prop = { schema_prop = {
"_out": {}, # GENERIC, List of dict "_out": {}, # GENERIC, List of dict
"toml": {}, "toml": {},
"validate": {}, "validate": {},
} }
def process_item(self, item) -> list: def process_item(self, item) -> list:
pass pass
# # Candidate Classes # # Candidate Classes
# # ============================= # # =============================
# # class Candidate: # # class Candidate:

26
kheops/plugin/scope/hier.py
View File

@ -10,6 +10,7 @@ log = logging.getLogger(__name__)
from pprint import pprint from pprint import pprint
class Plugin(ScopePlugin, ScopeExtLoop): class Plugin(ScopePlugin, ScopeExtLoop):
"""Hierarchy plugin""" """Hierarchy plugin"""
@ -25,7 +26,6 @@ class Plugin(ScopePlugin, ScopeExtLoop):
{ {
"type": "string", "type": "string",
}, },
{ {
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
@ -59,31 +59,29 @@ class Plugin(ScopePlugin, ScopeExtLoop):
} }
} }
def _process_item(self, data, ctx): def _process_item(self, data, ctx):
return path_assemble_hier(data, return path_assemble_hier(
sep=ctx['var_split'], data,
reverse=ctx['var_reversed'], sep=ctx["var_split"],
start_index=ctx['var_start'], reverse=ctx["var_reversed"],
) start_index=ctx["var_start"],
)
def process_items(self, lookups, conf): def process_items(self, lookups, conf):
ctx = { ctx = {
"var_split": conf.get('split', '/'), "var_split": conf.get("split", "/"),
"var_reversed": conf.get('reversed', False), "var_reversed": conf.get("reversed", False),
"var_start": conf.get('start', 0), "var_start": conf.get("start", 0),
} }
lookups = self.loop_over( lookups = self.loop_over(
lookups, lookups,
conf=conf, conf=conf,
var_name='item_hier', var_name="item_hier",
callback=self._process_item, callback=self._process_item,
callback_context=ctx, callback_context=ctx,
) )
return lookups return lookups

11
kheops/plugin/scope/loop.py
View File

@ -9,7 +9,7 @@ from kheops.utils import path_assemble_hier
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class Plugin(ScopePlugin,ScopeExtLoop): class Plugin(ScopePlugin, ScopeExtLoop):
"""Hierarchy plugin""" """Hierarchy plugin"""
_plugin_name = "hier" _plugin_name = "hier"
@ -56,16 +56,15 @@ class Plugin(ScopePlugin,ScopeExtLoop):
} }
} }
def process_items(self, lookups, conf): def process_items(self, lookups, conf):
item_name = conf.get('var', "item_loop") item_name = conf.get("var", "item_loop")
item_data = conf.get('data', None) item_data = conf.get("data", None)
lookups = self.loop_over( lookups = self.loop_over(
lookups, lookups,
conf=conf, conf=conf,
var_name='item_loop', var_name="item_loop",
) )
return lookups return lookups

10
kheops/plugin/strategy/last.py
View File

@ -5,7 +5,8 @@ from kheops.plugin.common import StrategyPlugin
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
#class Plugin(PluginStrategyClass): # class Plugin(PluginStrategyClass):
class Plugin(StrategyPlugin): class Plugin(StrategyPlugin):
"""Last strategy plugin""" """Last strategy plugin"""
@ -13,7 +14,7 @@ class Plugin(StrategyPlugin):
_plugin_name = "last" _plugin_name = "last"
_schema_props_new = None _schema_props_new = None
selector = 'last' selector = "last"
def merge_results(self, candidates: list, rule: dict, query) -> (list, dict): def merge_results(self, candidates: list, rule: dict, query) -> (list, dict):
"""Return results""" """Return results"""
@ -22,7 +23,7 @@ class Plugin(StrategyPlugin):
result = None result = None
for cand in reversed(candidates): for cand in reversed(candidates):
#try: # try:
data = cand.data data = cand.data
if key is None: if key is None:
@ -35,8 +36,7 @@ class Plugin(StrategyPlugin):
except KeyError: except KeyError:
pass pass
#else: # else:
# raise Exception(f"Data must be a dict, not something else ... {data}") # raise Exception(f"Data must be a dict, not something else ... {data}")
return result return result

18
kheops/plugin/strategy/merge_deep.py
View File

@ -17,21 +17,21 @@ class Plugin(StrategyPlugin):
_plugin_name = "merge_deep" _plugin_name = "merge_deep"
_schema_props_new = None _schema_props_new = None
selector = 'matched' selector = "matched"
def _init(self): def _init(self):
# Fetch module config # Fetch module config
# See documentation: https://github.com/clarketm/mergedeep # See documentation: https://github.com/clarketm/mergedeep
algo = self.ns.config['config'].get('merge_deep_algo', "replace").upper() algo = self.ns.config["config"].get("merge_deep_algo", "replace").upper()
strategy = getattr(Strategy, algo, None) strategy = getattr(Strategy, algo, None)
if strategy is None: if strategy is None:
strategies = [ i.lower() for i in dir(Strategy) if i.isupper() ] strategies = [i.lower() for i in dir(Strategy) if i.isupper()]
raise Exception (f"Unknown algorithm: {algo}, please choose one of: {strategies}") raise Exception(
f"Unknown algorithm: {algo}, please choose one of: {strategies}"
)
self.strategy = strategy self.strategy = strategy
def merge_results(self, candidates: list, rule: dict, query) -> (list, dict): def merge_results(self, candidates: list, rule: dict, query) -> (list, dict):
"""Return results""" """Return results"""
@ -51,14 +51,12 @@ class Plugin(StrategyPlugin):
except KeyError: except KeyError:
pass pass
# else:
#else:
# raise Exception(f"Data must be a dict, not something else ... {data}") # raise Exception(f"Data must be a dict, not something else ... {data}")
log.debug("Merging %s results", len(results)) log.debug("Merging %s results", len(results))
result = None result = None
if len(results) > 0 : if len(results) > 0:
result = merge(*results, strategy=self.strategy) result = merge(*results, strategy=self.strategy)
return result return result

18
kheops/utils.py
View File

@ -14,9 +14,6 @@ log = logging.getLogger(__name__)
# ===================== # =====================
def glob_files(path, pattern): def glob_files(path, pattern):
"""Return a list of path that match a glob""" """Return a list of path that match a glob"""
log.debug("Search glob '%s' in '%s'", pattern, path) log.debug("Search glob '%s' in '%s'", pattern, path)
@ -35,12 +32,9 @@ def path_assemble_hier(path, sep="/", reverse=False, start_index=0):
else: else:
raise Exception(f"This function only accepts string or lists, got: {path}") raise Exception(f"This function only accepts string or lists, got: {path}")
if reverse: if reverse:
list_data = list_data[::-1] list_data = list_data[::-1]
if start_index > 0: if start_index > 0:
fixed_part = list_data[:start_index] fixed_part = list_data[:start_index]
if reverse: if reverse:
@ -53,11 +47,10 @@ def path_assemble_hier(path, sep="/", reverse=False, start_index=0):
new_data.extend(hier_part) new_data.extend(hier_part)
list_data = new_data list_data = new_data
assert isinstance(list_data, list), f"Got: {list_data}" assert isinstance(list_data, list), f"Got: {list_data}"
ret = [] ret = []
for index, part in enumerate(list_data): for index, part in enumerate(list_data):
prefix ='' prefix = ""
try: try:
prefix = ret[index - 1] prefix = ret[index - 1]
prefix = f"{prefix}/" prefix = f"{prefix}/"
@ -74,9 +67,10 @@ def render_template(text, params):
tpl = Template(text) tpl = Template(text)
return tpl.render(**params) return tpl.render(**params)
class Default(dict): class Default(dict):
def __missing__(self, key): def __missing__(self, key):
return '' return ""
def render_template_python(text, params, ignore_missing=True): def render_template_python(text, params, ignore_missing=True):
@ -92,8 +86,6 @@ def render_template_python(text, params, ignore_missing=True):
return None return None
# Schema Methods # Schema Methods
# ===================== # =====================
@ -136,9 +128,7 @@ def schema_validate(config, schema):
path = list(collections.deque(err.schema_path)) path = list(collections.deque(err.schema_path))
path = "/".join([str(i) for i in path]) path = "/".join([str(i) for i in path])
path = f"schema/{path}" path = f"schema/{path}"
raise Exception( raise Exception(f"Failed validating {path} for resource with content: {config}")
f"Failed validating {path} for resource with content: {config}"
)
return config return config