pro-barbu-foss/kheops
pro-barbu-foss/kheops
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update: schema management across plugins, trailing spaces cleanup

Browse Source
This commit is contained in:
mrjk 2022-01-15 17:44:02 -05:00
parent 50c7af2f58
commit 4854bb240f
8 changed files with 212 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
albero/app.py
View File

@ -22,15 +22,15 @@ log = logging.getLogger(__name__)
class App(): class App():
schema = { schema = {
"$schema": 'http://json-schema.org/draft-04/schema#', "$schema": 'http://json-schema.org/draft-07/schema#',
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
"default": {}, "default": {},
"$def" :{ "$def" :{
'backends_items': None, 'backends_items': {},
'backends_config': None, 'backends_config': {},
'rules_items': None, 'rules_items': {},
'rules_config': None, 'rules_config': {},
}, },
"patternProperties": { "patternProperties": {
".*": { ".*": {
@ -65,17 +65,20 @@ class App():
"default": {}, "default": {},
}, },
}, },
}, },
"tree": { "tree": {
"type": "array", "type": "array",
"default": [], "default": [],
"arrayItem": { "$ref": "#/$defs/backends_items" }, "items": {
"type": "object",
"properties": { "$ref": "#/$defs/backends_items" },
},
}, },
"rules": { "rules": {
"type": "array", "type": "array",
"default": [], "default": [],
"arrayItem": { "$ref": "#/$defs/rules_items" }, # "arrayItem": { "$ref": "#/$defs/rules_items" },
}, },
}, },
}, },
@ -122,12 +125,9 @@ class App():
r2 = RulesManager.get_schema(AlberoPlugins) r2 = RulesManager.get_schema(AlberoPlugins)
d = self.schema d = self.schema
d['$def']['backends_items'] = r1 d['patternProperties']['.*']['properties'] ['tree']['items']['properties'] = r1
d['$def']['rules_items'] = r2 d['patternProperties']['.*']['properties'] ['tree']['items'] = r2
d['$def']['backends_config'] = None
d['$def']['rules_config'] = None
#pprint(d)
print(json.dumps(d, indent=2)) print(json.dumps(d, indent=2))

2
albero/cli.py
View File

@ -153,7 +153,7 @@ class CmdApp:
config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml' config = '/home/jez/prj/bell/training/tiger-ansible/tree.yml'
# self.log.debug(f"Command line vars: {vars(self.args)}") # self.log.debug(f"Command line vars: {vars(self.args)}")
keys = self.args.key or [None] keys = self.args.key or [None]
# Parse payload from enf file: # Parse payload from enf file:
new_params = {} new_params = {}

103
albero/managers.py
View File

@ -1,4 +1,6 @@
import dpath.util
import copy import copy
import json import json
import textwrap import textwrap
@ -47,37 +49,32 @@ class LoadPlugin():
class Manager(): class Manager():
plugins_kind = [] plugins_kind = []
_schema_props_default = None
_schema_props_default = {}
_schema_props_default = {}
@classmethod @classmethod
def get_schema(cls, plugins_db): def get_schema(cls, plugins_db):
pprint (cls.plugins_kind)
ret = {}
ret3 = []
# Properties
ret3 = {}
for kind in cls.plugins_kind: for kind in cls.plugins_kind:
ret[kind] = {} # ret[kind] = {}
plugin_kind = getattr(plugins_db, kind) plugin_kind = getattr(plugins_db, kind)
for plugin_name in [i for i in dir(plugin_kind) if not i.startswith('_')]: for plugin_name in [i for i in dir(plugin_kind) if not i.startswith('_')]:
plugin = getattr(plugin_kind, plugin_name) plugin = getattr(plugin_kind, plugin_name)
print (plugin.Plugin)
#pprint (dir(plugin))
plugin_cls = getattr(plugin, 'Plugin', None) plugin_cls = getattr(plugin, 'Plugin', None)
if plugin_cls: if plugin_cls:
schema_props = getattr(plugin_cls, '_schema_props_new', 'MISSING ITEM') schema_props = getattr(plugin_cls, '_schema_props_new', 'MISSING ITEM')
if schema_props: if schema_props:
ret[kind][plugin_name] = schema_props # ret[kind][plugin_name] = schema_props
print (plugin_name) ret3.update( schema_props )
ret3.append( schema_props ) ret3.update( cls._schema_props_new )
ret3.append( cls._schema_props_new )
# Injection
ret1 = cls._schema_props_default ret1 = cls._schema_props_default
ret1["$def"]["items"] = ret3 position = cls._props_position
dpath.util.set(ret1, position, ret3)
return ret1 return ret1
@ -86,37 +83,42 @@ class BackendsManager(Manager):
plugins_kind = ['engine', 'backend'] plugins_kind = ['engine', 'backend']
_schema_props_new = { _schema_props_new = {
"engine": { "engine": {
"type": "string", "type": "string",
"default": "jerakia", "default": "jerakia",
"optional": False, "optional": False,
}, },
"value": { "value": {
"default": 'UNSET', "default": 'UNSET',
"optional": False, "optional": False,
}, },
#### INSERT HERE SUBSCHEMA !!!!!
} }
_schema_props_default = { _props_position = 'oneOf/0/properties'
"$schema": 'http://json-schema.org/draft-04/schema#', _schema_props_default = {
"$schema": 'http://json-schema.org/draft-07/schema#',
"default": "", "default": "",
"$def": { # This does not work :(
"items": {}, #"$def": {
}, # "props": {},
# },
"oneOf": [ "oneOf": [
{ {
"type": "string", "type": "object",
"default": "BLAAAAHHH" "additionalProperties": True,
"default": {},
"title": "object",
"properties": {},
"description": "Object to configure a bacjend item",
}, },
{ {
"type": "object", "type": "string",
"additionalProperties": True, "default": "BLAAAAHHH",
"default": {}, "title": "string",
"properties": { "$ref": "#/$defs/name" }, "description": "Enter a simple string configuration value for default engine",
}, },
] ]
} }
def _validate_item(self, item): def _validate_item(self, item):
if isinstance(item, str): if isinstance(item, str):
@ -251,28 +253,29 @@ class RulesManager(Manager):
}, },
} }
_schema_props_default = { _props_position = 'oneOf/1/properties'
"$schema": 'http://json-schema.org/draft-04/schema#', _schema_props_default = {
"$schema": 'http://json-schema.org/draft-07/schema#',
"default": "", "default": "",
"$def": { "$def": {
"items": {}, "items": {},
}, },
"oneOf": [ "oneOf": [
{ {
"type": "string", "type": "string",
"default": "BLAAAAHHH" "default": "BLAAAAHHH"
}, },
{ {
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"default": {}, "default": {},
"properties": { "$ref": "#/$defs/name" }, "properties": { "$ref": "#/$defs/name" },
}, },
] ]
} }
OLD_rule_schema = { OLD_rule_schema = {
"$schema": 'http://json-schema.org/draft-04/schema#', "$schema": 'http://json-schema.org/draft-07/schema#',
"type": "object", "type": "object",
"additionalProperties": False, "additionalProperties": False,
"properties": { "properties": {
@ -285,13 +288,13 @@ class RulesManager(Manager):
"type": "null", "type": "null",
}, },
{ {
"type": "string", "type": "string",
}, },
{ {
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"default": {}, "default": {},
"properties": { "$ref": "#/$defs/name" }, "properties": { "$ref": "#/$defs/name" },
}, },
], ],
}, },
@ -331,7 +334,7 @@ class RulesManager(Manager):
else: else:
matched_rule = rule[0] matched_rule = rule[0]
log.debug(f"Matcher rule for {key}: {matched_rule}") log.debug(f"Matcher rule for {key}: {matched_rule}")
matched_rule['trace'] = trace matched_rule['trace'] = trace
matched_rule['explain'] = explain matched_rule['explain'] = explain
schema_validate(matched_rule, self._schema_props_default) schema_validate(matched_rule, self._schema_props_default)

115
albero/plugin/backend/loop.py
View File

@ -14,41 +14,110 @@ import textwrap
class Plugin(PluginBackendClass): class Plugin(PluginBackendClass):
_plugin_name = "loop" _plugin_name = "loop"
_plugin_help = """
This module helps to loop over a backend
""",
_schema_props_new = { _schema_props_new = {
"loop": { "loop": {
"description": _plugin_help,
"default": None, "default": None,
"optional": True, "optional": True,
"examples": [
{
"value": "site/{{ loop_env }}/config/{{ os }}",
"loop": {
"var": "loop_env",
"data": [
"dev",
"preprod",
"prod",
],
},
"comment": "The module will loop three time over the value, and the variable `loop_env` will consecutely have `dev`, `preprod` and `prod` as value",
},
{
"value": "site/{{ loop_env2 }}/config/{{ os }}",
"loop": {
"var": "loop_env2",
"data": "my_scope_var",
},
"comment": "Like the previous example, but it will fetch the list from any scope variables",
},
{
"loop": None,
"comment": "Disable this module, no loop will operate",
},
# "loop": {
# "var": "my_var",
# },
# },
# "loop": {
# "var": "my_var",
# },
# "example": "",
# },
# "loop": {
# "var": "my_var",
# },
# "example": "",
# },
],
"oneOf": [ "oneOf": [
{ {
"type": "null", "type": "object",
}, "additionalProperties": False,
{
"type": "string",
},
{
"type": "object",
"additionalProperties": True,
"default": {}, "default": {},
"properties": { "title": "Complete config",
"data": { "description": "",
"properties": {
"data": {
"default": None, "default": None,
"optional": False, "optional": False,
"title": "Module configuration",
"description": "Data list used for iterations. It only accept lists as type. It disable the module if set to `null`.",
"anyOf":[ "anyOf":[
{"type": "null"}, {
{"type": "string"}, "type": "null",
{"type": "array"}, "title": "Disable Module",
"description": "Disable the module",
},
{
"type": "string",
"title": "Scope variable",
"description": "Will look the value of the loop list from the scope. TOFIX: What if variablle does not exists?",
},
{
"type": "array",
"title": "Hardcoded list",
"description": "Simply enter the list of value to be iterated to.",
},
] ]
}, },
"var": { "var": {
"type": "string", "type": "string",
"default": "loop_item", "default": "loop_item",
"optional": True, "optional": True,
}, "title": "Module configuration",
}, "description": "Name of the variable to be used in templating language",
},
},
},
{
"type": "string",
"title": "Short config",
"description": "If set to string, it will define the name of the variable to lookup into the scope.",
},
{
"type": "null",
"title": "Disable",
"description": "If set to null, it disable the module",
}, },
] ]
} }
} }
def process(self, backends: list, ctx: dict) -> (list, dict): def process(self, backends: list, ctx: dict) -> (list, dict):
@ -84,7 +153,7 @@ class Plugin(PluginBackendClass):
_cand['_run']['scope'][loop_var] = item _cand['_run']['scope'][loop_var] = item
#_cand.scope[loop_var] = item #_cand.scope[loop_var] = item
ret.append(_cand) ret.append(_cand)
new_backends.extend(ret) new_backends.extend(ret)
return new_backends, ctx return new_backends, ctx

18
albero/plugin/common.py
View File

@ -16,10 +16,10 @@ import copy
# Candidate Classes # Candidate Classes
# ============================= # =============================
class Candidate(): class Candidate():
engine = None engine = None
found = False found = False
data = None data = None
run = None run = None
scope = None scope = None
key = None key = None
@ -27,9 +27,9 @@ class Candidate():
def __init__(self, run): def __init__(self, run):
self.run = copy.deepcopy(run) self.run = copy.deepcopy(run)
def __repr__(self): def __repr__(self):
return f"{self.__dict__}" return f"{self.__dict__}"
def _report_data(self, data=None): def _report_data(self, data=None):
default_data = { default_data = {
#"rule": self.config, #"rule": self.config,
@ -149,7 +149,7 @@ class PluginEngineClass(PluginClass):
schema = getattr(self, key) schema = getattr(self, key)
props.update(schema) props.update(schema)
self.schema = { self.schema = {
"$schema": 'http://json-schema.org/draft-04/schema#', "$schema": 'http://json-schema.org/draft-07/schema#',
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"properties": props, "properties": props,
@ -205,7 +205,7 @@ class PluginFileGlob():
} }
def _glob(self, item): def _glob(self, item):
# DIRECT CALL TO APP< TOFIX # DIRECT CALL TO APP< TOFIX
app_config = self.app.conf2 app_config = self.app.conf2
root = app_config.get("default", {}).get("config", {}).get("root", f"{Path.cwd()}/tree") root = app_config.get("default", {}).get("config", {}).get("root", f"{Path.cwd()}/tree")

34
albero/plugin/strategy/schema.py
View File

@ -20,41 +20,41 @@ class Plugin(PluginStrategyClass):
"optional": True, "optional": True,
"oneOf": [ "oneOf": [
{ {
"type": "null", "type": "null",
}, },
{ {
"type": "string", "type": "string",
}, },
{ {
"type": "array", "type": "array",
}, },
{ {
"type": "object", "type": "object",
"additionalProperties": True, "additionalProperties": True,
"default": {}, "default": {},
"properties": { "properties": {
"data": { "data": {
"default": None, "default": None,
"optional": False, "optional": False,
"anyOf":[ "anyOf":[
{"type": "null"}, {"type": "null"},
{"type": "string"}, {"type": "string"},
{"type": "array"}, {"type": "array"},
] ]
}, },
"var": { "var": {
"type": "string", "type": "string",
"default": "loop_item", "default": "loop_item",
"optional": True, "optional": True,
}, },
}, },
}, },
] ]
} }
} }
default_merge_schema = { default_merge_schema = {
"$schema": 'http://json-schema.org/draft-04/schema#', "$schema": 'http://json-schema.org/draft-07/schema#',
"oneOf": [ "oneOf": [
{ {
"type": "array", "type": "array",

52
albero/utils.py
View File

@ -15,46 +15,46 @@ log = logging.getLogger(__name__)
# # File parsers # # File parsers
# # ===================== # # =====================
# #
# class FileParserClass(): # class FileParserClass():
# #
# def __init__(self, path): # def __init__(self, path):
# self.path = path # self.path = path
# #
# def from_file(self, file): # def from_file(self, file):
# raise Exception ("Not implemented") # raise Exception ("Not implemented")
# #
# def from_string(self, data): # def from_string(self, data):
# raise Exception ("Not implemented") # raise Exception ("Not implemented")
# #
# def from_dict(self, data): # def from_dict(self, data):
# raise Exception ("Not implemented") # raise Exception ("Not implemented")
# #
# class FilesYAMLParser(FileParserClass): # class FilesYAMLParser(FileParserClass):
# def get_data(self): # def get_data(self):
# with open(self.path, "r") as stream: # with open(self.path, "r") as stream:
# try: # try:
# return yaml.safe_load(stream) # return yaml.safe_load(stream)
# except yaml.YAMLError as exc: # except yaml.YAMLError as exc:
# raise Exception(exc) # raise Exception(exc)
# print(exc) # print(exc)
# #
# #
# class FilesJSONParser(FileParserClass): # class FilesJSONParser(FileParserClass):
# pass # pass
# class FilesRawParser(FileParserClass): # class FilesRawParser(FileParserClass):
# pass # pass
# class FilesTOMLParser(FileParserClass): # class FilesTOMLParser(FileParserClass):
# pass # pass
# class FilesCSVParser(FileParserClass): # class FilesCSVParser(FileParserClass):
# pass # pass
# class FilesINIParser(FileParserClass): # class FilesINIParser(FileParserClass):
# pass # pass
# #
# format_db = { # format_db = {
# ".raw": FilesRawParser, # ".raw": FilesRawParser,
# ".yml": FilesYAMLParser, # ".yml": FilesYAMLParser,
# ".yaml": FilesYAMLParser, # ".yaml": FilesYAMLParser,
# ".json": FilesJSONParser, # ".json": FilesJSONParser,
# } # }

1
pyproject.toml
View File

@ -12,6 +12,7 @@ jsonmerge = "^1.8.0"
anyconfig = "^0.12.0" anyconfig = "^0.12.0"
python-box = "^5.4.1" python-box = "^5.4.1"
prettytable = "^3.0.0" prettytable = "^3.0.0"
dpath = "^2.0.5"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
json-schema-for-humans = "^0.40" json-schema-for-humans = "^0.40"