python-shctl-iam/iam/catalog.py
2023-10-06 19:16:30 -04:00

871 lines
23 KiB
Python

import logging
from collections import namedtuple
from pprint import pprint
from types import SimpleNamespace
from . import exceptions as error
from .framework import DictCtrl, DictItem, KeyValue, KeyValueExtra
from .idents import Ident
from .lib.utils import format_render, jinja_render, uniq
logger = logging.getLogger(__name__)
IamException = error.IamException
# Config classes: Plugin
# ================
# class Plugin(DictItem):
# "Hold provider entities"
# default_attrs = {
# "desc": "Resource without description",
# "input": {},
# "needs": [],
# }
# class Plugins(DictCtrl):
# "Hold provider entities"
# default_attrs = {
# "desc": "Resource without description",
# "input": {},
# "needs": [],
# }
# Config classes: Services
# ================
class ServiceCommand(DictItem):
"Hold provider services command"
default_attrs = {
"desc": "Service without description",
"cmd": "",
}
# Overrides
# ---------------
def payload_transform(self, name, kwargs=None):
"Transform short form into long form"
payload = self._payload
if not isinstance(payload, dict):
payload = {"cmd": payload}
self._payload = payload
class Service(DictItem):
"Hold provider services"
default_attrs = {
"desc": "Service without description",
"enabled": False,
"input": {},
"resources_lookup": [],
"required_services": [],
"commands": {},
# "resolved_resources": [],
"resources_matches": 1,
}
# Overrides
# ---------------
def prepare(self):
self.catalog = self.parent.catalog
def init(self):
ret = {}
for cmd_name, cmd in self.commands.items():
ret[cmd_name] = ServiceCommand(cmd_name, cmd)
self.commands = ret
# def require_resolved_deps(func):
# "Decorator that ensure resource resolution is called before method call"
# def wrap(self, *args, **kwargs):
# "Wrapper that ensure dependencies are resolved"
# if not self._resolved:
# self.resolve_deps(skip_missing=True)
# return func(self, *args, **kwargs)
# return wrap
# Catalog dependent is built !
# ---------------
def resolve_resources(self, catalog):
"Resolve required resources"
# catalog.res_inst,
# catalog.context
self._catalog = catalog
requests = self.resources_lookup
curr_vars = catalog.context.get_vars()
ret = []
for pattern in requests:
req = pattern.format(**curr_vars)
if req in catalog.res_inst:
resource = catalog.res_inst[req]
tmp = {
"resource": resource,
"vars": resource.resolve_inputs(),
}
ret.append(tmp)
self.resolved_resources_names = [x["resource"].name for x in ret]
self._resolved_resources = ret
def _prepare_runtime(self):
if not hasattr(self, "_resolved_resources"):
assert (
self._resolved_resources
), f"Please run {self}.resolve_resources() first"
print("Process each item")
processed_vars = []
# Deduplicate common resources
for box in self._resolved_resources:
res = box["resource"]
name = res.name
curr_vars = res.resolve_inputs()
# WIPPPP
expand_inputs(curr_vars)
var_matches = [x for x in processed_vars if x.value == curr_vars]
if len(var_matches) > 0:
print(f" Skip duplicate resource vars {res.name}")
continue
# print (f" Process: {res.name}")
processed_vars.append(KeyValueExtra(name, curr_vars, res))
# print("PROCESSEDS")
# pprint(processed_vars)
return processed_vars
# Workflow
# ---------------
def run_cmd(self, command=None):
processed_vars = self._prepare_runtime()
cmd = self.commands.get(command, None)
if not cmd:
choices = (
",".join(self.commands.keys())
or "No available commands for this service"
)
raise IamException(
f"Unknown command: {command}, please choose one of: {choices}"
)
svc_input = self.input
# Run on each requested resources
max_items = self.resources_matches
for res in processed_vars[0:max_items]:
print("process item:", res.key)
# if command:
# pprint (res.value)
# # pprint (self.__dict__)
# # assert False
# pprint (self._catalog.context.get_vars())
env_var = {}
env_var.update(svc_input)
env_var.update(res.value) # == res.resolve_inputs()
env_var.update(self._catalog.context.get_vars())
# pprint (env_var)
try:
out = cmd.format(**env_var)
except KeyError as err:
msg = f"Missing variable: {err} for service: {res.key}"
# print (msg)
out = msg
# raise IamException(msg)
print("\n" + "-- %<--" * 6)
print(out)
print("-- %<--" * 6)
# def _list_cmds_shell(self):
# "Li"
# def _list_cmds(self):
def list_cmds(self):
"List all services commands"
prefix = self.name.split(".")[-1]
shell_prefix = "shell_"
ret2 = {
"shell": {},
"cmds": {},
}
for cmd_name, conf in self.commands.items():
target = ret2["cmds"]
if cmd_name.startswith(shell_prefix):
target = ret2["shell"]
cmd_name = cmd_name[len(shell_prefix) :]
name = f"{prefix} {cmd_name}"
target[name] = conf.desc
return ret2
def get_linked_resource(self):
"Return linked resource or None"
catalog = self.catalog
res_name = f"service.{self.name}:{catalog.ident.name}"
ret = catalog.resources.get(res_name, None)
logger.debug(f"Linked resource for service {self.name}: {ret}")
return ret
def is_active(self):
return True if self.get_linked_resource() is not None else False
class Services(DictCtrl):
"Simple wrapper class to manage services"
items_class = Service
def prepare(self):
self.catalog = self.parent
def get_linked_resources(self):
"""
Like get method, but only grab enabled services
Return a list of an object containing service and its
matched resource
"""
enabled_services = {}
for name, service in self.items():
res = service.get_linked_resource()
if res:
enabled_services[service.name] = SimpleNamespace(
svc=service,
res=res, # order=None
)
return enabled_services
def get_loading_order(self, reverse=False):
"Process resolution order"
# Filter out disabled services
enabled_services = self.get_linked_resources()
# Process resolution order
queue_todo = list(enabled_services.keys())
queue_done = []
loop_index = 0
while len(queue_todo) > 0:
loop_index = loop_index + 1
_queue_todo = list(queue_todo)
for svc_name in queue_todo:
service = enabled_services[svc_name].svc
resource = enabled_services[svc_name].res
# Check services requirements
required_services = list(set(service.required_services))
# if not svc_name.startswith("service.id:"):
if svc_name != "id":
required_services.insert(0, f"id")
missing = False
_missing = []
for req in required_services:
if not req in queue_done:
missing = True
_missing.append(req)
if not missing:
# Save result
queue_todo.remove(svc_name)
queue_done.append(svc_name)
if queue_todo == _queue_todo:
print("Loop Index ERROR", loop_index)
raise IamException(f"Loop detexted ! {loop_index}")
if reverse:
queue_done.reverse()
for index, svc_name in enumerate(queue_done):
enabled_services[svc_name].order = index
# Return list of ordered service names, and other useful data
return queue_done, enabled_services
# Config classes: ResourcesKinds
# ================
ResolvedRes = namedtuple("ResolvedRes", "name inputs")
class ResourceKind(DictItem):
"Hold resource kinds"
default_attrs = {
"desc": "ResourceKind without description",
"input": {},
"needs": [],
"remap": {},
}
# def config_validate(self):
# if ":" in self.name:
# raise IamException(
# "Colons are not accepted for catalog resources", self.__dict__
# )
class ResourcesKinds(DictCtrl):
"Simple wrapper class to manage resources kinds"
items_class = ResourceKind
# def prepare(self):
# self.catalog = self.parent
# Config classes: Resources
# ================
class Resource(DictItem):
"Create a new resource"
default_attrs = {
"desc": "",
"input": {},
"uses": [], # Optional
"needs": [], # Required dependencies AND
"loop_limit": 1,
"loop": [],
"_ctx_vars": {},
}
# Overrides
# ---------------
def prepare(self):
self.catalog = self.parent.catalog
def config_validate(self):
# Fetch info from controller
resources_kind = self.parent.available_resources_kinds
loaded_resources = self.parent.loaded_resources
# Check overrides
if self.name in loaded_resources:
_msg = f"User resource overrides: {self.name}"
logger.debug(_msg)
# Check for the kind
kind = self.get_kind()
if not kind in resources_kind:
choices = ", ".join(resources_kind.keys())
_msg = (
f"Unknown resource kind: {self.name}, please choose one of: {choices}"
)
raise error.UnknownResourceKind(_msg)
def init(self):
"Init each resources"
self.name_raw = self.name
ctx_vars = self.parent.ctx_vars
self.name = format_render(self.name, vars=ctx_vars)
# self.name = format_render(self.name, vars=self.catalog.context.get_vars())
# Append common base dependency on all resources
if len(self.uses) == 0:
self.uses.insert(0, "account:{user}")
# self.uses = list(set(self.uses))
self.uses = uniq(self.uses)
# Resolution management vars
self._resolved = False
self.resources_deps = None
self.resources_missing = None
# Helpers
# ---------------
def _parse_name(self):
ret = self.name.split(":", 2)
if len(ret) == 2:
return ret[0], ret[1]
# Raise error if name is invalid
_msg = f"Invalid resource name: {self.name}, missing :<name>"
raise IamException(_msg)
def get_kind(self):
"Return resource kind"
ret, _ = self._parse_name()
return ret
def get_name(self):
"Return resource instance name"
_, ret = self._parse_name()
return ret
# Methods
# ---------------
def require_resolved_deps(func):
"Decorator that ensure resource resolution is called before method call"
def wrap(self, *args, **kwargs):
"Wrapper that ensure dependencies are resolved"
if not self._resolved:
self.resolve_deps(skip_missing=True)
return func(self, *args, **kwargs)
return wrap
def resolve_deps(self, vars=None, add_self=True, cache=True, skip_missing=False):
"Recursive uses dependency resolver"
if cache and self.resources_deps is not None:
return self.resources_deps
# try:
# ret, _missing = self._resolve_deps(vars=vars, add_self=add_self)
# except error.UnresolvedResourceDependencies:
# return None
dependencies, missings = self._resolve_deps(vars=vars, add_self=add_self)
self.resources_missing = missings
self.resources_deps = dependencies
self._is_active = True if len(missings) == 0 else False
# Raise error on missing deps
if not skip_missing and len(missings) > 0:
pprint(self.catalog.resources.get())
_msg = f"Missing deps for {self.name}: {missings}"
raise error.UnresolvedResourceDependencies(_msg)
if cache:
self._resolved_deps = dependencies
self._resolved = True
return dependencies
def _resolve_deps(self, vars=None, add_self=True, _results=None, _missings=None):
"Recursive dependency helper"
# Recurisive control
_missings = _missings or []
_results = _results or []
if self in _results:
return _results
# Get root context
# ctx_vars = vars or self.catalog.context.get_vars()
catalog = self.catalog
ctx_vars = catalog.context.get_vars()
# Check each dependency
uses = self.uses
for dep_name in uses:
# Parse resource name
if ctx_vars:
dep_name = format_render(dep_name, ctx_vars)
if dep_name == self.name:
continue
if dep_name in _missings:
_missings.delete(dep_name)
# Load child or quit if missing
child = catalog.resources.find_closest(dep_name)
if not child:
_missings.append(dep_name)
# _missings.append((self.name, dep_name, ))
# _msg = f"Undeclared resource: {dep_name}"
# raise error.UnresolvedResourceDependencies(_msg)
continue
# Avoid already processed resources
if child and child in _results:
continue
_results.append(child)
child._resolve_deps(vars=vars, _results=_results)
if add_self:
_results.append(self)
return _results, _missings
# Resolved methods
# ==================
@require_resolved_deps
def is_active(self):
"Return True if this resource is active"
return self._is_active
# return True if len(self.resources_missing) == 0 else False
@require_resolved_deps
def resolve_inputs(self, vars=None):
"Resolve uses dependencies recursively"
# WIPPPP, split this function in 2 !!!
# self.catalog.
# deps = self.resolve_deps()
deps = self.resources_deps
output = {}
for res in deps:
output.update(res.input)
return output
def loop_extend(self):
"Extend loops from catalog"
ctx_vars = self.catalog.context.get_vars()
loops = self.loop
loop_limit = self.loop_limit
matches = []
stop = False
for loop_search in loops:
if len(matches) >= loop_limit:
stop = True
if not stop:
loop_search = loop_search.format(**ctx_vars)
res = self.catalog.resources.get(loop_search, skip_missing=True)
if res:
matches.append(res)
return matches
class Resources(DictCtrl):
"Simple wrapper class to manage resources"
items_class = Resource
# Overrides
# ---------------
def prepare(self):
# Save Catalog
self.catalog = self.parent
self.ctx_vars = self.catalog.context.get_vars()
self.available_resources_kinds = self.catalog.resources_kind
self.loaded_resources = []
# pprint (self.__dict__)
# print ("INIT RES CATALOG")
def find_closest(self, dep_name):
"Find closest match on type"
assert ":" in dep_name, f"We have unresolved query here: {dep_name}"
catalog = self.catalog
# Direct access
child = catalog.resources.get(dep_name, None)
if child:
return child
# Loop over resources
target = dep_name.split(":")[0]
scope = dep_name.split(":")[1]
match = None
while target:
# Query other resources
# query_name = f"{target}:{scope}"
matches = catalog.resources.select("startswith", target)
matches = {
key: val for key, val in matches.items() if key.endswith(f":{scope}")
}
# Check closest matches
if len(matches) == 1:
match = list(matches.values())[0]
# print ("Longer MATCH", match.name)
logger.info(
f"Closest resource of {dep_name} was mapped to: {match.name}"
)
break
elif len(matches) > 1:
logger.debug(
f"More than one closest resources for {dep_name}: {matches}"
)
# raise Exception(f"Bug here, too many matches: {matches}")
# Reduce again the common lookup pattern
if "." in target:
target = ".".join(target.split(".")[:-1])
else:
break
# print ("QUERY", target, scope)
return match
# view = scoped_ident(ident)
# Catalog
# ============================================================
# Context classes
# ================
class Context(DictItem):
"Class that hold a context"
default_attrs = {
"ident": None,
}
# Overrides
# ---------------
def config_validate(self):
assert isinstance(self.ident, Ident), f"Gotr: {self.ident}"
def init(self):
self._vars = {
"ident": self.ident.name,
"user": self.ident.name,
}
def get_vars(self):
"Return all vars context"
return dict(self._vars)
# Catalog classes
# ================
class Catalog:
"Manage catalog resources"
def __init__(self, mgr, ident):
# Prepare catalog
self.mgr = mgr
self.ident = ident
# Prepare context
ctx_conf = {
"ident": ident,
}
self.context = Context("current_context", ctx_conf)
self._prepare_catalog()
# Catalog building classes
# ================
def _merge_inst_configs(self, merge=True):
plugins_confs = self.mgr.providers.get_resource_configs()
user_confs = self.ident.get_resource_configs()
final_config = {}
# Remap config names
for source in [plugins_confs, user_confs]:
# source = source or {}
for name, conf in source.items():
# Check name
if not ":" in name:
ident_name = f"{name}:{self.ident.name}"
# print (f"Renaming resource '{name}' to '{ident_name}'")
name = ident_name
# Check config
existing = final_config.get(name, None)
if existing:
# print (f"Overrides: {name}")
if merge:
new_conf = dict(existing)
new_conf.update(conf)
conf = new_conf
final_config[name] = conf
return final_config
def _prepare_catalog(self):
"Prepare catalog resources configs"
# Get resources from providers and user
providers = self.mgr.providers
ident = self.ident
# Build kind catalog
res_kind_configs = providers.get_resource_kinds()
self.resources_kind = ResourcesKinds(
"CatalogResourcesKinds", payload=res_kind_configs, parent=self
)
# Build resources catalog
res_inst_configs = self._merge_inst_configs()
self.resources = Resources(
"CatalogResources", payload=res_inst_configs, parent=self
)
self._resources_config = res_inst_configs
# Prepare services catalog
services_confs = providers.get_services_configs()
self.services = Services("CatalogServices", payload=services_confs, parent=self)
def dump(self):
print("Catalog settings")
print("=" * 16)
# pprint (self.__dict__)
# return
print("\n=== Catalog resources kinds:")
pprint(self.resources_kind)
print("\n=== Catalog resources configs:")
pprint(self._resources_config)
print("\n=== Catalog resources instances:")
pprint(self.resources)
print("\n=== Catalog services instances:")
# for name, srv in self.services.items():
# pprint(srv.dump_item())
pprint({name: srv.dump_item() for name, srv in self.services.items()})
# Catalog interfaces
# ================
# Catalog helpers
# ================
# def resolve_service_order(self, reverse=False):
# "Resolve service order"
# return self.services.get_loading_order(reverse=reverse)
# Shell helpers
# ================
def shell_enable(self):
"Enable shell"
return self._shell_action()
def shell_disable(self):
"Disable shell"
return self._shell_action(reverse=True)
def _shell_action(self, reverse=False):
"Run command order"
ident = self.ident
vars = self.context.get_vars()
# Fetch execution order
order, services = self.services.get_loading_order(reverse=reverse)
# Prepare context
action_name = "shell_enable"
log_action_name = "Enable"
if reverse:
action_name = "shell_disable"
log_action_name = "Disable"
logger.info(f"{log_action_name} identity: {ident.name}")
# Execute on each plugins commands
output_code = []
for srv_name in order:
service = services[srv_name].svc
res = services[srv_name].res
# Load service command
command = service.commands.get(action_name, None)
if not command:
continue
# Build loop with at least ONE item
cmd = command.cmd
if not cmd:
continue
# Create context var dict
ctx_vars = dict()
ctx_vars.update(service.input)
ctx_vars.update(vars)
ctx_vars.update(res.resolve_inputs(vars=ctx_vars))
loops = res.loop_extend() or [res]
res_vars = [x.resolve_inputs(vars=ctx_vars) for x in loops]
ctx_vars.update(
{
"item": res_vars[0],
"loop": res_vars,
}
)
logger.debug(f"{log_action_name} service: {res.name}")
# pprint (ctx_vars)
cmd = jinja_render(cmd, ctx_vars)
output_code.append(f"# Loading of {res.name} ({service.name})")
output_code.append(f"# =====================")
output_code.append(cmd)
output_code.append("\n")
return "\n".join(output_code)