import logging import os from collections import namedtuple from pprint import pprint from types import SimpleNamespace # TO BE REMOVED !!!! import click import sh from . import exceptions as error from .framework import DictCtrl, DictItem, KeyValue, KeyValueExtra from .idents import Ident from .lib.utils import (empty, format_render, jinja_render, jinja_template_vars, uniq) logger = logging.getLogger(__name__) cli_logger = logging.getLogger("iam.cli") IamException = error.IamException # Config classes: Plugin # ================ # class Plugin(DictItem): # "Hold provider entities" # default_attrs = { # "desc": "Resource without description", # "input": {}, # "needs": [], # } # class Plugins(DictCtrl): # "Hold provider entities" # default_attrs = { # "desc": "Resource without description", # "input": {}, # "needs": [], # } # Config classes: Services # ================ class ServiceCommand(DictItem): "Hold provider services command" default_attrs = { "desc": "Service without description", "cmd": "", # Direct commands to launch "shell": "", # Will be run in a shell (or specific shell below) # "shell_sh": "", # "shell_bash": "", # "shell_zsh": "", # "shell_fish": "", # "shell_bash": "", # "source_output": False, # True for shell commands by default ! } # Overrides # --------------- def init(self): "Transform short form into long form" self.service = self.parent # def payload_transform(self, name, kwargs=None): # "Transform short form into long form" # self.service = self.parent # payload = self._payload # if not isinstance(payload, dict): # payload = {"cmd": payload} # self._payload = payload def cmd_name(self): return self.name.replace("_", " ") class Service(DictItem): "Hold provider services" default_attrs = { "desc": "Service without description", "enabled": False, "input": {}, "resources_lookup": [], "required_services": [], "commands": {}, # "resolved_resources": [], "resources_matches": 1, } # Overrides # --------------- def prepare(self): self.catalog = self.parent.catalog def init(self): ret = {} for cmd_name, cmd in self.commands.items(): ret[cmd_name] = ServiceCommand(cmd_name, cmd, parent=self) self.commands = ret # def require_resolved_deps(func): # "Decorator that ensure resource resolution is called before method call" # def wrap(self, *args, **kwargs): # "Wrapper that ensure dependencies are resolved" # if not self._resolved: # self.resolve_deps(skip_missing=True) # return func(self, *args, **kwargs) # return wrap # Catalog dependent is built ! # --------------- def resolve_resources(self, catalog): "Resolve required resources" # catalog.res_inst, # catalog.context self._catalog = catalog requests = self.resources_lookup curr_vars = catalog.context.get_vars() ret = [] for pattern in requests: req = pattern.format(**curr_vars) if req in catalog.res_inst: resource = catalog.res_inst[req] tmp = { "resource": resource, "vars": resource.resolve_inputs(), } ret.append(tmp) self.resolved_resources_names = [x["resource"].name for x in ret] self._resolved_resources = ret def _prepare_runtime(self): if not hasattr(self, "_resolved_resources"): assert ( self._resolved_resources ), f"Please run {self}.resolve_resources() first" print("Process each item") processed_vars = [] # Deduplicate common resources for box in self._resolved_resources: res = box["resource"] name = res.name curr_vars = res.resolve_inputs() # WIPPPP expand_inputs(curr_vars) var_matches = [x for x in processed_vars if x.value == curr_vars] if len(var_matches) > 0: print(f" Skip duplicate resource vars {res.name}") continue # print (f" Process: {res.name}") processed_vars.append(KeyValueExtra(name, curr_vars, res)) # print("PROCESSEDS") # pprint(processed_vars) return processed_vars # Workflow # --------------- def run_cmd(self, command=None): processed_vars = self._prepare_runtime() cmd = self.commands.get(command, None) if not cmd: choices = ( ",".join(self.commands.keys()) or "No available commands for this service" ) raise IamException( f"Unknown command: {command}, please choose one of: {choices}" ) svc_input = self.input # Run on each requested resources max_items = self.resources_matches for res in processed_vars[0:max_items]: print("process item:", res.key) # if command: # pprint (res.value) # # pprint (self.__dict__) # # assert False # pprint (self._catalog.context.get_vars()) env_var = {} env_var.update(svc_input) env_var.update(res.value) # == res.resolve_inputs() env_var.update(self._catalog.context.get_vars()) # pprint (env_var) try: out = cmd.format(**env_var) except KeyError as err: msg = f"Missing variable: {err} for service: {res.key}" logger.warning(msg) out = msg # raise IamException(msg) print("\n" + "-- %<--" * 6) print(out) print("-- %<--" * 6) # def _list_cmds_shell(self): # "Li" # def _list_cmds(self): def get_cmds(self): "Get all services commands objects" return [ cmd for name, cmd in self.commands.items() if not name.startswith("shell") ] def list_cmds(self): "List all services commands" prefix = self.name.split(".")[-1] shell_prefix = "shell_" ret2 = { "shell": {}, "cmds": {}, } for cmd_name, conf in self.commands.items(): target = ret2["cmds"] if cmd_name.startswith(shell_prefix): target = ret2["shell"] # cmd_name = cmd_name[len(shell_prefix):] name = f"{cmd_name}_{prefix}" else: name = cmd_name.replace("_", " ") target[name] = conf.desc return ret2 def get_linked_resource(self): "Return linked resource or None" catalog = self.catalog res_name = f"service.{self.name}:{catalog.ident.name}" ret = catalog.resources.get(res_name, None) logger.debug(f"Linked resource for service {self.name}: {ret}") return ret def is_active(self): return True if self.get_linked_resource() is not None else False class Services(DictCtrl): "Simple wrapper class to manage services" items_class = Service RESERVED_CMD_PREFIX = ["kind", "res", "svc", "shell", "run"] def prepare(self): self.catalog = self.parent def get_svc_command(self, cmd): "Return the command to be run" # Prepare context cmd_parts = cmd if isinstance(cmd, str): cmd_parts = cmd.split(" ") cmd_req = " ".join(cmd_parts) # Retrieve command list cmds = self.list_cmds() cmd_names = [cmd.name for cmd in cmds] # Find best matching command cmd_split_idx = None cmd = None curr = None for index, part in enumerate(cmd_parts): curr = f"{curr} {part}" if curr is not None else part if curr in cmd_names: cmd_idx = cmd_names.index(curr) cmd = cmds[cmd_idx] cmd_split_idx = index + 1 break # Validate result if not cmd: _choices = ",".join(cmd_names) _msg = f"No such services command named: {cmd_req}, please choose one of: {_choices}" raise error.UnknownServiceCommand(_msg) args = cmd_parts[cmd_split_idx:] return cmd, args def list_cmds(self): "List all services commands" ret = [] for name, service in self.items(): cmds = service.get_cmds() ret.extend(cmds) # Check for invalid configs conf = [] for cmd in ret: prefix = cmd.name.split(" ")[0] if prefix in self.RESERVED_CMD_PREFIX: _msg = f"Forbidden prefix! {cmd}" raise Exception(_msg) conf.append(cmd.name) dump = uniq(conf) if conf != dump: _msg = f"Duplicates values! {conf}" raise Exception(_msg) return ret def get_linked_resources(self): """ Like get method, but only grab enabled services Return a list of an object containing service and its matched resource """ enabled_services = {} for name, service in self.items(): res = service.get_linked_resource() if res: enabled_services[service.name] = SimpleNamespace( svc=service, res=res, # order=None ) return enabled_services def get_loading_order(self, reverse=False): "Process resolution order" # Filter out disabled services enabled_services = self.get_linked_resources() # Process resolution order queue_todo = list(enabled_services.keys()) queue_done = [] loop_index = 0 while len(queue_todo) > 0: loop_index = loop_index + 1 _queue_todo = list(queue_todo) for svc_name in queue_todo: service = enabled_services[svc_name].svc resource = enabled_services[svc_name].res # Check services requirements required_services = list(set(service.required_services)) # if not svc_name.startswith("service.id:"): if svc_name != "id": required_services.insert(0, f"id") missing = False _missing = [] for req in required_services: if not req in queue_done: missing = True _missing.append(req) if not missing: # Save result queue_todo.remove(svc_name) queue_done.append(svc_name) if queue_todo == _queue_todo: print("Loop Index ERROR", loop_index) raise IamException(f"Loop detexted ! {loop_index}") if reverse: queue_done.reverse() for index, svc_name in enumerate(queue_done): enabled_services[svc_name].order = index # Return list of ordered service names, and other useful data return queue_done, enabled_services # Config classes: ResourcesKinds # ================ ResolvedRes = namedtuple("ResolvedRes", "name inputs") class ResourceKind(DictItem): "Hold resource kinds" default_attrs = { "desc": "ResourceKind without description", "input": {}, "needs": [], "remap": {}, } # def config_validate(self): # if ":" in self.name: # raise IamException( # "Colons are not accepted for catalog resources", self.__dict__ # ) class ResourcesKinds(DictCtrl): "Simple wrapper class to manage resources kinds" items_class = ResourceKind # def prepare(self): # self.catalog = self.parent # Config classes: Resources # ================ class Resource(DictItem): "Create a new resource" default_attrs = { "desc": "", "input": {}, "uses": [], # Optional "needs": [], # Required dependencies AND "loop_limit": 1, "loop": [], "_ctx_vars": {}, } # Overrides # --------------- def prepare(self): self.catalog = self.parent.catalog def config_validate(self): # Fetch info from controller resources_kind = self.parent.available_resources_kinds loaded_resources = self.parent.loaded_resources # Check overrides if self.name in loaded_resources: _msg = f"User resource overrides: {self.name}" logger.debug(_msg) # Check for the kind kind = self.get_kind() if not kind in resources_kind: choices = ", ".join(resources_kind.keys()) _msg = ( f"Unknown resource kind: {self.name}, please choose one of: {choices}" ) raise error.UnknownResourceKind(_msg) def init(self): "Init each resources" self.name_raw = self.name ctx_vars = self.parent.ctx_vars self.name = format_render(self.name, vars=ctx_vars) # self.name = format_render(self.name, vars=self.catalog.context.get_vars()) # Append common base dependency on all resources if len(self.uses) == 0: self.uses.insert(0, "account:{user}") # self.uses = list(set(self.uses)) self.uses = uniq(self.uses) # Resolution management vars self._resolved = False self.resources_deps = None self.resources_missing = None # Helpers # --------------- def _parse_name(self): ret = self.name.split(":", 2) if len(ret) == 2: return ret[0], ret[1] # Raise error if name is invalid _msg = f"Invalid resource name: {self.name}, missing :" raise IamException(_msg) def get_kind(self): "Return resource kind" ret, _ = self._parse_name() return ret def get_name(self): "Return resource instance name" _, ret = self._parse_name() return ret # Methods # --------------- def require_resolved_deps(func): "Decorator that ensure resource resolution is called before method call" def wrap(self, *args, **kwargs): "Wrapper that ensure dependencies are resolved" if not self._resolved: self.resolve_deps(skip_missing=True) return func(self, *args, **kwargs) return wrap def resolve_deps(self, vars=None, add_self=True, cache=True, skip_missing=False): "Recursive uses dependency resolver" if cache and self.resources_deps is not None: return self.resources_deps # try: # ret, _missing = self._resolve_deps(vars=vars, add_self=add_self) # except error.UnresolvedResourceDependencies: # return None dependencies, missings = self._resolve_deps(vars=vars, add_self=add_self) self.resources_missing = missings self.resources_deps = dependencies self._is_active = True if len(missings) == 0 else False # Raise error on missing deps if not skip_missing and len(missings) > 0: pprint(self.catalog.resources.get()) _msg = f"Missing deps for {self.name}: {missings}" raise error.UnresolvedResourceDependencies(_msg) if cache: self._resolved_deps = dependencies self._resolved = True return dependencies def _resolve_deps(self, vars=None, add_self=True, _results=None, _missings=None): "Recursive dependency helper" # Recurisive control _missings = _missings or [] _results = _results or [] if self in _results: return _results # Get root context # ctx_vars = vars or self.catalog.context.get_vars() catalog = self.catalog ctx_vars = catalog.context.get_vars() # Check each dependency uses = self.uses for dep_name in uses: # Parse resource name if ctx_vars: dep_name = format_render(dep_name, ctx_vars) if dep_name == self.name: continue if dep_name in _missings: _missings.delete(dep_name) # Load child or quit if missing child = catalog.resources.find_closest(dep_name) if not child: _missings.append(dep_name) # _missings.append((self.name, dep_name, )) # _msg = f"Undeclared resource: {dep_name}" # raise error.UnresolvedResourceDependencies(_msg) continue # Avoid already processed resources if child and child in _results: continue _results.append(child) child._resolve_deps(vars=vars, _results=_results) if add_self: _results.append(self) return _results, _missings # Resolved methods # ================== @require_resolved_deps def is_active(self): "Return True if this resource is active" return self._is_active # return True if len(self.resources_missing) == 0 else False @require_resolved_deps def resolve_inputs(self, vars=None): "Resolve uses dependencies recursively" # WIPPPP, split this function in 2 !!! # self.catalog. # deps = self.resolve_deps() deps = self.resources_deps output = {} for res in deps: output.update(res.input) return output def loop_extend(self): "Extend loops from catalog" ctx_vars = self.catalog.context.get_vars() loops = self.loop loop_limit = self.loop_limit matches = [] stop = False for loop_search in loops: if len(matches) >= loop_limit: stop = True if not stop: loop_search = loop_search.format(**ctx_vars) res = self.catalog.resources.get(loop_search, skip_missing=True) if res: matches.append(res) return matches class Resources(DictCtrl): "Simple wrapper class to manage resources" items_class = Resource # Overrides # --------------- def prepare(self): # Save Catalog self.catalog = self.parent self.ctx_vars = self.catalog.context.get_vars() self.available_resources_kinds = self.catalog.resources_kind self.loaded_resources = [] # pprint (self.__dict__) # print ("INIT RES CATALOG") def find_closest(self, dep_name): "Find closest match on type" assert ":" in dep_name, f"We have unresolved query here: {dep_name}" catalog = self.catalog # Direct access child = catalog.resources.get(dep_name, None) if child: return child # Loop over resources target = dep_name.split(":")[0] scope = dep_name.split(":")[1] match = None while target: # Query other resources # query_name = f"{target}:{scope}" matches = catalog.resources.select("startswith", target) matches = { key: val for key, val in matches.items() if key.endswith(f":{scope}") } # Check closest matches if len(matches) == 1: match = list(matches.values())[0] # print ("Longer MATCH", match.name) logger.info( f"Closest resource of {dep_name} was mapped to: {match.name}" ) break elif len(matches) > 1: logger.debug( f"More than one closest resources for {dep_name}: {matches}" ) # raise Exception(f"Bug here, too many matches: {matches}") # Reduce again the common lookup pattern if "." in target: target = ".".join(target.split(".")[:-1]) else: break # print ("QUERY", target, scope) return match # view = scoped_ident(ident) # Catalog # ============================================================ # Context classes # ================ class Context(DictItem): "Class that hold a context" default_attrs = { "ident": None, } # Overrides # --------------- def config_validate(self): assert isinstance(self.ident, Ident), f"Gotr: {self.ident}" def init(self): self.catalog = self.parent root_dir = self.catalog.app.config_dir self._vars = { "ident": self.ident.name, "user": self.ident.name, "config_dir": root_dir, "scripts_dir": os.path.join(root_dir, "scripts"), "bin_dir": os.path.join(root_dir, "bin"), } def get_vars(self): "Return all vars context" return dict(self._vars) # Catalog classes # ================ class Catalog: "Manage catalog resources" def __init__(self, app, ident): # Prepare catalog self.app = app self.ident = ident # Prepare context ctx_conf = { "ident": ident, } self.context = Context("current_context", ctx_conf, parent=self) self._prepare_catalog() # Catalog building classes # ================ def _merge_inst_configs(self, merge=True): plugins_confs = self.app.providers.get_resource_configs() user_confs = self.ident.get_resource_configs() final_config = {} # Remap config names for source in [plugins_confs, user_confs]: # source = source or {} for name, conf in source.items(): # Check name if not ":" in name: ident_name = f"{name}:{self.ident.name}" # print (f"Renaming resource '{name}' to '{ident_name}'") name = ident_name # Check config existing = final_config.get(name, None) if existing: # print (f"Overrides: {name}") if merge: new_conf = dict(existing) new_conf.update(conf) conf = new_conf final_config[name] = conf return final_config def _prepare_catalog(self): "Prepare catalog resources configs" # Get resources from providers and user providers = self.app.providers ident = self.ident # Build kind catalog res_kind_configs = providers.get_resource_kinds() self.resources_kind = ResourcesKinds( "CatalogResourcesKinds", payload=res_kind_configs, parent=self ) # Build resources catalog res_inst_configs = self._merge_inst_configs() self.resources = Resources( "CatalogResources", payload=res_inst_configs, parent=self ) self._resources_config = res_inst_configs # Prepare services catalog services_confs = providers.get_services_configs() self.services = Services("CatalogServices", payload=services_confs, parent=self) def dump(self): print("Catalog settings") print("=" * 16) # pprint (self.__dict__) # return print("\n=== Catalog resources kinds:") pprint(self.resources_kind) print("\n=== Catalog resources configs:") pprint(self._resources_config) print("\n=== Catalog resources instances:") pprint(self.resources) print("\n=== Catalog services instances:") # for name, srv in self.services.items(): # pprint(srv.dump_item()) pprint({name: srv.dump_item() for name, srv in self.services.items()}) # Catalog interfaces # ================ # Catalog helpers # ================ # def resolve_service_order(self, reverse=False): # "Resolve service order" # return self.services.get_loading_order(reverse=reverse) # Shell helpers # ================ def shell_enable(self, run_start=True): "Enable shell" actions = ["shell_enable"] if run_start: actions.insert(0, "shell_start") # logger.warning("Shell Start") # logger.warning("Shell Enable") return self._shell_action(actions) def shell_disable(self, run_stop=False): "Disable shell" actions = ["shell_disable"] if run_stop: actions.insert(0, "shell_stop") # cli_logger.info("Shell Stop") # cli_logger.info("Shell Disable") return self._shell_action(actions) def _shell_action(self, action_names, reverse=False): "Run command order" ident = self.ident vars = self.context.get_vars() # Fetch execution order order, services = self.services.get_loading_order(reverse=reverse) # Prepare context log_action_name = "Enable" if reverse: log_action_name = "Disable" cli_logger.info(f"Identity {ident.name} action: {','.join(action_names)}") # Execute on each plugins commands output_code = [] for srv_name in order: service = services[srv_name].svc res = services[srv_name].res # Load service command for action_name in action_names: command = service.commands.get(action_name, None) if not command: continue # Build loop with at least ONE item cmd_shell = command.shell if not cmd_shell: continue # Create context var dict ctx_vars = dict() ctx_vars.update(service.input) ctx_vars.update(vars) ctx_vars.update(res.resolve_inputs(vars=ctx_vars)) loops = res.loop_extend() or [res] res_vars = [x.resolve_inputs(vars=ctx_vars) for x in loops] ctx_vars.update( { "item": res_vars[0], "loop": res_vars, } ) logger.debug(f"{log_action_name} service: {res.name}") # pprint (ctx_vars) cmd = jinja_render(cmd_shell, ctx_vars) output_code.append( f"# Loading: {action_name} {res.name} ({service.name})" ) output_code.append(f"# =====================") output_code.append(cmd) output_code.append("\n") return "\n".join(output_code) def run_svc_cmd(self, cmd): "Run a command against the services" cmd, args = self.services.get_svc_command(cmd) # pprint (cmd.cmd) # pprint (args) service = cmd.service # pprint (service) # pprint (service.__dict__) # pprint (dir(service)) res = service.get_linked_resource() # pprint (res) vars = self.context.get_vars() ctx_vars = dict() ctx_vars.update(vars) ctx_vars.update(service.inputs) # ctx_vars.update(res.resolve_inputs(vars=ctx_vars)) # loops = res.loop_extend() or [res] # res_vars = [x.resolve_inputs(vars=ctx_vars) for x in loops] # ctx_vars.update( # { # "item": res_vars[0], # "loop": res_vars, # } # ) # pprint (ctx_vars) new_env = dict(os.environ) new_env.update(ctx_vars) # pprint(new_env) if cmd.cmd and cmd.shell: logger.warning(f"Duplicate cmd and shell for {service.name}") out = None if cmd.shell: mode = "shell" payload = cmd.shell # Scan for missing vars !!! tmp = jinja_template_vars(payload) prompt_vars = [] for var_name in tmp: if var_name not in ctx_vars: prompt_vars.append(var_name) else: var_value = ctx_vars[var_name] if empty(var_value): prompt_vars.append(var_name) # Ask for missing vars if len(prompt_vars) > 0: # print ("MISSING VARS") # pprint (prompt_vars) answered_items = {} for missed in prompt_vars: default = ctx_vars.get(missed, None) result = click.prompt(f"Select value for {missed}", default=default) answered_items[missed] = result ctx_vars.update(answered_items) real_cmd = jinja_render(payload, ctx_vars) # print ("RUN SHELL") builtins = sh.bash.bake("-c") out = builtins(real_cmd, _fg=True, _env=new_env) # print (out) elif cmd.cmd: mode = "cmd" payload = cmd.cmd real_cmd = jinja_render(payload, ctx_vars) cmd_parts = real_cmd.split(" ", 1) sh_cmd = cmd_parts[0] sh_args = cmd_parts[1] if len(cmd_parts) > 0 else [] # print ("Prepare", sh_cmd, " ||| ", sh_args) # proc = sh.bake(sh_cmd) # sh(sh_cmd, *sh_args, _fg=True) cmd = sh.Command(sh_cmd) if sh_args: out = cmd(sh_args, _fg=True, _env=new_env) else: out = cmd(_fg=True, _env=new_env) # print (out) else: raise Exception("MIssing cmd or shell in config !") return out # print ("RUN CMD", mode, "\n\n\n====\n", real_cmd)