From e7c63300206264c02737471fde85d2f83160fffa Mon Sep 17 00:00:00 2001 From: Robin Pierre Cordier Date: Wed, 4 May 2022 19:51:00 -0400 Subject: [PATCH] Change: Completely refactor inventory plugin --- plugins/inventory/kheops.py | 33 +- plugins/plugin_utils/common.py | 660 ++++++++++++++++++++------------- 2 files changed, 412 insertions(+), 281 deletions(-) diff --git a/plugins/inventory/kheops.py b/plugins/inventory/kheops.py index 43f929a..f786158 100644 --- a/plugins/inventory/kheops.py +++ b/plugins/inventory/kheops.py @@ -8,6 +8,9 @@ from __future__ import (absolute_import, division, print_function) # __metaclass__ = type +from pyinstrument import Profiler +profiler = Profiler() + from ansible import constants as C from ansible.errors import AnsibleError from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable, Constructable @@ -79,6 +82,9 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable): config_data = self._read_config_data(path) self._consume_options(config_data) + + #profiler.start() + # Get options from inventory self.jinja2_native = self.get_option('jinja2_native') self.strict = self.get_option('strict') @@ -88,15 +94,8 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable): self.keyed_groups = self.get_option('keyed_groups') # Prepare Kheops instance - self.config_file = self.get_option('config') - ansible_config = { - "instance_namespace": self.get_option('instance_namespace'), - "instance_log_level": self.get_option('instance_log_level'), - "instance_explain": self.get_option('instance_explain'), - } configs = [ - ansible_config, - self.config_file, + self.get_option('config'), path, ] self.kheops = AnsibleKheops(configs=configs, display=self.display) @@ -109,21 +108,29 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable): self.display.error(f"Got errors while processing Kheops lookup for host: %s, %s" % (host_name, err)) raise err + #profiler.stop() + #profiler.print() + #profiler.open_in_browser() def _populate_host(self, host_name): host = self.inventory.get_host(host_name) - try: - ret = self.kheops.super_lookup( + ret = self.kheops.super_lookup_seq( keys=None, - scope=None, + scope_vars=host.get_vars(), _templar=self.templar, - _variables=host.get_vars(), jinja2_native=self.jinja2_native, ) + + # Aggregate results + aggregated = {} + for res in ret: + aggregated.update(dict(res)) + ret = aggregated + except AnsibleError as err: - self.display.error (f"Could lookup Kheops data for host: {host_name}") + self.display.error (f"Could lookup Kheops data for host: {host_name}, {err}") raise err # Inject variables into host diff --git a/plugins/plugin_utils/common.py b/plugins/plugin_utils/common.py index 010c8fd..857d44f 100644 --- a/plugins/plugin_utils/common.py +++ b/plugins/plugin_utils/common.py @@ -7,6 +7,7 @@ from typing import Any, Union from copy import deepcopy import yaml +from diskcache import Cache from ansible.errors import AnsibleError, AnsibleUndefinedVariable from ansible.module_utils.common.text.converters import to_native from ansible.utils.display import Display @@ -36,29 +37,75 @@ DOCUMENTATION_OPTION_FRAGMENT = """ env: - name: ANSIBLE_KHEOPS_CONFIG + + # Query configuration + # ========================== + namespace: + description: + - The Kheops namespace to use + default: 'default' + env: + - name: ANSIBLE_KHEOPS_DEFAULT_NAMESPACE + scope: + description: + - A hash containing the scope to use for the request, the values will be resolved as Ansible facts. + - Use a dot notation to dig deeper into nested hash facts. + default: + node: inventory_hostname + groups: group_names + + keys: + description: + - A list of keys to lookup + default: Null + + + # Behavior configuration + # ========================== + process_scope: + description: + - This setting defines how is parsed the `scope` configuration + - Set `vars` to enable simple variable interpolation + - Set `jinja` to enable jinja string interpolation + default: 'jinja' + choices: ['vars', 'jinja'] + + process_results: + description: + - This setting defines how is parsed the returned results. + - Set `none` to disable jinja interpolation from result. + - Set `jinja` to enable jinja result interpolation. + - Using jinja may pose some security issues, as you need to be sure that your source of data is properly secured. + default: 'none' + choices: ['none', 'jinja'] + + jinja2_native: + description: + - Controls whether to use Jinja2 native types. + - It is off by default even if global jinja2_native is True. + - Has no effect if global jinja2_native is False. + - This offers more flexibility than the template module which does not use Jinja2 native types at all. + - Mutually exclusive with the convert_data option. + default: False + type: bool + env: + - name: ANSIBLE_JINJA2_NATIVE + notes: + - Kheops documentation is available on http://kheops.io/ + - You can add more parameters as documented in http://kheops.io/server/api + + + # Backend configuration (Direct/Client) + # ========================== mode: description: - - Choose `client` to use a remote Khéops instance + - Choose `client` to use a remote Khéops instance (Not implemented yet) - Choose `instance` to use a Khéops directly default: 'instance' - choices: ['instance', 'client'] + choices: ['instance'] env: - name: ANSIBLE_KHEOPS_MODE - # default_namespace: - # description: - # - The Kheops namespace to use - # default: 'default' - # env: - # - name: ANSIBLE_KHEOPS_DEFAULT_NAMESPACE - - # default_scope: - # description: - # - A list of default variables to inject in scope. - # default: 'default' - # env: - # - name: ANSIBLE_KHEOPS_DEFAULT_SCOPE - # Instance configuration (Direct) # ========================== @@ -131,63 +178,6 @@ DOCUMENTATION_OPTION_FRAGMENT = """ # turfu env: # turfu - name: ANSIBLE_KHEOPS_VALIDATE_CERTS - - # Query configuration - # ========================== - namespace: - description: - - The Kheops namespace to use - default: 'default' - env: - - name: ANSIBLE_KHEOPS_DEFAULT_NAMESPACE - scope: - description: - - A hash containing the scope to use for the request, the values will be resolved as Ansible facts. - - Use a dot notation to dig deeper into nested hash facts. - default: - node: inventory_hostname - groups: group_names - - keys: - description: - - A list of keys to lookup - default: Null - - - # Behavior configuration - # ========================== - process_scope: - description: - - This setting defines how is parsed the `scope` configuration - - Set `vars` to enable simple variable interpolation - - Set `jinja` to enable jinja string interpolation - default: 'jinja' - choices: ['vars', 'jinja'] - - process_results: - description: - - This setting defines how is parsed the returned results. - - Set `none` to disable jinja interpolation from result. - - Set `jinja` to enable jinja result interpolation. - - Using jinja may pose some security issues, as you need to be sure that your source of data is properly secured. - default: 'none' - choices: ['none', 'jinja'] - - jinja2_native: - description: - - Controls whether to use Jinja2 native types. - - It is off by default even if global jinja2_native is True. - - Has no effect if global jinja2_native is False. - - This offers more flexibility than the template module which does not use Jinja2 native types at all. - - Mutually exclusive with the convert_data option. - default: False - type: bool - env: - - name: ANSIBLE_JINJA2_NATIVE - notes: - - Kheops documentation is available on http://kheops.io/ - - You can add more parameters as documented in http://kheops.io/server/api - """ @@ -195,72 +185,230 @@ KEY_NS_SEP = "/" if USE_JINJA2_NATIVE: from ansible.utils.native_jinja import NativeJinjaText +KHEOPS_CACHE=Cache("/tmp/ansible_kheops_cache/") +KHEOPS_CACHE.clear() + + +class Key(): + """Key instance class""" + default_config = { + "key": None, + "remap": None, + "scope": {}, + "namespace": None, + "namespace_prefix": False, + "explain": False, + "trace": False, + } + + def __init__(self, config): + + self._config = self.parse_config(config) + + # Create all attributes + for key, value in self._config.items(): + setattr(self, key, value) + + def parse_config(self, key_def): + """ + Parse a key configuration and return an object + + A key configuration can either be: + - a string: for quick queries + - a dict: for full control, that can be extended later for more options + + String configuration: + - + - : + - :: + + Dict configuration: + - key: key to query + - namespace: namespace to query + - remap: rename key to ansible variable + """ + + ret = dict(self.default_config) + + if isinstance(key_def, dict): + ret.update(key_def) + + elif isinstance(key_def, str): + + key = None + remap = key + namespace = self.default_namespace + + # Extract config from string + parts = key_def.split(KEY_NS_SEP, 3) + if len(parts) == 1: + key = parts[0] + elif len(parts) > 1: + namespace = parts[0] + key = parts[1] + + if len(parts) == 3: + remap = parts[2] + + # Generate new config + string_conf = { + "key": key, + "namespace": namespace, + "remap": remap, + } + ret.update(string_conf) + + else: + raise Exception(f"Key configuration is invalid, expected a string or dict, got: {key_def}") + + return ret -@dataclass -class Key: - key: str - remap: Union[str, type(None)] - namespace: Union[type(None), str] def show(self): + """Method to show the accepatable string format for Kheops""" ret = self.key if self.namespace is not None: ret = f"{self.namespace}{KEY_NS_SEP}{ret}" return ret +class Keys(): + """Keys config instance class""" + + def __init__(self, config, default_namespace="default"): + + self.raw_config = config + self.default_namespace = default_namespace + + self.key_list = self.parse_config(config) + + def parse_config(self, keys_def, default_namespace="default"): + """Parse a key config structure + + A keydef can either be: + - a string: Single lookup + - a list: ordered lookup + - a dict: unordered lookup + """ + + ret = [] + if isinstance(keys_def, list): + for key_def in keys_def: + ret.append(Key(key_def)) + + elif isinstance(keys_def, str): + ret.append(Key(keys_def)) + + elif isinstance(keys_def, dict): + for key, value in keys_def.items(): + if isinstance(value, str): + key_def = value or key + + if isinstance(value, dict): + key_def = value or {} + key_def['key'] = key_def.get('key', key) + + ret.append(Key(key_def)) + else: + raise AnsibleError(f"Unable to process Kheops keys: {ret}") + + if len(ret) == 0: + raise AnsibleError(f"Kheops query needs at least one key to lookup") + + return ret + + class AnsibleKheops: """Main Ansible Kheops Class""" - def __init__(self, configs=None, display=None): + # Extract default config from documentation + default_config = { + key: value.get("default", None) + for key, value in yaml.safe_load( + DOCUMENTATION_OPTION_FRAGMENT).items() + } - self.configs = configs or [] + # Kheops instance management + # ---------------------------- + + def __init__(self, configs=None, display=None): self.display = display or Display() - config = self.get_config() + configs = configs or [] + self.config = self.parse_configs(configs) + self.display.v("Kheops instance has been created, with config: %s" % self.config['instance_config']) # Instanciate Kheops - if config["mode"] == "instance": + if self.config["mode"] == "instance": # Configure logging logger = logging.getLogger("kheops") - logger.setLevel(config["instance_log_level"]) + logger.setLevel(self.config["instance_log_level"]) # See for logging: https://medium.com/opsops/debugging-requests-1989797736cc - class ListLoggerHandler(logging.Handler): - def emit(self, record): - msg = self.format(record) + #class ListLoggerHandler(logging.Handler): + # def emit(self, record): + # msg = self.format(record) main_logger = logging.getLogger() - main_logger.addHandler(ListLoggerHandler()) + #main_logger.addHandler(ListLoggerHandler()) main_logger.setLevel(logging.DEBUG) # Start instance self.kheops = Kheops( - config=config["instance_config"], namespace=config["instance_namespace"] + config=self.config["instance_config"], + namespace=self.config["instance_namespace"], + cache=KHEOPS_CACHE ) - elif config["mode"] == "client": + elif self.config["mode"] == "client": raise AnsibleError("Kheops client mode is not implemented") - self.config = config - self.display.v("Kheops instance has been created, with config: %s" % config['instance_config']) - def get_config(self): + def parse_configs(self, configs): """ - Processing order: - - Fetch the value of config or fallback on ANSIBLE_KHEOPS_CONFIG - - Load the config if any - - Overrides with other options + Take a list of config items that will be merged together. + + A config item can either be: + - A string: Represents the path of the config + - A dict: Represent a direct configuration + - None: Just ignore this entry + + #Processing order: + #- Fetch the value of config or fallback on ANSIBLE_KHEOPS_CONFIG + #- Load the config if any + #- Overrides with other options """ - # Extract default value from doc - data_doc = yaml.safe_load(DOCUMENTATION_OPTION_FRAGMENT) - default_config = { - key: value.get("default", None) for key, value in data_doc.items() - } + # Get default configuration from environment + env_items = [ + # We exclude 'config' on purpose + "mode", + "default_namespace", + + "instance_config", + "instance_namespace", + "instance_log_level", + "instance_log_explain", + + # Future: + # "token", + # "host", + # "port", + # "protocol", + # "validate_certs", + ] + env_config = {} + for item in env_items: + envvar = "ANSIBLE_KHEOPS_" + item.upper() + try: + env_config[item] = os.environ[envvar] + except KeyError: + pass + + # Merge/process runtime configurations merged_configs = {} - for config in self.configs: + for config in configs: conf_data = None if isinstance(config, str): @@ -277,207 +425,183 @@ class AnsibleKheops: elif isinstance(config, type(None)): continue else: - assert False, f"Bad config for: {config}" + assert False, f"Bad config for, expected a path or a dict, got type {type(config)}: {config}" assert isinstance(conf_data, dict), f"Bug with conf_data: {conf_data}" if isinstance(conf_data, dict): merged_configs.update(conf_data) - # Get environment config - items = [ - # We exclude 'config' - "mode", - "instance_config", - "instance_namespace", - "instance_log_level", - "namespace", - "scope", - "keys", - ] - env_config = {} - for item in items: - envvar = "ANSIBLE_KHEOPS_" + item.upper() - try: - env_config[item] = os.environ[envvar] - except KeyError: - pass - # Merge results - combined_config = {} - combined_config.update(default_config) + combined_config = dict(self.default_config) combined_config.update(env_config) combined_config.update(merged_configs) return combined_config - @staticmethod - def parse_string(item, default_namespace): - key = None - remap = key - namespace = default_namespace - if isinstance(item, str): + # Lookup methods + # ---------------------------- - parts = item.split(KEY_NS_SEP, 3) - if len(parts) > 0: - key = parts[0] - if len(parts) > 1: - namespace = parts[0] - key = parts[1] - if len(parts) > 2: - remap = parts[2] - elif isinstance(item, dict): - key = item.get("key") - remap = item.get("remap", key) - namespace = item.get("namespace", namespace) + def super_lookup_seq( + self, + keys, + scope_config=None, + scope_vars=None, + namespace=None, + kwargs=None, - return Key(key=key, remap=remap, namespace=namespace) + _templar=None, + _process_scope=None, + _process_results=None, - @classmethod - def parse_keys(self, data, namespace): - keys = [] - if isinstance(data, str): - keys.append(self.parse_string(data, namespace)) - - elif isinstance(data, list): - for item in data: - keys.append(self.parse_string(item, namespace)) - elif isinstance(data, dict): - for key, value in data.items(): - item = key - if value: - assert isinstance(value, str), f"Need a string, got: {value}" - item = f"{key}{KEY_NS_SEP}{value}" - - keys.append(self.parse_string(item, namespace)) - - else: - raise AnsibleError(f"Unable to process Kheops keys: {keys}") - - return keys - - def get_scope_from_host_inventory(self, host_vars, scope=None): + jinja2_native=False, + ): """ - Build scope from host vars + Sequential Lookup method wrapper """ - scope = scope or self.config["scope"] + + # Determine runtime options + _process_scope = _process_scope if _process_scope in ["vars", "jinja"] else self.config["process_scope"] + _process_results = _process_results if isinstance(_process_results, bool ) else self.config["process_results"] + _reinject_results = True + + # Build query context + namespace = namespace or self.config.get('default_namespace') + scope_config = scope_config or self.config.get('scope') + + keys = keys or self.config.get('keys', []) + keys_config = Keys(keys, default_namespace=namespace) + + scope_final = self.get_scope( + _process_scope, + scope_vars, scope_config, + _templar=_templar, jinja2_native=jinja2_native + ) + + # Query kheops + result = [] ret = {} - for key, val in scope.items(): - # Tofix should this fail silently ? - ret[key] = host_vars.get(val, None) + for key in keys_config.key_list: - return ret + if _reinject_results: + scope_vars.update(dict(ret)) + scope_final = self.get_scope( + _process_scope, + scope_vars, scope_config, + _templar=_templar, jinja2_native=jinja2_native + ) - def get_scope_from_jinja(self, host_vars, templar, scope=None, jinja2_native=False): + # Query kheops + ret = self.kheops.lookup( + keys=[key.show()], + + scope=scope_final, + trace=key.trace, + explain=key.explain, + namespace_prefix=key.namespace_prefix + ) + + # Remap output + if key.remap is not None and key.remap != key.key: + #print (f"REMAP KEY: {key.key} => {key.remap}") + ret[key.remap] = ret[key.key] + del ret[key.key] + + # Process output + if _process_results == "jinja": + with _templar.set_temporary_context(available_variables=scope_vars): + ret = _templar.template( + ret, + preserve_trailing_newlines=True, + convert_data=False, + escape_backslashes=False, + ) + if USE_JINJA2_NATIVE and not jinja2_native: + ret = NativeJinjaText(ret) + + result.append(ret) + + return result + + # Scope management methods + # ---------------------------- + + def get_scope(self, method, scope_vars, scope_config, _templar=None, jinja2_native=False): """ - Parse in jinja a dict scope + Simple wrapper for _get_scope_template and _get_scope_vars """ - scope = scope or self.config["scope"] + + ret = {} + if method == "jinja": + ret = self._get_scope_template(scope_vars, scope_config, + _templar, jinja2_native=jinja2_native) + elif method == "vars": + ret = self._get_scope_vars(scope_vars, scope_config) + elif method == "all": + ret = scope_vars + else: + raise AnsibleError(f"Get_scope only accept 'jinja','vars' or 'all', got: {method}") + + #print ("===> Scope Config:", method) + #print (scope_config) + #print ("===> Scope Vars:", scope_vars.get('tiger_profiles', 'MISSING_PROFILE')) + #print (scope_vars.keys()) + #print ("===> Scope Result:") + #print (ret) + #print ("===> Scope EOF") + + return dict(ret) + + def _get_scope_template(self, data, scope_config, templar, jinja2_native=False): + """ + Create a scope context from data, jinja2 interpolation + """ + scope_config = scope_config or self.config.get('scope', {}) + assert isinstance(data, dict) + assert templar if USE_JINJA2_NATIVE and not jinja2_native: _templar = templar.copy_with_new_env(environment_class=AnsibleEnvironment) else: _templar = templar - _vars = deepcopy(host_vars) + # Process template with jinja + _vars = dict(data) ret = {} with _templar.set_temporary_context(available_variables=_vars): - for key, value in scope.items(): - res = value - try: - res = _templar.template( - value, - preserve_trailing_newlines=True, - convert_data=True, - escape_backslashes=False, - ) - if USE_JINJA2_NATIVE and not jinja2_native: - # jinja2_native is true globally but off for the lookup, we need this text - # not to be processed by literal_eval anywhere in Ansible - res = NativeJinjaText(res) - self.display.vvv(f"Transformed scope value: {value} => {res}") - except AnsibleUndefinedVariable as err: - self.display.error(f"Got templating error for string '{value}': {err}") - raise err - - ret[key] = res - - return ret - - def lookup(self, keys, namespace=None, scope=None, explain=None): - """ - Start a lookup query - """ - - if explain is None: - explain = self.config["instance_explain"] - - namespace = namespace or self.config["namespace"] - scope = scope or self.config["scope"] - keys = keys or self.config["keys"] - keys_config = self.parse_keys(keys, namespace) - keys = [i.show() for i in keys_config] - - self.display.v(f"Kheops keys: {keys}") - self.display.vv(f"Kheops scope: {scope}") - - ret = self.kheops.lookup( - keys=keys, - scope=scope, - # trace=True, - explain=explain, - namespace_prefix=False - ) - - # Remap output - for key in keys_config: - if key.remap is not None and key.remap != key.key: - ret[key.remap] = ret[key.key] - del ret[key.key] - - return ret or {} - - def super_lookup( - self, - keys, - namespace=None, - scope=None, - kwargs=None, - _templar=None, - _variables=None, - _process_scope=None, - _process_results=None, - jinja2_native=False, - ): - """ - Lookup method wrapper - """ - - _process_scope = _process_scope or self.config["process_scope"] - _process_results = _process_results or self.config["process_results"] - - scope = scope or self.config["scope"] - if _process_scope == "vars": - scope = self.get_scope_from_host_inventory(_variables, scope=scope) - elif _process_scope == "jinja": - assert _templar, f"BUG: We expected a templar object here, got: {_templar}" - scope = self.get_scope_from_jinja( - _variables, _templar, scope=scope, jinja2_native=jinja2_native - ) - - ret = self.lookup(keys, namespace=namespace, scope=scope) - - if _process_results == "jinja": - with _templar.set_temporary_context(available_variables=_variables): + try: ret = _templar.template( - ret, + scope_config, preserve_trailing_newlines=True, - convert_data=False, + convert_data=True, escape_backslashes=False, ) - if USE_JINJA2_NATIVE and not jinja2_native: - ret = NativeJinjaText(ret) + if USE_JINJA2_NATIVE and not jinja2_native: + # jinja2_native is true globally but off for the lookup, we need this text + # not to be processed by literal_eval anywhere in Ansible + ret = NativeJinjaText(ret) + self.display.vvv(f"Transformed scope value: {scope_config} => {ret}") + + except AnsibleUndefinedVariable as err: + self.display.error(f"Got templating error for string '{scope_config}': {err}") + raise err return ret + + + def _get_scope_vars(self, data, scope_config): + """ + Create a scope context from data, simple interpolation + """ + + scope_config = scope_config or self.config.get('scope', {}) + assert isinstance(data, dict) + + ret = {} + for key, val in scope_config.items(): + ret[key] = data.get(val, None) + + return ret +