plugins/inventory/kheops.py

@@ -8,9 +8,6 @@
 from __future__ import (absolute_import, division, print_function)
 # __metaclass__ = type
 
-from pyinstrument import Profiler
-profiler = Profiler()
-
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable, Constructable
@@ -82,9 +79,6 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable):
         config_data = self._read_config_data(path)
         self._consume_options(config_data)
 
-
-        #profiler.start()
-
         # Get options from inventory
         self.jinja2_native = self.get_option('jinja2_native')
         self.strict = self.get_option('strict')
@@ -94,8 +88,15 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable):
         self.keyed_groups = self.get_option('keyed_groups')
 
         # Prepare Kheops instance
+        self.config_file = self.get_option('config')
+        ansible_config = {
+                "instance_namespace": self.get_option('instance_namespace'),
+                "instance_log_level": self.get_option('instance_log_level'),
+                "instance_explain": self.get_option('instance_explain'),
+                }
         configs = [
-            self.get_option('config'),
+            ansible_config,
+            self.config_file,
             path,
             ]
         self.kheops = AnsibleKheops(configs=configs, display=self.display)
@@ -108,29 +109,21 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable):
                 self.display.error(f"Got errors while processing Kheops lookup for host: %s, %s" % (host_name, err))
                 raise err
 
-        #profiler.stop()
-        #profiler.print()
-        #profiler.open_in_browser()
 
     def _populate_host(self, host_name):
 
             host = self.inventory.get_host(host_name)
+
             try:
-                ret = self.kheops.super_lookup_seq(
+                ret = self.kheops.super_lookup(
                     keys=None,
-                    scope_vars=host.get_vars(),
+                    scope=None,
                     _templar=self.templar,
+                    _variables=host.get_vars(),
                     jinja2_native=self.jinja2_native,
                 )
-
-                # Aggregate results
-                aggregated = {}
-                for res in ret:
-                    aggregated.update(dict(res))
-                ret = aggregated
-
             except AnsibleError as err:
-                self.display.error (f"Could lookup Kheops data for host: {host_name}, {err}")
+                self.display.error (f"Could lookup Kheops data for host: {host_name}")
                 raise err
 
             # Inject variables into host

plugins/lookup/kheops.py

@@ -87,8 +87,6 @@ from pprint import pprint
 
 # Entry point for Ansible starts here with the LookupModule class
 class LookupModule(LookupBase):
-
-
     def run(self, terms, variables=None, scope=None, **kwargs):
 
         self.set_options(direct=kwargs)
@@ -100,8 +98,9 @@ class LookupModule(LookupBase):
 
 
         # Prepare Kheops instance
+        self.config_file = self.get_option('config')
         configs = [
-          self.get_option('config'),
+          self.config_file,
           kwargs,
           #{
           #    "instance_log_level": 'DEBUG',
@@ -116,63 +115,36 @@ class LookupModule(LookupBase):
         else:
             templar = self._templar
 
+        # Create scope
+        if process_scope == 'vars':
+            scope = kheops.get_scope_from_host_inventory(variables, scope=None)
+        elif process_scope == 'jinja':
+            scope = kheops.get_scope_from_jinja(variables, self._templar, scope=None)
+
         # Transform dict to list for lookup/queries
         ret = []
-        #for term in terms:
+        for term in terms:
+            result = kheops.super_lookup(
+                  keys=term,
+                  scope=scope,
+                  _variables=variables,
+                  _templar=templar,
+              )
 
-        results = kheops.super_lookup_seq(
+            # Render data with Templar
-              keys=terms,
+            #if process_results == 'jinja':
-              #scope=scope,
+            #    with templar.set_temporary_context(available_variables=variables):
-              scope_vars=variables,
+            #        result = templar.template(result,
-              _templar=templar,
+            #                    preserve_trailing_newlines=True,
-              jinja2_native=jinja2_native,
+            #                    convert_data=False, escape_backslashes=False)
-              _process_results='jinja',
-          )
 
-        final = []
+            #    if USE_JINJA2_NATIVE and not jinja2_native:
-        for result in results:
+            #        # jinja2_native is true globally but off for the lookup, we need this text
-            only_key = list(result.keys())
+            #        # not to be processed by literal_eval anywhere in Ansible
-            if len(only_key) > 0:
+            #        result = NativeJinjaText(result)
-                final.append(result[only_key[0]])
 
-        return final
+            # Return result
-
+            subkey = list(result.keys())[0]
-        try:
+            ret.append(result[subkey])
-            found_term = list(result[0].keys())
-            if len(found_term) > 0:
-                return result[0][found_term[0]]
-        except Exception as err:
-            print (err)
-        return result[0]
-
-
-        return None
-
-        # Merge results if any
-        #from pprint import pprint
-        #pprint (result)
-        final = None
-        if len(result) > 0:
-            final_keys = list(result[0].keys())
-            #print ("Final key", final_keys)
-            if len(final_keys) > 0:
-                #print ("yeahhhh", final_keys[0], result[0][final_keys[0]])
-                final = result[0][final_keys[0]]
-            if len(final_keys) > 1:
-                print ("Warning, other results are ignored !!!")
-
-        return final
-
-
-        #final = result[0].keys()[0] if len(result) > 0 else None
-
-        #final = {}
-        #for res in result:
-        #    pprint (res)
-        #    final.update(res)
-
-        #final = result
-
-        ret.append(final)
 
         return ret

plugins/plugin_utils/common.py

@@ -7,7 +7,6 @@ from typing import Any, Union
 from copy import deepcopy
 import yaml
 
-from diskcache import Cache
 from ansible.errors import AnsibleError, AnsibleUndefinedVariable
 from ansible.module_utils.common.text.converters import to_native
 from ansible.utils.display import Display
@@ -37,75 +36,29 @@ DOCUMENTATION_OPTION_FRAGMENT = """
         env:
             - name: ANSIBLE_KHEOPS_CONFIG
 
-
-      # Query configuration
-      # ==========================
-      namespace:
-        description:
-          - The Kheops namespace to use
-        default: 'default'
-        env:
-            - name: ANSIBLE_KHEOPS_DEFAULT_NAMESPACE
-      scope:
-        description:
-          - A hash containing the scope to use for the request, the values will be resolved as Ansible facts.
-          - Use a dot notation to dig deeper into nested hash facts.
-        default:
-          node: inventory_hostname
-          groups: group_names
-
-      keys:
-        description:
-          - A list of keys to lookup
-        default: Null
-
-
-      # Behavior configuration
-      # ==========================
-      process_scope:
-        description:
-          - This setting defines how is parsed the `scope` configuration
-          - Set `vars` to enable simple variable interpolation
-          - Set `jinja` to enable jinja string interpolation
-        default: 'jinja'
-        choices: ['vars', 'jinja']
-
-      process_results:
-        description:
-          - This setting defines how is parsed the returned results.
-          - Set `none` to disable jinja interpolation from result.
-          - Set `jinja` to enable jinja result interpolation.
-          - Using jinja may pose some security issues, as you need to be sure that your source of data is properly secured.
-        default: 'none'
-        choices: ['none', 'jinja']
-
-      jinja2_native:
-        description:
-            - Controls whether to use Jinja2 native types.
-            - It is off by default even if global jinja2_native is True.
-            - Has no effect if global jinja2_native is False.
-            - This offers more flexibility than the template module which does not use Jinja2 native types at all.
-            - Mutually exclusive with the convert_data option.
-        default: False
-        type: bool
-        env:
-            - name: ANSIBLE_JINJA2_NATIVE
-        notes:
-          - Kheops documentation is available on http://kheops.io/
-          - You can add more parameters as documented in http://kheops.io/server/api
-
-
-      # Backend configuration (Direct/Client)
-      # ==========================
       mode:
         description:
-          - Choose `client` to use a remote Khéops instance (Not implemented yet)
+          - Choose `client` to use a remote Khéops instance
           - Choose `instance` to use a Khéops directly
         default: 'instance'
-        choices: ['instance']
+        choices: ['instance', 'client']
         env:
             - name: ANSIBLE_KHEOPS_MODE
 
+      # default_namespace:
+      #   description:
+      #     - The Kheops namespace to use
+      #   default: 'default'
+      #   env:
+      #       - name: ANSIBLE_KHEOPS_DEFAULT_NAMESPACE
+
+      # default_scope:
+      #   description:
+      #     - A list of default variables to inject in scope.
+      #   default: 'default'
+      #   env:
+      #       - name: ANSIBLE_KHEOPS_DEFAULT_SCOPE
+
 
       # Instance configuration (Direct)
       # ==========================
@@ -178,6 +131,63 @@ DOCUMENTATION_OPTION_FRAGMENT = """
       # turfu   env:
       # turfu       - name: ANSIBLE_KHEOPS_VALIDATE_CERTS
 
+
+      # Query configuration
+      # ==========================
+      namespace:
+        description:
+          - The Kheops namespace to use
+        default: 'default'
+        env:
+            - name: ANSIBLE_KHEOPS_DEFAULT_NAMESPACE
+      scope:
+        description:
+          - A hash containing the scope to use for the request, the values will be resolved as Ansible facts.
+          - Use a dot notation to dig deeper into nested hash facts.
+        default:
+          node: inventory_hostname
+          groups: group_names
+
+      keys:
+        description:
+          - A list of keys to lookup
+        default: Null
+
+
+      # Behavior configuration
+      # ==========================
+      process_scope:
+        description:
+          - This setting defines how is parsed the `scope` configuration
+          - Set `vars` to enable simple variable interpolation
+          - Set `jinja` to enable jinja string interpolation
+        default: 'jinja'
+        choices: ['vars', 'jinja']
+
+      process_results:
+        description:
+          - This setting defines how is parsed the returned results.
+          - Set `none` to disable jinja interpolation from result.
+          - Set `jinja` to enable jinja result interpolation.
+          - Using jinja may pose some security issues, as you need to be sure that your source of data is properly secured.
+        default: 'none'
+        choices: ['none', 'jinja']
+
+      jinja2_native:
+        description:
+            - Controls whether to use Jinja2 native types.
+            - It is off by default even if global jinja2_native is True.
+            - Has no effect if global jinja2_native is False.
+            - This offers more flexibility than the template module which does not use Jinja2 native types at all.
+            - Mutually exclusive with the convert_data option.
+        default: False
+        type: bool
+        env:
+            - name: ANSIBLE_JINJA2_NATIVE
+        notes:
+          - Kheops documentation is available on http://kheops.io/
+          - You can add more parameters as documented in http://kheops.io/server/api
+
 """
 
 
@@ -185,230 +195,72 @@ KEY_NS_SEP = "/"
 if USE_JINJA2_NATIVE:
     from ansible.utils.native_jinja import NativeJinjaText
 
-KHEOPS_CACHE=Cache("/tmp/ansible_kheops_cache/")
-KHEOPS_CACHE.clear()
-
-
-class Key():
-    """Key instance class"""
-    default_config = {
-            "key": None,
-            "remap": None,
-            "scope": {},
-            "namespace": None,
-            "namespace_prefix": False,
-            "explain": False,
-            "trace": False,
-            }
-
-    def __init__(self, config):
-
-        self._config = self.parse_config(config)
-
-        # Create all attributes
-        for key, value in self._config.items():
-            setattr(self, key, value)
-
-    def parse_config(self, key_def):
-        """
-        Parse a key configuration and return an object
-
-        A key configuration can either be:
-        - a string: for quick queries
-        - a dict: for full control, that can be extended later for more options
-
-        String configuration:
-        - <KEY>
-        - <NAMESPACE>:<KEY>
-        - <NAMESPACE>:<KEY>:<REMAP>
-
-        Dict configuration:
-        - key: key to query
-        - namespace: namespace to query
-        - remap: rename key to ansible variable
-        """
-
-        ret = dict(self.default_config)
-
-        if isinstance(key_def, dict):
-            ret.update(key_def)
-
-        elif isinstance(key_def, str):
-
-            key = None
-            remap = key
-            namespace = ret['namespace']
-
-            # Extract config from string
-            parts = key_def.split(KEY_NS_SEP, 3)
-            if len(parts) == 1:
-                key = parts[0]
-            elif len(parts) > 1:
-                namespace = parts[0]
-                key = parts[1]
-
-                if len(parts) == 3:
-                    remap = parts[2]
-
-            # Generate new config
-            string_conf = {
-                    "key": key,
-                    "namespace": namespace,
-                    "remap": remap,
-                    }
-            ret.update(string_conf)
-
-        else:
-            raise Exception(f"Key configuration is invalid, expected a string or dict, got: {key_def}")
-
-        return ret
 
+@dataclass
+class Key:
+    key: str
+    remap: Union[str, type(None)]
+    namespace: Union[type(None), str]
 
     def show(self):
-        """Method to show the accepatable string format for Kheops"""
         ret = self.key
         if self.namespace is not None:
             ret = f"{self.namespace}{KEY_NS_SEP}{ret}"
         return ret
 
 
-class Keys():
-    """Keys config instance class"""
-
-    def __init__(self, config, default_namespace="default"):
-
-        self.raw_config = config
-        self.default_namespace = default_namespace
-
-        self.key_list = self.parse_config(config)
-
-    def parse_config(self, keys_def, default_namespace="default"):
-        """Parse a key config structure
-
-        A keydef can either be:
-        - a string: Single lookup
-        - a list: ordered lookup
-        - a dict: unordered lookup
-        """
-
-        ret = []
-        if isinstance(keys_def, list):
-            for key_def in keys_def:
-                ret.append(Key(key_def))
-
-        elif isinstance(keys_def, str):
-            ret.append(Key(keys_def))
-
-        elif isinstance(keys_def, dict):
-            for key, value in keys_def.items():
-                if isinstance(value, str):
-                    key_def = value or key
-
-                if isinstance(value, dict):
-                    key_def = value or {}
-                    key_def['key'] = key_def.get('key', key)
-
-                ret.append(Key(key_def))
-        else:
-            raise AnsibleError(f"Unable to process Kheops keys: {ret}")
-
-        if len(ret) == 0:
-            raise AnsibleError(f"Kheops query needs at least one key to lookup")
-
-        return ret
-
-
 class AnsibleKheops:
     """Main Ansible Kheops Class"""
 
-    # Extract default config from documentation
-    default_config = {
-        key: value.get("default", None)
-        for key, value in yaml.safe_load(
-            DOCUMENTATION_OPTION_FRAGMENT).items()
-    }
-
-    # Kheops instance management
-    # ----------------------------
-
     def __init__(self, configs=None, display=None):
+
+        self.configs = configs or []
         self.display = display or Display()
 
-        configs = configs or []
+        config = self.get_config()
-        self.config = self.parse_configs(configs)
-        self.display.v("Kheops instance has been created, with config: %s" % self.config['instance_config'])
 
         # Instanciate Kheops
-        if self.config["mode"] == "instance":
+        if config["mode"] == "instance":
 
             # Configure logging
             logger = logging.getLogger("kheops")
-            logger.setLevel(self.config["instance_log_level"])
+            logger.setLevel(config["instance_log_level"])
 
             # See for logging: https://medium.com/opsops/debugging-requests-1989797736cc
-            #class ListLoggerHandler(logging.Handler):
+            class ListLoggerHandler(logging.Handler):
-            #    def emit(self, record):
+                def emit(self, record):
-            #        msg = self.format(record)
+                    msg = self.format(record)
 
             main_logger = logging.getLogger()
-            #main_logger.addHandler(ListLoggerHandler())
+            main_logger.addHandler(ListLoggerHandler())
             main_logger.setLevel(logging.DEBUG)
 
             # Start instance
             self.kheops = Kheops(
-                config=self.config["instance_config"],
+                config=config["instance_config"], namespace=config["instance_namespace"]
-                namespace=self.config["instance_namespace"],
-                cache=KHEOPS_CACHE
             )
-        elif self.config["mode"] == "client":
+        elif config["mode"] == "client":
             raise AnsibleError("Kheops client mode is not implemented")
 
+        self.config = config
+        self.display.v("Kheops instance has been created, with config: %s" % config['instance_config'])
 
-    def parse_configs(self, configs):
+    def get_config(self):
         """
-        Take a list of config items that will be merged together.
+        Processing order:
-
+        - Fetch the value of config or fallback on ANSIBLE_KHEOPS_CONFIG
-        A config item can either be:
+        - Load the config if any
-        - A string: Represents the path of the config
+        - Overrides with other options
-        - A dict: Represent a direct configuration
-        - None: Just ignore this entry
-
-        #Processing order:
-        #- Fetch the value of config or fallback on ANSIBLE_KHEOPS_CONFIG
-        #- Load the config if any
-        #- Overrides with other options
         """
 
+        # Extract default value from doc
+        data_doc = yaml.safe_load(DOCUMENTATION_OPTION_FRAGMENT)
+        default_config = {
+            key: value.get("default", None) for key, value in data_doc.items()
+        }
 
-        # Get default configuration from environment
-        env_items = [
-            # We exclude 'config' on purpose
-            "mode",
-            "default_namespace",
-
-            "instance_config",
-            "instance_namespace",
-            "instance_log_level",
-            "instance_log_explain",
-
-            # Future:
-            # "token",
-            # "host",
-            # "port",
-            # "protocol",
-            # "validate_certs",
-        ]
-        env_config = {}
-        for item in env_items:
-            envvar = "ANSIBLE_KHEOPS_" + item.upper()
-            try:
-                env_config[item] = os.environ[envvar]
-            except KeyError:
-                pass
-
-        # Merge/process runtime configurations
         merged_configs = {}
-        for config in configs:
+        for config in self.configs:
 
             conf_data = None
             if isinstance(config, str):
@@ -425,183 +277,207 @@ class AnsibleKheops:
             elif isinstance(config, type(None)):
                 continue
             else:
-                assert False, f"Bad config for, expected a path or a dict, got type {type(config)}: {config}"
+                assert False, f"Bad config for: {config}"
 
             assert isinstance(conf_data, dict), f"Bug with conf_data: {conf_data}"
             if isinstance(conf_data, dict):
                 merged_configs.update(conf_data)
 
+        # Get environment config
+        items = [
+            # We exclude 'config'
+            "mode",
+            "instance_config",
+            "instance_namespace",
+            "instance_log_level",
+            "namespace",
+            "scope",
+            "keys",
+        ]
+        env_config = {}
+        for item in items:
+            envvar = "ANSIBLE_KHEOPS_" + item.upper()
+            try:
+                env_config[item] = os.environ[envvar]
+            except KeyError:
+                pass
+
         # Merge results
-        combined_config = dict(self.default_config)
+        combined_config = {}
+        combined_config.update(default_config)
         combined_config.update(env_config)
         combined_config.update(merged_configs)
 
         return combined_config
 
+    @staticmethod
+    def parse_string(item, default_namespace):
+        key = None
+        remap = key
+        namespace = default_namespace
 
-    # Lookup methods
+        if isinstance(item, str):
-    # ----------------------------
 
+            parts = item.split(KEY_NS_SEP, 3)
+            if len(parts) > 0:
+                key = parts[0]
+            if len(parts) > 1:
+                namespace = parts[0]
+                key = parts[1]
+            if len(parts) > 2:
+                remap = parts[2]
 
-    def super_lookup_seq(
+        elif isinstance(item, dict):
-        self,
+            key = item.get("key")
-        keys,
+            remap = item.get("remap", key)
-        scope_config=None,
+            namespace = item.get("namespace", namespace)
-        scope_vars=None,
-        namespace=None,
-        kwargs=None,
 
-        _templar=None,
+        return Key(key=key, remap=remap, namespace=namespace)
-        _process_scope=None,
-        _process_results=None,
 
-        jinja2_native=False,
+    @classmethod
-    ):
+    def parse_keys(self, data, namespace):
-        """
+        keys = []
-        Sequential Lookup method wrapper
+        if isinstance(data, str):
-        """
+            keys.append(self.parse_string(data, namespace))
 
-        # Determine runtime options
+        elif isinstance(data, list):
-        _process_scope = _process_scope if _process_scope in ["vars", "jinja"]  else self.config["process_scope"]
+            for item in data:
-        _process_results = _process_results if isinstance(_process_results, bool ) else self.config["process_results"]
+                keys.append(self.parse_string(item, namespace))
-        _reinject_results = True
+        elif isinstance(data, dict):
+            for key, value in data.items():
+                item = key
+                if value:
+                    assert isinstance(value, str), f"Need a string, got: {value}"
+                    item = f"{key}{KEY_NS_SEP}{value}"
 
-        # Build query context
+                keys.append(self.parse_string(item, namespace))
-        namespace = namespace or self.config.get('default_namespace')
-        scope_config = scope_config or self.config.get('scope')
 
-        keys = keys or self.config.get('keys', [])
-        keys_config = Keys(keys, default_namespace=namespace)
-
-        scope_final = self.get_scope(
-                _process_scope,
-                scope_vars, scope_config,
-                _templar=_templar, jinja2_native=jinja2_native
-                )
-
-        # Query kheops
-        result = []
-        ret = {}
-        for key in keys_config.key_list:
-
-            if _reinject_results:
-                scope_vars.update(dict(ret))
-                scope_final = self.get_scope(
-                        _process_scope,
-                        scope_vars, scope_config,
-                        _templar=_templar, jinja2_native=jinja2_native
-                        )
-
-            # Query kheops
-            ret = self.kheops.lookup(
-                keys=[key.show()],
-
-                scope=scope_final,
-                trace=key.trace,
-                explain=key.explain,
-                namespace_prefix=key.namespace_prefix
-            )
-
-            # Remap output
-            if key.remap is not None and key.remap != key.key:
-                #print (f"REMAP KEY: {key.key} => {key.remap}")
-                ret[key.remap] = ret[key.key]
-                del ret[key.key]
-
-            # Process output
-            if _process_results == "jinja":
-                with _templar.set_temporary_context(available_variables=scope_vars):
-                    ret = _templar.template(
-                        ret,
-                        preserve_trailing_newlines=True,
-                        convert_data=False,
-                        escape_backslashes=False,
-                    )
-                if USE_JINJA2_NATIVE and not jinja2_native:
-                    ret = NativeJinjaText(ret)
-
-            result.append(ret)
-
-        return result
-
-    # Scope management methods
-    # ----------------------------
-
-    def get_scope(self, method, scope_vars, scope_config, _templar=None, jinja2_native=False):
-        """
-        Simple wrapper for _get_scope_template and _get_scope_vars
-        """
-
-        ret = {}
-        if method == "jinja":
-            ret = self._get_scope_template(scope_vars, scope_config,
-                    _templar, jinja2_native=jinja2_native)
-        elif method == "vars":
-            ret = self._get_scope_vars(scope_vars, scope_config)
-        elif method == "all":
-            ret = scope_vars
         else:
-            raise AnsibleError(f"Get_scope only accept 'jinja','vars' or 'all', got: {method}")
+            raise AnsibleError(f"Unable to process Kheops keys: {keys}")
 
-        #print ("===> Scope Config:", method)
+        return keys
-        #print (scope_config)
-        #print ("===> Scope Vars:", scope_vars.get('tiger_profiles', 'MISSING_PROFILE'))
-        #print (scope_vars.keys())
-        #print ("===> Scope Result:")
-        #print (ret)
-        #print ("===> Scope EOF")
 
-        return dict(ret)
+    def get_scope_from_host_inventory(self, host_vars, scope=None):
-
-    def _get_scope_template(self, data, scope_config, templar, jinja2_native=False):
         """
-        Create a scope context from data, jinja2 interpolation
+        Build scope from host vars
         """
-        scope_config = scope_config or self.config.get('scope', {})
+        scope = scope or self.config["scope"]
-        assert isinstance(data, dict)
+        ret = {}
-        assert templar
+        for key, val in scope.items():
+            # Tofix should this fail silently ?
+            ret[key] = host_vars.get(val, None)
+
+        return ret
+
+    def get_scope_from_jinja(self, host_vars, templar, scope=None, jinja2_native=False):
+        """
+        Parse in jinja a dict scope
+        """
+        scope = scope or self.config["scope"]
 
         if USE_JINJA2_NATIVE and not jinja2_native:
             _templar = templar.copy_with_new_env(environment_class=AnsibleEnvironment)
         else:
             _templar = templar
 
-        # Process template with jinja
+        _vars = deepcopy(host_vars)
-        _vars = dict(data)
         ret = {}
         with _templar.set_temporary_context(available_variables=_vars):
 
-            try:
+            for key, value in scope.items():
+                res = value
+                try:
+                    res = _templar.template(
+                        value,
+                        preserve_trailing_newlines=True,
+                        convert_data=True,
+                        escape_backslashes=False,
+                    )
+                    if USE_JINJA2_NATIVE and not jinja2_native:
+                        # jinja2_native is true globally but off for the lookup, we need this text
+                        # not to be processed by literal_eval anywhere in Ansible
+                        res = NativeJinjaText(res)
+                    self.display.vvv(f"Transformed scope value: {value} => {res}")
+                except AnsibleUndefinedVariable as err:
+                    self.display.error(f"Got templating error for string '{value}': {err}")
+                    raise err
+
+                ret[key] = res
+
+        return ret
+
+    def lookup(self, keys, namespace=None, scope=None, explain=None):
+        """
+        Start a lookup query
+        """
+
+        if explain is None:
+            explain = self.config["instance_explain"]
+
+        namespace = namespace or self.config["namespace"]
+        scope = scope or self.config["scope"]
+        keys = keys or self.config["keys"]
+        keys_config = self.parse_keys(keys, namespace)
+        keys = [i.show() for i in keys_config]
+
+        self.display.v(f"Kheops keys: {keys}")
+        self.display.vv(f"Kheops scope: {scope}")
+
+        ret = self.kheops.lookup(
+            keys=keys,
+            scope=scope,
+            # trace=True,
+            explain=explain,
+            namespace_prefix=False
+        )
+
+        # Remap output
+        for key in keys_config:
+            if key.remap is not None and key.remap != key.key:
+                ret[key.remap] = ret[key.key]
+                del ret[key.key]
+
+        return ret or {}
+
+    def super_lookup(
+        self,
+        keys,
+        namespace=None,
+        scope=None,
+        kwargs=None,
+        _templar=None,
+        _variables=None,
+        _process_scope=None,
+        _process_results=None,
+        jinja2_native=False,
+    ):
+        """
+        Lookup method wrapper
+        """
+
+        _process_scope = _process_scope or self.config["process_scope"]
+        _process_results = _process_results or self.config["process_results"]
+
+        scope = scope or self.config["scope"]
+        if _process_scope == "vars":
+            scope = self.get_scope_from_host_inventory(_variables, scope=scope)
+        elif _process_scope == "jinja":
+            assert _templar, f"BUG: We expected a templar object here, got: {_templar}"
+            scope = self.get_scope_from_jinja(
+                _variables, _templar, scope=scope, jinja2_native=jinja2_native
+            )
+
+        ret = self.lookup(keys, namespace=namespace, scope=scope)
+
+        if _process_results == "jinja":
+            with _templar.set_temporary_context(available_variables=_variables):
                 ret = _templar.template(
-                    scope_config,
+                    ret,
                     preserve_trailing_newlines=True,
-                    convert_data=True,
+                    convert_data=False,
                     escape_backslashes=False,
                 )
-                if USE_JINJA2_NATIVE and not jinja2_native:
+            if USE_JINJA2_NATIVE and not jinja2_native:
-                    # jinja2_native is true globally but off for the lookup, we need this text
+                ret = NativeJinjaText(ret)
-                    # not to be processed by literal_eval anywhere in Ansible
-                    ret = NativeJinjaText(ret)
-                self.display.vvv(f"Transformed scope value: {scope_config} => {ret}")
-
-            except AnsibleUndefinedVariable as err:
-                self.display.error(f"Got templating error for string '{scope_config}': {err}")
-                raise err
 
         return ret
-
-
-    def _get_scope_vars(self, data, scope_config):
-        """
-        Create a scope context from data, simple interpolation
-        """
-
-        scope_config = scope_config or self.config.get('scope', {})
-        assert isinstance(data, dict)
-
-        ret = {}
-        for key, val in scope_config.items():
-            ret[key] = data.get(val, None)
-
-        return ret
-