pro-barbu-foss/ansible-collection-kheops
pro-barbu-foss/ansible-collection-kheops
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Clean: and lint plugin code

Browse Source
This commit is contained in:
Robin Pierre Cordier 2022-03-10 12:16:02 -05:00
parent 5bda51ff36
commit 3bf4bc8da3
2 changed files with 179 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
plugins/inventory/kheops.py
View File

@ -6,17 +6,11 @@
# pylint: disable=super-with-arguments # pylint: disable=super-with-arguments
from __future__ import (absolute_import, division, print_function) from __future__ import (absolute_import, division, print_function)
__metaclass__ = type # __metaclass__ = type
import sys
import os
import logging
from ansible import constants as C from ansible import constants as C
from ansible.errors import AnsibleError from ansible.errors import AnsibleError
from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable, Constructable from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable, Constructable
#sys.path.append("/home/jez/prj/bell/training/tiger-ansible/ext/kheops")
from ansible_collections.barbu_it.ansible_kheops.plugins.plugin_utils.common import DOCUMENTATION_OPTION_FRAGMENT, AnsibleKheops from ansible_collections.barbu_it.ansible_kheops.plugins.plugin_utils.common import DOCUMENTATION_OPTION_FRAGMENT, AnsibleKheops
DOCUMENTATION = ''' DOCUMENTATION = '''
@ -65,7 +59,6 @@ query_scope:
environment: foreman_environment_name environment: foreman_environment_name
''' '''
from pprint import pprint
class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable): class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable):
@ -78,6 +71,7 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable):
valid = True valid = True
return valid return valid
def parse(self, inventory, loader, path, cache): def parse(self, inventory, loader, path, cache):
'''Return dynamic inventory from source ''' '''Return dynamic inventory from source '''
@ -89,6 +83,7 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable):
self.jinja2_native = self.get_option('jinja2_native') self.jinja2_native = self.get_option('jinja2_native')
self.strict = self.get_option('strict') self.strict = self.get_option('strict')
self.compose = self.get_option('compose') self.compose = self.get_option('compose')
self.groups = self.get_option('groups') self.groups = self.get_option('groups')
self.keyed_groups = self.get_option('keyed_groups') self.keyed_groups = self.get_option('keyed_groups')
@ -97,49 +92,40 @@ class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable):
# Prepare Kheops instance # Prepare Kheops instance
self.config_file = self.get_option('config') self.config_file = self.get_option('config')
ansible_config = {
"instance_namespace": self.get_option('instance_namespace'),
"instance_log_level": self.get_option('instance_log_level'),
"instance_explain": self.get_option('instance_explain'),
}
configs = [ configs = [
self.config_file, ansible_config,
path, self.config_file,
] path,
]
kheops = AnsibleKheops(configs=configs, display=self.display) kheops = AnsibleKheops(configs=configs, display=self.display)
# Loop over each hosts # Loop over each hosts
for host_name in inventory.hosts: for host_name in inventory.hosts:
host = self.inventory.get_host(host_name) host = self.inventory.get_host(host_name)
ret = kheops.super_lookup( ret = kheops.super_lookup(
keys=None, keys=None,
scope=None, scope=None,
_templar=self.templar, _templar=self.templar,
_variables=host.get_vars(), _variables=host.get_vars(),
jinja2_native=self.jinja2_native, jinja2_native=self.jinja2_native,
#trace=True, #trace=True,
#explain=True, #explain=True,
) )
# # Create the scope
# if self.process_scope == 'vars':
# scope = kheops.get_scope_from_host_inventory(host.get_vars(), scope=None)
# elif self.process_scope == 'jinja':
# scope = kheops.get_scope_from_jinja(host.get_vars(), self.templar, scope=None)
# # Fetch the results
# ret = kheops.lookup(
# keys=None,
# scope=scope,
# #trace=True,
# #explain=True,
# )
# Inject variables into host
for key, value in ret.items():
# Inject variables into host
for key, value in ret.items():
self.display.vv (f"Define variable for {host_name}: {key}={value}") self.display.vv (f"Define variable for {host_name}: {key}={value}")
host.set_variable(key, value) host.set_variable(key, value)
# Call constructed inventory plugin methods # Call constructed inventory plugin methods
hostvars = host.get_vars() hostvars = self.inventory.get_host(host_name).get_vars()
self._set_composite_vars(self.compose, hostvars, host_name, self.strict) self._set_composite_vars(self.compose, hostvars, host_name, self.strict)
self._add_host_to_composed_groups(self.groups, hostvars, host_name, self.strict) self._add_host_to_composed_groups(self.groups, hostvars, host_name, self.strict)
self._add_host_to_keyed_groups(self.keyed_groups, hostvars, host_name, self.strict) self._add_host_to_keyed_groups(self.keyed_groups, hostvars, host_name, self.strict)

276
plugins/plugin_utils/common.py
View File

@ -1,9 +1,25 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os
import logging
from dataclasses import dataclass
from typing import Any, Union
from copy import deepcopy
import yaml
from ansible.errors import AnsibleError, AnsibleUndefinedVariable
from ansible.module_utils.common.text.converters import to_native
from ansible.utils.display import Display
from ansible.template import (
generate_ansible_template_vars,
AnsibleEnvironment,
USE_JINJA2_NATIVE,
)
from kheops.app import Kheops
DOCUMENTATION_OPTION_FRAGMENT = """
DOCUMENTATION_OPTION_FRAGMENT = '''
# Plugin configuration # Plugin configuration
# ========================== # ==========================
@ -14,6 +30,9 @@ DOCUMENTATION_OPTION_FRAGMENT = '''
- All settings of the target files can be overriden from this file - All settings of the target files can be overriden from this file
- Ignored if blank or Null - Ignored if blank or Null
default: Null default: Null
ini:
- section: inventory
key: kheops_config
env: env:
- name: ANSIBLE_KHEOPS_CONFIG - name: ANSIBLE_KHEOPS_CONFIG
@ -47,22 +66,33 @@ DOCUMENTATION_OPTION_FRAGMENT = '''
instance_config: instance_config:
description: description:
- The Kheops configuration file to use. - The Kheops configuration file to use.
- Require mode=instance
default: 'site/kheops.yml' default: 'site/kheops.yml'
env: env:
- name: ANSIBLE_KHEOPS_INSTANCE_CONFIG - name: ANSIBLE_KHEOPS_INSTANCE_CONFIG
instance_namespace: instance_namespace:
description: description:
- The Kheops configuration file to use. - The Kheops configuration file to use.
- Require mode=instance
default: 'default' default: 'default'
env: env:
- name: ANSIBLE_KHEOPS_INSTANCE_NAMESPACE - name: ANSIBLE_KHEOPS_INSTANCE_NAMESPACE
instance_log_level: instance_log_level:
description: description:
- Khéops logging level - Khéops logging level
- Require mode=instance
choices: ['DEBUG', 'INFO', 'WARNING', 'ERROR'] choices: ['DEBUG', 'INFO', 'WARNING', 'ERROR']
default: 'WARNING' default: 'WARNING'
env: env:
- name: ANSIBLE_KHEOPS_LOG_LEVEL - name: ANSIBLE_KHEOPS_INSTANCE_LOG_LEVEL
instance_explain:
description:
- Khéops logging explain
- Require mode=instance
type: boolean
default: False
env:
- name: ANSIBLE_KHEOPS_INSTANCE_EXPLAIN
# Instance configuration (Client) # Instance configuration (Client)
@ -158,51 +188,10 @@ DOCUMENTATION_OPTION_FRAGMENT = '''
- Kheops documentation is available on http://kheops.io/ - Kheops documentation is available on http://kheops.io/
- You can add more parameters as documented in http://kheops.io/server/api - You can add more parameters as documented in http://kheops.io/server/api
"""
# Uneeded # Misc
# Uneeded version:
# Uneeded description:
# Uneeded - Kheops API version to use.
# Uneeded default: 1
# Uneeded choices: [1]
# Uneeded env:
# Uneeded - name: ANSIBLE_KHEOPS_VERSION
# Uneeded cache:
# Uneeded description:
# Uneeded - Enable Kheops inventory cache.
# Uneeded default: false
# Uneeded type: boolean
# Uneeded env:
# Uneeded - name: ANSIBLE_KHEOPS_CACHE
# Uneeded policy:
# Uneeded description:
# Uneeded - Kheops policy to use for the lookups.
# Uneeded default: 'default'
''' KEY_NS_SEP = "/"
import os
import sys
import yaml
import logging
from dataclasses import dataclass
from typing import Any, Union
from copy import deepcopy
# Devel mode
sys.path.append("/home/jez/prj/bell/training/tiger-ansible/ext/kheops")
from kheops.app import Kheops
from ansible.errors import AnsibleError
from ansible.module_utils.common.text.converters import to_native
from ansible.template import generate_ansible_template_vars, AnsibleEnvironment, USE_JINJA2_NATIVE
from ansible.utils.display import Display
from pprint import pprint
KEY_NS_SEP = ':'
if USE_JINJA2_NATIVE: if USE_JINJA2_NATIVE:
from ansible.utils.native_jinja import NativeJinjaText from ansible.utils.native_jinja import NativeJinjaText
@ -220,8 +209,8 @@ class Key:
return ret return ret
class AnsibleKheops(): class AnsibleKheops:
"""Main Ansible Kheops Class"""
def __init__(self, configs=None, display=None): def __init__(self, configs=None, display=None):
@ -229,16 +218,12 @@ class AnsibleKheops():
self.display = display or Display() self.display = display or Display()
config = self.get_config() config = self.get_config()
# print ("CURRENT CONFIG vv")
# pprint (config)
# print ("CURRENT CONFIG ^^")
# Instanciate Kheops # Instanciate Kheops
if config["mode"] == 'instance': if config["mode"] == "instance":
# Configure logging # Configure logging
logger = logging.getLogger('kheops') logger = logging.getLogger("kheops")
logger.setLevel(config["instance_log_level"]) logger.setLevel(config["instance_log_level"])
# See for logging: https://medium.com/opsops/debugging-requests-1989797736cc # See for logging: https://medium.com/opsops/debugging-requests-1989797736cc
@ -252,15 +237,13 @@ class AnsibleKheops():
# Start instance # Start instance
self.kheops = Kheops( self.kheops = Kheops(
config=config['instance_config'], config=config["instance_config"], namespace=config["instance_namespace"]
namespace=config['instance_namespace'] )
) elif config["mode"] == "client":
elif config["mode"] == 'client':
raise AnsibleError("Kheops client mode is not implemented") raise AnsibleError("Kheops client mode is not implemented")
self.config = config self.config = config
self.display.v(f"Kheops instance has been created") self.display.v("Kheops instance has been created")
def get_config(self): def get_config(self):
""" """
@ -272,11 +255,10 @@ class AnsibleKheops():
# Extract default value from doc # Extract default value from doc
data_doc = yaml.safe_load(DOCUMENTATION_OPTION_FRAGMENT) data_doc = yaml.safe_load(DOCUMENTATION_OPTION_FRAGMENT)
default_config = {key: value.get("default", None) for key, value in data_doc.items()} default_config = {
key: value.get("default", None) for key, value in data_doc.items()
}
#print ("Show configs")
#pprint (self.configs)
merged_configs = {} merged_configs = {}
for config in self.configs: for config in self.configs:
@ -284,34 +266,40 @@ class AnsibleKheops():
if isinstance(config, str): if isinstance(config, str):
self.display.vv("Read Kheops file config", config) self.display.vv("Read Kheops file config", config)
if os.path.isfile(config): if os.path.isfile(config):
data = open(config, "r") data = open(config, "r", encoding="utf-8")
conf_data = yaml.safe_load(data) conf_data = yaml.safe_load(data)
else: else:
raise AnsibleError("Unable to find configuration file %s" % config_file) raise AnsibleError(f"Unable to find configuration file {config}")
elif isinstance(config, dict): elif isinstance(config, dict):
self.display.vv ("Read Kheops direct config", config) self.display.vv("Read Kheops direct config", config)
conf_data = config conf_data = config
elif isinstance(config, type(None)):
continue
else: else:
assert False, f"Bad config for: {config}" assert False, f"Bad config for: {config}"
assert isinstance(conf_data, dict), f"Bug with conf_data: {config_data}" assert isinstance(conf_data, dict), f"Bug with conf_data: {conf_data}"
if isinstance(conf_data, dict): if isinstance(conf_data, dict):
merged_configs.update(conf_data) merged_configs.update(conf_data)
# Get environment config # Get environment config
items = [ items = [
# We exclude 'config' # We exclude 'config'
'mode', "mode",
'instance_config', 'instance_namespace', 'instance_log_level', "instance_config",
'namespace', 'scope', 'keys'] "instance_namespace",
"instance_log_level",
"namespace",
"scope",
"keys",
]
env_config = {} env_config = {}
for item in items: for item in items:
envvar = "ANSIBLE_KHEOPS_" + item.upper() envvar = "ANSIBLE_KHEOPS_" + item.upper()
try: try:
env_config[item] = os.environ[envvar] env_config[item] = os.environ[envvar]
except KeyError as err: except KeyError:
pass pass
# Merge results # Merge results
@ -320,22 +308,8 @@ class AnsibleKheops():
combined_config.update(env_config) combined_config.update(env_config)
combined_config.update(merged_configs) combined_config.update(merged_configs)
# Debug report
# out = {
# "0_default": default_config,
# "1_env": env_config,
# "2_common": merged_configs,
# }
# print ('=' * 20)
# pprint (out)
# print ('=' * 20)
# pprint (combined_config)
# print ('=' * 20)
return combined_config return combined_config
@staticmethod @staticmethod
def parse_string(item, default_namespace): def parse_string(item, default_namespace):
key = None key = None
@ -354,20 +328,15 @@ class AnsibleKheops():
remap = parts[2] remap = parts[2]
elif isinstance(item, dict): elif isinstance(item, dict):
key = item.get('key') key = item.get("key")
remap = item.get('remap', key) remap = item.get("remap", key)
namespace = item.get('namespace', namespace) namespace = item.get("namespace", namespace)
return Key(key=key,
remap=remap,
namespace=namespace)
return Key(key=key, remap=remap, namespace=namespace)
@classmethod @classmethod
def parse_keys(self, data, namespace): def parse_keys(self, data, namespace):
keys = [] keys = []
if isinstance(data, str): if isinstance(data, str):
keys.append(self.parse_string(data, namespace)) keys.append(self.parse_string(data, namespace))
@ -384,13 +353,15 @@ class AnsibleKheops():
keys.append(self.parse_string(item, namespace)) keys.append(self.parse_string(item, namespace))
else: else:
raise AnsibleError("Unable to process Kheops keys: %s" % keys) raise AnsibleError(f"Unable to process Kheops keys: {keys}")
return keys return keys
def get_scope_from_host_inventory(self, host_vars, scope=None): def get_scope_from_host_inventory(self, host_vars, scope=None):
scope = scope or self.config['scope'] """
Build scope from host vars
"""
scope = scope or self.config["scope"]
ret = {} ret = {}
for key, val in scope.items(): for key, val in scope.items():
# Tofix should this fail silently ? # Tofix should this fail silently ?
@ -398,9 +369,11 @@ class AnsibleKheops():
return ret return ret
def get_scope_from_jinja(self, host_vars, templar, scope=None, jinja2_native=False): def get_scope_from_jinja(self, host_vars, templar, scope=None, jinja2_native=False):
scope = scope or self.config['scope'] """
Parse in jinja a dict scope
"""
scope = scope or self.config["scope"]
if USE_JINJA2_NATIVE and not jinja2_native: if USE_JINJA2_NATIVE and not jinja2_native:
_templar = templar.copy_with_new_env(environment_class=AnsibleEnvironment) _templar = templar.copy_with_new_env(environment_class=AnsibleEnvironment)
@ -414,39 +387,51 @@ class AnsibleKheops():
for key, value in scope.items(): for key, value in scope.items():
res = value res = value
try: try:
res = _templar.template(value, preserve_trailing_newlines=True, res = _templar.template(
convert_data=True, escape_backslashes=False) value,
preserve_trailing_newlines=True,
convert_data=True,
escape_backslashes=False,
)
if USE_JINJA2_NATIVE and not jinja2_native: if USE_JINJA2_NATIVE and not jinja2_native:
# jinja2_native is true globally but off for the lookup, we need this text # jinja2_native is true globally but off for the lookup, we need this text
# not to be processed by literal_eval anywhere in Ansible # not to be processed by literal_eval anywhere in Ansible
res = NativeJinjaText(res) res = NativeJinjaText(res)
self.display.vvv(f"Transformed: {value} =====> {res}") self.display.vvv(f"Transformed scope value: {value} => {res}")
except Exception as err: except AnsibleUndefinedVariable as err:
self.display.v(f"Got templating error for value: {value} => {err}") self.display.error(f"Got templating error for string '{value}': {err}")
raise Exception() from err
ret[key] = res ret[key] = res
return ret return ret
def lookup(self, keys, namespace=None, scope=None, explain=None):
"""
Start a lookup query
"""
if explain is None:
explain = self.config["instance_explain"]
def lookup(self, keys, namespace=None, scope=None, kwargs=None): namespace = namespace or self.config["namespace"]
scope = scope or self.config["scope"]
namespace = namespace or self.config['namespace'] keys = keys or self.config["keys"]
scope = scope or self.config['scope']
keys = keys or self.config['keys']
keys_config = self.parse_keys(keys, namespace) keys_config = self.parse_keys(keys, namespace)
keys = [ i.show() for i in keys_config ] keys = [i.show() for i in keys_config]
self.display.v(f"Kheops keys: {keys}") self.display.v(f"Kheops keys: {keys}")
self.display.vv(f"Kheops scope: {scope}") self.display.vv(f"Kheops scope: {scope}")
# try:
ret = self.kheops.lookup( ret = self.kheops.lookup(
keys=keys, keys=keys,
scope=scope, scope=scope,
#trace=True, # trace=True,
#explain=True, explain=explain,
) )
# except Exception as err:
# raise AnsibleError(err)
# Remap output # Remap output
for key in keys_config: for key in keys_config:
@ -456,34 +441,45 @@ class AnsibleKheops():
return ret or {} return ret or {}
def super_lookup(
self,
keys,
namespace=None,
scope=None,
kwargs=None,
_templar=None,
_variables=None,
_process_scope=None,
_process_results=None,
jinja2_native=False,
):
"""
Lookup method wrapper
"""
def super_lookup(self, keys, namespace=None, scope=None, kwargs=None, _process_scope = _process_scope or self.config["process_scope"]
_templar=None, _process_results = _process_results or self.config["process_results"]
_variables=None,
_process_scope=None,
_process_results=None,
jinja2_native=False
):
_process_scope = _process_scope or self.config['process_scope'] scope = scope or self.config["scope"]
_process_results = _process_results or self.config['process_results'] if _process_scope == "vars":
if _process_scope == 'vars':
scope = self.get_scope_from_host_inventory(_variables, scope=scope) scope = self.get_scope_from_host_inventory(_variables, scope=scope)
elif _process_scope == 'jinja': elif _process_scope == "jinja":
assert _templar, f"BUG: We expected a templar object here, got: {_templar}" assert _templar, f"BUG: We expected a templar object here, got: {_templar}"
scope = self.get_scope_from_jinja(_variables, _templar, scope=scope, jinja2_native=jinja2_native) scope = self.get_scope_from_jinja(
_variables, _templar, scope=scope, jinja2_native=jinja2_native
)
ret = self.lookup(keys, namespace=namespace, scope=scope) ret = self.lookup(keys, namespace=namespace, scope=scope)
if _process_results == 'jinja': if _process_results == "jinja":
with _templar.set_temporary_context(available_variables=_variables): with _templar.set_temporary_context(available_variables=_variables):
ret = _templar.template(ret, ret = _templar.template(
preserve_trailing_newlines=True, ret,
convert_data=False, escape_backslashes=False) preserve_trailing_newlines=True,
convert_data=False,
escape_backslashes=False,
)
if USE_JINJA2_NATIVE and not jinja2_native: if USE_JINJA2_NATIVE and not jinja2_native:
ret = NativeJinjaText(ret) ret = NativeJinjaText(ret)
return ret return ret