From 53f75848771b79bc0f85e6ac60f016db203008c9 Mon Sep 17 00:00:00 2001 From: mrjk Date: Wed, 2 Feb 2022 17:44:08 -0500 Subject: [PATCH] Doc: Update documentation --- docs/build_doc.sh | 5 + docs/docs/app/internal.md | 8 +- docs/docs/learn/101.md | 4 +- docs/docs/learn/105.md | 2 +- docs/docs/learn/index.rst | 1 + docs/index.rst | 1 - .../learn101-checkpoint.ipynb | 1171 ----------------- docs/jupyter/learn101.ipynb | 6 +- 8 files changed, 15 insertions(+), 1183 deletions(-) delete mode 100644 docs/jupyter/.ipynb_checkpoints/learn101-checkpoint.ipynb diff --git a/docs/build_doc.sh b/docs/build_doc.sh index a224873..da9ba54 100755 --- a/docs/build_doc.sh +++ b/docs/build_doc.sh @@ -5,4 +5,9 @@ make clean #sphinx-apidoc -f -M -o ./api ../kheops/ sphinx-apidoc -M -o ./api ../kheops/ +mkdir -p learn +jupyter nbconvert --to markdown --output=../learn/learn101.md jupyter/learn101.ipynb + +# See: https://www.datacamp.com/community/tutorials/jinja2-custom-export-templates-jupyter + make html diff --git a/docs/docs/app/internal.md b/docs/docs/app/internal.md index 2e61bf4..5bf9020 100644 --- a/docs/docs/app/internal.md +++ b/docs/docs/app/internal.md @@ -7,9 +7,9 @@ + Can be a simple string or complex data + Can be anything in json compatible - Kheops will resolve query in 2 parts: - + It will first look in different locations the key. It is usually files, but it can also be an url or anything. The queried data must be a valid json data type (TOFIX: So it works also for yaml, toml .... it supports [anyconfig]() ) + + It will first look in different locations the key. It is usually files, but it can also be an url or anything. The queried data must be a valid json data type (TOFIX: So it works also for yaml, toml .... it supports [anyconfig](https://github.com/ssato/python-anyconfig) ) * Example: The `path` strategy will allow you to target - + Then it will process all results and load a [strategy]() to resolve which results to keep. + + Then it will process all results and load a strategy to resolve which results to keep. Example: The `last` strategy consists in keeping always the last result while the `merge` strategy consists in merging inteligentelly data. This is quite useful for dict or lists. + - Lookup data in a tree of files @@ -21,8 +21,8 @@ + Rules match to a key and apply a strategy (essentially determine if and how the different value are merged or replaced). It can also apply filter to the result and modify its content (future). * This is modular * Strategy: last/schema - - Last will always take the last found value, whatever what it previously found. [schema](plugins/strategy_last.md) - - schema: It will take into account the strucutre of the data and try to merge them intelligentelly. See more on [schema](plugins/strategy_schema.md) + - Last will always take the last found value, whatever what it previously found. + - schema: It will take into account the strucutre of the data and try to merge them intelligentelly. See more on + Then it returns the result - Use cases + Ansible integration diff --git a/docs/docs/learn/101.md b/docs/docs/learn/101.md index 9fb689a..f920ce2 100644 --- a/docs/docs/learn/101.md +++ b/docs/docs/learn/101.md @@ -13,7 +13,7 @@ + Rules match to a key and apply a strategy (essentially determine if and how the different value are merged or replaced). It can also apply filter to the result and modify its content (future). * This is modular * Strategy: last/schema - - Last will always take the last found value, whatever what it previously found. [schema](plugins/strategy_last.md) - - schema: It will take into account the strucutre of the data and try to merge them intelligentelly. See more on [schema](plugins/strategy_schema.md) + - Last will always take the last found value, whatever what it previously found. + - schema: It will take into account the strucutre of the data and try to merge them intelligentelly. + Then it returns the result diff --git a/docs/docs/learn/105.md b/docs/docs/learn/105.md index 2386835..c17547e 100644 --- a/docs/docs/learn/105.md +++ b/docs/docs/learn/105.md @@ -2,7 +2,7 @@ * 105: Integrations - Integration in Ansible - - Integration as a web server (future) [webserver](webserver.md) + - Integration as a web server (future) - Integration with Puppet - Integration with Foreman diff --git a/docs/docs/learn/index.rst b/docs/docs/learn/index.rst index cff0bb0..5de2fdf 100644 --- a/docs/docs/learn/index.rst +++ b/docs/docs/learn/index.rst @@ -7,6 +7,7 @@ Learn :maxdepth: 2 :caption: Contents: + ../../learn/learn101.md 100.rst 101.md 102.md diff --git a/docs/index.rst b/docs/index.rst index afd212d..f2f7d80 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,7 +34,6 @@ Content Python API - Indices and tables ================== * :ref:`genindex` diff --git a/docs/jupyter/.ipynb_checkpoints/learn101-checkpoint.ipynb b/docs/jupyter/.ipynb_checkpoints/learn101-checkpoint.ipynb deleted file mode 100644 index 2914d2d..0000000 --- a/docs/jupyter/.ipynb_checkpoints/learn101-checkpoint.ipynb +++ /dev/null @@ -1,1171 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "100c8a75", - "metadata": {}, - "source": [ - "# Khéops 101" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "98d4907b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/home/jez/volumes/data/prj/bell/training/tiger-ansible/ext/kheops\n" - ] - } - ], - "source": [ - "cd /home/jez/volumes/data/prj/bell/training/tiger-ansible/ext/kheops\n", - "echo $PWD\n", - "export KHEOPS_NAMESPACE=ex1_enc \n", - "export KHEOPS_CONFIG=examples/kheops.yml\n", - "rm -rf \"examples/$KHEOPS_NAMESPACE\"" - ] - }, - { - "cell_type": "markdown", - "id": "cd52a40b", - "metadata": {}, - "source": [ - "## Command line" - ] - }, - { - "cell_type": "markdown", - "id": "28501b9d", - "metadata": {}, - "source": [ - "Let's check first that kheops works correclty, and start to read the manual." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "6ede46a3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "usage: kheops [-h] [-v] [-c CONFIG]\n", - " {schema,gen_doc,lookup,demo,subcommand2} ...\n", - "\n", - "Kheops, hierarchical data lookup tool\n", - "\n", - "options:\n", - " -h, --help show this help message and exit\n", - " -v, --verbose Increase verbosity (KHEOPS_VERBOSE) (default: 0)\n", - " -c CONFIG, --config CONFIG\n", - " Kheops configuration file (KHEOPS_CONFIG) (default:\n", - " examples/kheops.yml)\n", - "\n", - "subcommands:\n", - " valid subcommands\n", - "\n", - " {schema,gen_doc,lookup,demo,subcommand2}\n" - ] - } - ], - "source": [ - "kheops --help" - ] - }, - { - "cell_type": "markdown", - "id": "1d08340e", - "metadata": {}, - "source": [ - "So we have a working `kheops` command, and we will focus on the `lookup` command. On it's simplest form, a lookup consists in querying a `key` for a given `scope`. The output of the `key` will change depending the `scope` value. A `key` is in simple word." - ] - }, - { - "cell_type": "markdown", - "id": "99969862", - "metadata": {}, - "source": [ - "## Defining a hierarchy" - ] - }, - { - "cell_type": "markdown", - "id": "8192f49a", - "metadata": {}, - "source": [ - "To illustrate how Khéops works, let's start with a simple example, we will try to lookup the `profile` key of the following two (fictive) servers:\n", - "\n", - "* web.domain.org: which act as a webserver role\n", - "* mysql.domain.org: which act as mysql role\n", - "\n", - "But first we need to create our hierarchy. It's as simple as creating directories and put some json or yaml data into different files. Let's create our hierarchy. We will first create the default profile:\n" - ] - }, - { - "cell_type": "markdown", - "id": "63e43682", - "metadata": {}, - "source": [ - "From our use case, we will build a lookup tree. We want to be able to organise data depending the 3 criterias:\n", - "\n", - "* node: name of the node\n", - "* role: assigned role to the node\n", - "* environment: it can either be dev or prod\n", - "\n", - "Let's create our lookup hierarchy:" - ] - }, - { - "cell_type": "markdown", - "id": "efb14a8c", - "metadata": {}, - "source": [ - "default:\n", - " lookups:\n", - " - path: default # Simplest form, just a path\n", - " - path: \"roles/{role}\" # If list, it's auto expanded like in bash\n", - " - path: \"env_{env}\" # If list, it's auto expanded like in bash\n", - " - path: \"nodes/{node}\"\n" - ] - }, - { - "cell_type": "markdown", - "id": "8dd4ed81", - "metadata": {}, - "source": [ - "So for a given key, Khéops will iterate all over those paths to find the requested `key` , and then it will merge all results. Some paths are variabilized, and those variable comes from the scope. The scope come along the `key`, it's can be any metadata. For complex metadata you may want to store those in a file and load your scope with the `-f ` option:\n", - "\n", - "```\n", - "kheops lookup -e -e \n", - "```\n", - "\n", - "A scope is completely optional while keys are required." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b4887796", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "9dfe25dc", - "metadata": {}, - "source": [ - "## Basic hierarchy" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "d3bb7b3f", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[01;34mexamples/ex1_enc\u001b[00m\n", - "└── default.yml\n", - "\n", - "0 directories, 1 file\n", - "---\n", - "profile:\n", - " env: \"NO_ENV\"\n", - " product: \"NO_PRODUCT\"\n", - " \n" - ] - } - ], - "source": [ - "# We create a fresh hierachy\n", - "mkdir -p examples/$KHEOPS_NAMESPACE\n", - "\n", - "# We create a profile key, which is a dict\n", - "cat > examples/$KHEOPS_NAMESPACE/default.yml < examples/$KHEOPS_NAMESPACE/roles/web.yml < examples/$KHEOPS_NAMESPACE/roles/mysql.yml < examples/ex1_enc/default.yml <==\n", - "---\n", - "profile:\n", - " env: \"NO_ENV\"\n", - " product: \"NO_PRODUCT\"\n", - " \n", - "\n", - "==> examples/ex1_enc/roles/mysql.yml <==\n", - "---\n", - "profile:\n", - " product: \"mysql_server\"\n", - "\n", - " mysql_database: \"NO_DATABASE\"\n", - " mysql_users:\n", - " - \"sysadmin@10.0.42%\"\n", - " mysql_port: 3306\n", - " mysql_cluster: False\n", - " \n", - "\n", - "==> examples/ex1_enc/roles/web.yml <==\n", - "---\n", - "profile:\n", - " product: \"httpd_server\"\n", - "\n", - " web_top_domain: \"\"\n", - " web_app: \"NO_APP\"\n", - " web_port: 80\n", - " web_user_list:\n", - " - sysadmins\n", - " \n" - ] - } - ], - "source": [ - "tail -n 999 examples/$KHEOPS_NAMESPACE/{*.yml,*/*.yml}" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "f0835aa6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "env: NO_ENV\n", - "product: httpd_server\n", - "web_top_domain: ''\n", - "web_app: NO_APP\n", - "web_port: 80\n", - "web_user_list:\n", - "- sysadmins\n", - "\n", - "env: NO_ENV\n", - "product: mysql_server\n", - "mysql_database: NO_DATABASE\n", - "mysql_users:\n", - "- sysadmin@10.0.42%\n", - "mysql_port: 3306\n", - "mysql_cluster: false\n", - "\n" - ] - } - ], - "source": [ - "kheops lookup -e node=web.infra.net -e role=web profile\n", - "kheops lookup -e node=mysql.infra.net -e role=mysql profile" - ] - }, - { - "cell_type": "markdown", - "id": "c9c9db00", - "metadata": {}, - "source": [ - "## Per node override" - ] - }, - { - "cell_type": "markdown", - "id": "c538a811", - "metadata": {}, - "source": [ - "It's getting better, we can see that the profile key has been merged with the key values, across the different locations. \n", - "\n", - "However, we will have those placeholders, and we want to have personalized value, depending if it's aweb server, it need an unique domain and some unique parameters. So let's create a `nodes` directory and place some data inside.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "e79adbea", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[01;34mexamples/ex1_enc\u001b[00m\n", - "├── default.yml\n", - "├── \u001b[01;34mnodes\u001b[00m\n", - "│   ├── mysql.infra.net.yml\n", - "│   └── web.infra.net.yml\n", - "└── \u001b[01;34mroles\u001b[00m\n", - " ├── mysql.yml\n", - " └── web.yml\n", - "\n", - "2 directories, 5 files\n" - ] - } - ], - "source": [ - "mkdir -p examples/$KHEOPS_NAMESPACE/nodes\n", - "\n", - "# We create a new web role\n", - "cat > examples/$KHEOPS_NAMESPACE/nodes/web.infra.net.yml < examples/$KHEOPS_NAMESPACE/nodes/mysql.infra.net.yml < examples/$KHEOPS_NAMESPACE/env_dev.yml < examples/$KHEOPS_NAMESPACE/env_prod.yml < Per environment view\n", - "env: prod\n", - "product: NO_PRODUCT\n", - "web_top_domain: infra.com\n", - "web_cache: 12h\n", - "\n", - "env: dev\n", - "product: NO_PRODUCT\n", - "web_top_domain: dev.infra.net\n", - "web_cache: 1m\n", - "web_user_list:\n", - "- debug_user\n", - "mysql_users:\n", - "- debug@10.0.%\n", - "debug: true\n", - "\n", - "==> Per role and environment view\n", - "env: prod\n", - "product: mysql_server\n", - "mysql_database: NO_DATABASE\n", - "mysql_users:\n", - "- sysadmin@10.0.42%\n", - "mysql_port: 3306\n", - "mysql_cluster: false\n", - "web_top_domain: infra.com\n", - "web_cache: 12h\n", - "\n", - "env: prod\n", - "product: httpd_server\n", - "web_top_domain: infra.com\n", - "web_app: NO_APP\n", - "web_port: 80\n", - "web_user_list:\n", - "- sysadmins\n", - "web_cache: 12h\n", - "\n", - "==> Per node view\n", - "env: dev\n", - "product: httpd_server\n", - "web_top_domain: dev.infra.net\n", - "web_app: myapp\n", - "web_port: 80\n", - "web_user_list:\n", - "- sysadmins\n", - "- debug_user\n", - "- domain_org\n", - "- domain_org_external\n", - "web_cache: 1m\n", - "mysql_users:\n", - "- debug@10.0.%\n", - "debug: true\n", - "\n" - ] - } - ], - "source": [ - "kheops lookup profile\n", - "\n", - "echo \"==> Per environment view\"\n", - "kheops lookup -e env=prod profile\n", - "kheops lookup -e env=dev profile\n", - "\n", - "echo \"==> Per role and environment view\"\n", - "kheops lookup -e role=mysql -e env=prod profile\n", - "kheops lookup -e role=web -e env=prod profile\n", - "\n", - "echo \"==> Per node view\"\n", - "kheops lookup -e node=web.infra.net -e role=web -e env=dev profile" - ] - }, - { - "cell_type": "markdown", - "id": "bd04d751", - "metadata": {}, - "source": [ - "Even if somwaht clunky, this method can help to troubleshoot wrong data by dichotomy." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d591f865", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "f0b3c9a4", - "metadata": {}, - "source": [ - "## Tooling and applications" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "974608dd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18f40e34", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f6cd8acd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "365517a7", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37c30f37", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "8bd79659", - "metadata": {}, - "source": [ - "## Troubleshooting" - ] - }, - { - "cell_type": "markdown", - "id": "95f04208", - "metadata": {}, - "source": [ - "Sometimes, it can may be hard to navigate across file and hierachy, but GNU Utils are here to help. There is a selection of small tips:" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "b779c74a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+ : Find where a key has been defined\n", - "+ grep --colour=auto -r '^profile:' examples/ex1_enc\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_prod.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/web.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K \n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/web.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/default.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_dev.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "+ : Find where a key has been defined and 5 first lines\n", - "+ grep --colour=auto -r -A 5 '^profile:' examples/ex1_enc\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_prod.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_prod.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K env: prod\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_prod.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K \n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_prod.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K # On production environment, we always want to use public faced domain and 12 hour cache.\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_prod.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_top_domain: infra.com\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_prod.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_cache: 12h\n", - "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K product: \"mysql_server\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K mysql_database: \"NO_DATABASE\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K mysql_users:\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K - \"sysadmin@10.0.42%\"\n", - "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/web.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/web.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K product: \"httpd_server\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/web.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/web.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_top_domain: \"\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/web.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_app: \"NO_APP\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/web.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_port: 80\n", - "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K \n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K mysql_database: \"app_domain_org\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K mysql_users:\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K - \"app_domain_org@10.0.51%\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K \n", - "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/web.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/web.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_app: 'myapp'\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/web.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_user_list:\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/web.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K - domain_org\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/web.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K - domain_org_external\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/web.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K \n", - "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/default.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/default.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K env: \"NO_ENV\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/default.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K product: \"NO_PRODUCT\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/default.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K \n", - "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_dev.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K\u001b[01;31m\u001b[Kprofile:\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_dev.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K env: dev\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_dev.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K \n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_dev.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K # We change the top domain for dev environment, and reduce the cache\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_dev.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_top_domain: dev.infra.net\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/env_dev.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K web_cache: 1m\n", - "+ : Search from anything related to database\n", - "+ grep --colour=auto -R -C 3 database examples/ex1_enc\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[Kprofile:\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K product: \"mysql_server\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K mysql_\u001b[01;31m\u001b[Kdatabase\u001b[m\u001b[K: \"NO_DATABASE\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K mysql_users:\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K - \"sysadmin@10.0.42%\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/roles/mysql.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K mysql_port: 3306\n", - "\u001b[36m\u001b[K--\u001b[m\u001b[K\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K---\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[Kprofile: \n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K:\u001b[m\u001b[K mysql_\u001b[01;31m\u001b[Kdatabase\u001b[m\u001b[K: \"app_domain_org\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K mysql_users:\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K - \"app_domain_org@10.0.51%\"\n", - "\u001b[35m\u001b[Kexamples/ex1_enc/nodes/mysql.infra.net.yml\u001b[m\u001b[K\u001b[36m\u001b[K-\u001b[m\u001b[K \n", - "+ set +x\n" - ] - } - ], - "source": [ - "set -x\n", - "\n", - ": Find where a key has been defined\n", - "grep -r '^profile:' examples/$KHEOPS_NAMESPACE\n", - "\n", - ": Find where a key has been defined and 5 first lines\n", - "grep -r -A 5 '^profile:' examples/$KHEOPS_NAMESPACE\n", - "\n", - ": Search from anything related to database\n", - "grep -R -C 3 'database' examples/$KHEOPS_NAMESPACE\n", - "\n", - "set +x" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b82b27d5", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "55c95dee", - "metadata": {}, - "source": [ - "The tail/head command is quite usefull to look at multiple files at the same time, it add a nice header for each file:" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "6f451038", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "==> examples/ex1_enc/roles/mysql.yml <==\n", - "---\n", - "profile:\n", - " product: \"mysql_server\"\n", - "\n", - " mysql_database: \"NO_DATABASE\"\n", - " mysql_users:\n", - " - \"sysadmin@10.0.42%\"\n", - " mysql_port: 3306\n", - " mysql_cluster: False\n", - " \n", - "\n", - "==> examples/ex1_enc/roles/web.yml <==\n", - "---\n", - "profile:\n", - " product: \"httpd_server\"\n", - "\n", - " web_top_domain: \"\"\n", - " web_app: \"NO_APP\"\n", - " web_port: 80\n", - " web_user_list:\n", - " - sysadmins\n", - " \n" - ] - } - ], - "source": [ - "head -n 999 examples/ex1_enc/roles/*" - ] - }, - { - "cell_type": "markdown", - "id": "f7a34dfe", - "metadata": {}, - "source": [ - "You can also have a view of all files with this command:\n", - "```\n", - "find . -type f| xargs head -n 999 | less\n", - "```\n", - "\n", - "From there, you will be able to have a nice overview of your data." - ] - }, - { - "cell_type": "markdown", - "id": "5c5bcb81", - "metadata": {}, - "source": [ - "You can even diff your change with this command. There is this simple trick to compare the data difference between 2 lookups:" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "ac14e38f", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--- /dev/fd/63\t2022-02-01 20:10:53.094525316 -0500\n", - "+++ /dev/fd/62\t2022-02-01 20:10:53.094525316 -0500\n", - "@@ -1,11 +1,15 @@\n", - "-env: prod\n", - "+env: dev\n", - " product: httpd_server\n", - "-web_top_domain: infra.com\n", - "+web_top_domain: dev.infra.net\n", - " web_app: myapp\n", - " web_port: 80\n", - " web_user_list:\n", - " - sysadmins\n", - "+- debug_user\n", - " - domain_org\n", - " - domain_org_external\n", - "-web_cache: 12h\n", - "+web_cache: 1m\n", - "+mysql_users:\n", - "+- debug@10.0.%\n", - "+debug: true\n", - " \n" - ] - }, - { - "ename": "", - "evalue": "1", - "output_type": "error", - "traceback": [] - } - ], - "source": [ - "diff -u \\\n", - "<(kheops lookup -e node=web.infra.net -e role=web -e env=prod profile) \\\n", - "<(kheops lookup -e node=web.infra.net -e role=web -e env=dev profile)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "348e5545", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "12b1730b", - "metadata": {}, - "source": [ - "You can also ask Kheops to explain you how he built the result, you can use the `-x` flag:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3ac0cc53", - "metadata": {}, - "outputs": [], - "source": [ - "kheops lookup -e node_role=web profile -x" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Bash", - "language": "bash", - "name": "bash" - }, - "language_info": { - "codemirror_mode": "shell", - "file_extension": ".sh", - "mimetype": "text/x-sh", - "name": "bash" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/jupyter/learn101.ipynb b/docs/jupyter/learn101.ipynb index 6185db4..e3354e0 100644 --- a/docs/jupyter/learn101.ipynb +++ b/docs/jupyter/learn101.ipynb @@ -610,11 +610,9 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "b7613ffd", + "cell_type": "markdown", + "id": "f5413f84", "metadata": {}, - "outputs": [], "source": [ "So it's become quite easy to compare the difference between environment, with a simple variable switch:" ]