New role to upgrade config files' format and some fixes

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,12 +2,12 @@
 # What does this PR do?
 <!--- Brief explanation of the code or documentation change you've made -->
 
-# How should this be tested?
+## How should this be tested?
 <!--- Automated tests are preferred, but not always doable - especially for infrastructure. Include commands to run your new feature, and also post-run commands to validate that it worked. (please use code blocks to format code samples) -->
 
-# Is there a relevant Issue open for this?
+## Is there a relevant Issue open for this?
 <!--- Provide a link to any open issues that describe the problem you are solving. -->
 resolves #[number]
 
-# Other Relevant info, PRs, etc
+## Other Relevant info, PRs, etc
 <!--- Please provide link to other PRs that may be related (blocking, resolves, etc. etc.) -->

.github/files/galaxy.yml.j2

@@ -5,28 +5,28 @@ version: {{ collection_version }}
 description: A collection of roles to extend functionality of aap_configuration collection
 readme: README.md
 authors:
-    - Andrew Huffman
-    - Adebisi Oyawale @aoyawale
-    - Kedar Kulkarni @kedark3
-    - Tom Page @Tompage1994
-    - Sean Sullivan @sean-m-sullivan
-    - David Danielsson @djdanielsson
-    - Ivan Aragonés
-    - Silvio Perez
-    - Adonis García
+  - Andrew Huffman
+  - Adebisi Oyawale @aoyawale
+  - Kedar Kulkarni @kedark3
+  - Tom Page @Tompage1994
+  - Sean Sullivan @sean-m-sullivan
+  - David Danielsson @djdanielsson
+  - Ivan Aragonés @ivarmu
+  - Silvio Perez
+  - Adonis García
 repository: {{ collection_repo }}/
 issues: {{ collection_repo }}/issues
 build_ignore:
-    - galaxy.yml.j2
-    - release.yml
-    - .github
-    - '*.tar.gz'
+  - galaxy.yml.j2
+  - release.yml
+  - .github
+  - '*.tar.gz'
 license:
-    - GPL-3.0-or-later
+  - GPL-3.0-or-later
 tags:
-    - controller
-    - collection
-    - aap_configuration_extended
-    - automation_platform
-    - infrastructure
+  - controller
+  - collection
+  - aap_configuration_extended
+  - automation_platform
+  - infrastructure
 ...

.gitignore

@@ -9,3 +9,4 @@ ansible.cfg
 vault-aap-controller.yaml
 .ansible/
 .vault-password
+sample_25

.yamllint.yml

@@ -14,7 +14,7 @@ rules:
     min-spaces-from-content: 1 # prettier compatibility
   comments-indentation: false
   document-start: disable
-  document-end: {present: true}
+  document-end: { present: true }
   indentation:
     level: error
     # Require indentation https://redhat-cop.github.io/automation-good-practices/#_yaml_and_jinja2_syntax

EXPORT_README.md

@@ -36,7 +36,7 @@ awx export --conf.host https://localhost --conf.username admin --conf.password *
      CONTROLLER_HOST: https://localhost
      CONTROLLER_USERNAME: admin
      CONTROLLER_PASSWORD: password
-     CONTROLLER_VERIFY_SSL: False
+     CONTROLLER_VERIFY_SSL: false
 
   tasks:
     - name: Export projects
@@ -50,7 +50,7 @@ awx export --conf.host https://localhost --conf.username admin --conf.password *
 
     - name: Export projects to file
       ansible.builtin.copy:
-        content: "{{ export_results | to_nice_yaml(width=50, explicit_start=True, explicit_end=True) }}"
+        content: "{{ export_results | to_nice_yaml(width=50, explicit_start=true, explicit_end=true) }}"
         dest: projects.yaml
 ...
 ```

changelogs/fragments/filetree_create_controller_applications.yaml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+  - Set the correct API URL for the controller applications endpoint
+...

changelogs/fragments/filetree_create_controller_credential_types.yaml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+  - There was an indentation error in the output at the `inputs` and `injectors` sections. The template has been fixed.
+...

changelogs/fragments/filetree_create_controller_inventories.yaml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+  - Change connection variables to AAP from 'controller_*' to 'aap_*' when exporting the 'inventory_sources'.
+...

changelogs/fragments/filetree_create_controller_workflow_job_templates.yaml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+  - Fix the exported contents of survey's choices in workflow job templates to avoid to have the clause '!unsafe' inside the generated string.
+...

changelogs/fragments/filetree_create_eda_credential_types.yaml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+  - There was an indentation error in the output at the `inputs` and `injectors` sections. The template has been fixed.
+...

changelogs/fragments/filetree_create_gateway_service_keys.yaml

@@ -0,0 +1,4 @@
+---
+bugfixes:
+  - There was a format error in the output after each key occurence. The template has been fixed.
+...

changelogs/fragments/new_role_upgrade_config.yml

@@ -0,0 +1,4 @@
+---
+major_changes:
+  - New role to update Configuration as Code files from 2.4 (infra.controller_configuration) format to 2.5 (infra.aap_configuration_extended)
+...

galaxy.yml

@@ -5,28 +5,28 @@ version: 1.1.1-devel
 description: A collection of roles to extend functionality of aap_configuration collection
 readme: README.md
 authors:
-    - Andrew Huffman
-    - Adebisi Oyawale @aoyawale
-    - Kedar Kulkarni @kedark3
-    - Tom Page @Tompage1994
-    - Sean Sullivan @sean-m-sullivan
-    - David Danielsson @djdanielsson
-    - Ivan Aragonés
-    - Silvio Perez
-    - Adonis García
+  - Andrew Huffman
+  - Adebisi Oyawale @aoyawale
+  - Kedar Kulkarni @kedark3
+  - Tom Page @Tompage1994
+  - Sean Sullivan @sean-m-sullivan
+  - David Danielsson @djdanielsson
+  - Ivan Aragonés @ivarmu
+  - Silvio Perez
+  - Adonis García
 repository: https://github.com/redhat-cop/aap_configuration_extended/
 issues: https://github.com/redhat-cop/aap_configuration_extended/issues
 build_ignore:
-    - galaxy.yml.j2
-    - release.yml
-    - .github
-    - '*.tar.gz'
+  - galaxy.yml.j2
+  - release.yml
+  - .github
+  - '*.tar.gz'
 license:
-    - GPL-3.0-or-later
+  - GPL-3.0-or-later
 tags:
-    - controller
-    - collection
-    - aap_configuration_extended
-    - automation_platform
-    - infrastructure
+  - controller
+  - collection
+  - aap_configuration_extended
+  - automation_platform
+  - infrastructure
 ...

playbooks/upgrade_config.yaml

@@ -0,0 +1,15 @@
+---
+#
+# ansible-playbook -i localhost, playbooks/upgrade_config.yaml -e '{sanitize: true}'
+#
+- name: "Playbook to upgrade CaC from AAP <= 2.4 to AAP >= 2.5 format"
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+    - name: "Call upgrade_config role"
+      ansible.builtin.include_role:
+        name: infra.aap_configuration_extended.upgrade_config
+      vars:
+        input_authenticator_name: "IDM LDAP"
+...

roles/filetree_create/tasks/controller_applications.yml

@@ -1,7 +1,7 @@
 ---
 - name: "Get current Applications from the API"
   ansible.builtin.set_fact:
-    applications_lookvar: "{{ query(controller_api_plugin, 'applications/',
+    applications_lookvar: "{{ query(controller_api_plugin, 'api/controller/v2/applications/',
                                 query_params=(query_params | combine({'organization': organization_id})) if organization_id is defined else query_params,
                                 host=aap_hostname, oauth_token=aap_oauthtoken, verify_ssl=aap_validate_certs,
                                 return_all=true, max_objects=query_controller_api_max_objects)

roles/filetree_create/templates/controller_credential_types.j2

@@ -8,14 +8,14 @@ controller_credential_types:
     kind: "{{ credential_type.kind }}"
     inputs:
 {{ template_overrides_resources.credential_type[credential_type.name].inputs
-   | default(credential_type.inputs) | to_nice_yaml(indent=2,sort_keys=False) | indent(width=6, first=True) }}
+   | default(credential_type.inputs) | to_nice_yaml(indent=2,sort_keys=false) | indent(width=8, first=true) }}
 {% if template_overrides_resources.credential_type[credential_type.name].injectors is defined
       or (credential_type.injectors and credential_type.injectors != '---') %}
     injectors:
 {# https://docs.ansible.com/ansible/latest/user_guide/playbooks_advanced_syntax.html#unsafe-or-raw-strings #}
 {{ template_overrides_resources.credential_type[credential_type.name].injectors
    | default(credential_type.injectors)
-   | to_nice_yaml(indent=2, sort_keys=False) | indent(width=6, first=True) | regex_replace('(^[^:]*): (.*){([{%])', '\\g<1>: !unsafe \\g<2>{\\g<3>', multiline=True)}}
-{%- endif %}
+   | to_nice_yaml(indent=2, sort_keys=false) | indent(width=8, first=true) | regex_replace('(^[^:]*): (.*){([{%])', '\\g<1>: !unsafe \\g<2>{\\g<3>', multiline=true)}}
+{% endif %}
 {% endfor %}
 ...

roles/filetree_create/templates/controller_credentials.j2

@@ -16,11 +16,11 @@ controller_credentials:
       or (current_credentials_asset_value.inputs is defined and current_credentials_asset_value.inputs is not match('{}')) %}
     inputs:
 {% if show_encrypted is defined and show_encrypted %}
-{{ current_credentials_asset_value.inputs | to_nice_yaml(indent=2, sort_keys=False) | indent(width=6, first=True) }}
+{{ current_credentials_asset_value.inputs | to_nice_yaml(indent=2, sort_keys=false) | indent(width=6, first=true) }}
 {% else %}
 {{ template_overrides_resources.credential[current_credentials_asset_value.name].inputs
    | default(current_credentials_asset_value.inputs)
-   | to_nice_yaml(indent=2, sort_keys=False) | indent(width=6, first=True) | replace("$encrypted$", "\'\'") }}
+   | to_nice_yaml(indent=2, sort_keys=false) | indent(width=6, first=true) | replace("$encrypted$", "\'\'") }}
 {% endif %}
 {% endif %}
 {% if last_credential | default(true) | bool %}

roles/filetree_create/templates/controller_workflow_job_templates.j2

@@ -58,10 +58,10 @@ controller_workflows:
 {%     if extra_data_item.value | type_debug is match('list') %}
           {{ extra_data_item.key }}:
 {%       for current_list_item in extra_data_item.value if ((extra_data_item.value | string | length) > 0 and (extra_data_item.value | string) is not match('None')) %}
-            - {{ (current_list_item | regex_replace("\n", "\\\\n") | regex_replace('"', '\\"')) | regex_replace('\\\\(?!n|")', '\\\\\\\\') | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=True) }}
+            - {{ (current_list_item | regex_replace("\n", "\\\\n") | regex_replace('"', '\\"')) | regex_replace('\\\\(?!n|")', '\\\\\\\\') | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=true) }}
 {%       endfor %}
 {%     else %}
-          {{ extra_data_item.key }}: {{ (extra_data_item.value | regex_replace("\n", "\\\\n") | regex_replace('"', '\\"')) | regex_replace('\\\\(?!n|")', '\\\\\\\\') | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=True) }}
+          {{ extra_data_item.key }}: {{ (extra_data_item.value | regex_replace("\n", "\\\\n") | regex_replace('"', '\\"')) | regex_replace('\\\\(?!n|")', '\\\\\\\\') | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=true) }}
 {%     endif %}
 {%   endfor %}
 {%- endif %}
@@ -117,10 +117,10 @@ controller_workflows:
 {%     if extra_vars_item.value | type_debug is match('list') %}
       {{ extra_vars_item.key }}:
 {%       for current_list_item in extra_vars_item.value if ((extra_vars_item.value | string | length) > 0 and (extra_vars_item.value | string) is not match('None')) %}
-        - {{ (current_list_item | regex_replace("\n", "\\\\n") | regex_replace('"', '\\"')) | regex_replace('\\\\(?!n|")', '\\\\\\\\') | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=True) }}
+        - {{ (current_list_item | regex_replace("\n", "\\\\n") | regex_replace('"', '\\"')) | regex_replace('\\\\(?!n|")', '\\\\\\\\') | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=true) }}
 {%       endfor %}
 {%     else %}
-      {{ extra_vars_item.key }}: {{ (extra_vars_item.value | regex_replace("\n", "\\\\n") | regex_replace('"', '\\"')) | regex_replace('\\\\(?!n|")', '\\\\\\\\') | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=True) }}
+      {{ extra_vars_item.key }}: {{ (extra_vars_item.value | regex_replace("\n", "\\\\n") | regex_replace('"', '\\"')) | regex_replace('\\\\(?!n|")', '\\\\\\\\') | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=true) }}
 {%     endif %}
 {%   endfor %}
 {% else %}
@@ -193,7 +193,11 @@ controller_workflows:
 {%           if survey_item_content.key is match('choices') and survey_item_content.value[0] is defined %}
           {{ survey_item_content.key }}:
 {%             for choice in survey_item_content.value if ((survey_item_content.value | string | length) > 0 and (survey_item_content.value | string) is not match('None')) %}
-            - "{{ choice | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=True) }}"
+{%               if (choice | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=true) != choice) %}
+            - {{ choice | regex_replace('(^[^{]*){([{%])(.*)', '!unsafe "\\g<1>{\\g<2>\\g<3>"', multiline=true) }}
+{%               else %}
+            - "{{ choice }}"
+{%               endif %}
 {%             endfor %}
 {%           endif %}
 {%         endif %}

roles/filetree_create/templates/eda_credential_types.j2

@@ -9,16 +9,16 @@ eda_credential_types:
     kind: "{{ eda_credential_type.kind }}"
     inputs:
 {{ template_overrides_resources.eda_credential_type[eda_credential_type.name].inputs
-   | default(eda_credential_type.inputs) | to_nice_yaml(indent=2,sort_keys=False) | indent(width=6, first=True) }}
+   | default(eda_credential_type.inputs) | to_nice_yaml(indent=2,sort_keys=false) | indent(width=8, first=true) }}
 {% if template_overrides_resources.eda_credential_type[eda_credential_type.name].injectors is defined
       or (eda_credential_type.injectors and eda_credential_type.injectors != '---') %}
     injectors:
 {# https://docs.ansible.com/ansible/latest/user_guide/playbooks_advanced_syntax.html#unsafe-or-raw-strings #}
 {{ template_overrides_resources.eda_credential_type[eda_credential_type.name].injectors
    | default(eda_credential_type.injectors)
-   | to_nice_yaml(indent=2, sort_keys=False) | indent(width=6, first=True) | regex_replace('(^[^:]*): (.*){([{%])', '\\g<1>: !unsafe \\g<2>{\\g<3>', multiline=True)}}
+   | to_nice_yaml(indent=2, sort_keys=false) | indent(width=8, first=true) | regex_replace('(^[^:]*): (.*){([{%])', '\\g<1>: !unsafe \\g<2>{\\g<3>', multiline=true)}}
 {% else %}
     injectors: {}
-{%- endif %}
+{% endif %}
 {% endfor %}
 ...

roles/filetree_create/templates/gateway_service_keys.j2

@@ -20,5 +20,6 @@ gateway_service_keys:
                      | default(template_overrides_global.gateway_route.mark_previous_inactive)
                      | default(current_service_key.mark_previous_inactive)
                      | default(false) }}" {# default(false) because it is required but not returned by the API #}
+
 {% endfor %}
 ...

roles/filetree_read/tests/.gitignore

@@ -1,2 +1,3 @@
 collections
+collections.old
 vars.yaml