Add VM lifecycle management to AAP configuration

collections/ansible_collections/cloudkit/config_as_code/roles/aap/vars/aap.yml

@@ -17,5 +17,7 @@ controller_roles:  # noqa: var-naming[no-role-prefix]
     job_templates:
       - "{{ aap_prefix }}-create-hosted-cluster"
       - "{{ aap_prefix }}-delete-hosted-cluster"
+      - "{{ aap_prefix }}-create-vm"
+      - "{{ aap_prefix }}-delete-vm"
     workflows:
       - "{{ aap_prefix }}-create-hosted-cluster-workflow"

collections/ansible_collections/cloudkit/config_as_code/roles/aap/vars/controller.yml

@@ -1,5 +1,5 @@
 ---
 gateway_settings:
   gateway_access_token_expiration: 7200
 
 # Create a project from the specified git repo
@@ -74,6 +74,30 @@
     allow_simultaneous: true
     ask_variables_on_launch: true
     verbosity: 0
+  - name: "{{ aap_prefix }}-create-vm"
+    project: "{{ aap_prefix }}"
+    organization: "{{ aap_organization_name }}"
+    job_type: run
+    playbook: "playbook_cloudkit_create_vm.yml"
+    inventory: "{{ aap_prefix }}-cluster-fulfillment"
+    execution_environment: "{{ aap_prefix }}-ee"
+    instance_groups:
+      - "{{ aap_prefix }}-vm-operations-ig"
+    allow_simultaneous: true
+    ask_variables_on_launch: true
+    verbosity: 0
+  - name: "{{ aap_prefix }}-delete-vm"
+    project: "{{ aap_prefix }}"
+    organization: "{{ aap_organization_name }}"
+    job_type: run
+    playbook: "playbook_cloudkit_delete_vm.yml"
+    inventory: "{{ aap_prefix }}-cluster-fulfillment"
+    execution_environment: "{{ aap_prefix }}-ee"
+    instance_groups:
+      - "{{ aap_prefix }}-vm-operations-ig"
+    allow_simultaneous: true
+    ask_variables_on_launch: true
+    verbosity: 0
 
 controller_job_template_surveys:  # noqa: var-naming[no-role-prefix]
   - name: "{{ aap_prefix }}-create-hosted-cluster-post-install"
@@ -127,6 +151,9 @@
   - name: "{{ aap_prefix }}-publish-templates"
     description: "Publish templates inventory"
     organization: "{{ aap_organization_name }}"
+  - name: "{{ aap_prefix }}-vm-operations"
+    description: "VM Operational Inventory - Dedicated BM Hypervisors"
+    organization: "{{ aap_organization_name }}"
 
 controller_inventory_sources:  # noqa: var-naming[no-role-prefix]
   - name: "{{ aap_prefix }}-cluster-fulfillment-is"
@@ -156,6 +183,15 @@
     overwrite: true
     overwrite_vars: true
     update_cache_timeout: 0
+  - name: "{{ aap_prefix }}-vm-operations-is"
+    organization: "{{ aap_organization_name }}"
+    source: scm
+    source_project: "{{ aap_prefix }}"
+    source_path: "inventory/localhost.yml"
+    inventory: "{{ aap_prefix }}-publish-templates"
+    overwrite: true
+    overwrite_vars: true
+    update_cache_timeout: 0
 
 controller_schedules:  # noqa: var-naming[no-role-prefix]
   # Sync project every 10min to get the latest updates from the git repository
@@ -379,3 +415,68 @@
               secretName: fulfillment-service-tls
   - name: "{{ aap_prefix }}-create-hosted-cluster-post-install-ig"
     is_container_group: true
+
+  - name: "{{ aap_prefix }}-vm-operations-ig"
+    is_container_group: true
+    pod_spec_override: |
+      apiVersion: v1
+      kind: Pod
+      metadata:
+        labels:
+          ansible_job: ''
+      spec:
+        serviceAccountName: cloudkit-sa
+        affinity:
+          podAntiAffinity:
+            preferredDuringSchedulingIgnoredDuringExecution:
+              - weight: 100
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                      - key: ansible_job
+                        operator: Exists
+                  topologyKey: kubernetes.io/hostname
+        containers:
+          - image: >-
+              {{ aap_ee_image }}
+            name: worker
+            args:
+              - ansible-runner
+              - worker
+              - '--private-data-dir=/runner'
+            volumeMounts:
+              - name: kube-api-access
+                mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+                readOnly: true
+            envFrom:
+              - secretRef:
+                  name: cluster-fulfillment-ig
+            env:
+              - name: CLOUDKIT_VM_OPERATIONS_NAMESPACE_DEFAULT
+                valueFrom:
+                  fieldRef:
+                    fieldPath: metadata.namespace
+        volumes:
+          - name: kube-api-access
+            projected:
+              sources:
+                - serviceAccountToken:
+                    path: token
+                    expirationSeconds: 3600
+                - configMap:
+                    name: kube-root-ca.crt
+                    items:
+                      - key: ca.crt
+                        path: ca.crt
+                - downwardAPI:
+                    items:
+                      - path: namespace
+                        fieldRef:
+                          apiVersion: v1
+                          fieldPath: metadata.namespace
+                - configMap:
+                    name: openshift-service-ca.crt
+                    items:
+                      - key: service-ca.crt
+                        path: service-ca.crt
+              defaultMode: 420

collections/requirements.yml

@@ -9,9 +9,6 @@ collections:
     version: 10.5.0
   - name: ansible.utils
     version: 5.1.2
-  - source: "https://github.com/IsaiahStapleton/ai-cluster-ansible.git"
-    type: git
-    version: 0.1.0
   - name: ansible.platform
     version: "2.5.20250326"
   - name: ansible.hub

playbook_cloudkit_create_vm.yml

@@ -0,0 +1,77 @@
+---
+- name: Create a virtual machine from a VMOrder
+  hosts: localhost
+  gather_facts: false
+
+  vars:
+    vm_order: "{{ ansible_eda.event.payload }}"
+    vm_settings_cloudkit_finalizer: "cloudkit.openshift.io/finalizer"
+  pre_tasks:
+    - name: Set VM order name
+      ansible.builtin.set_fact:
+        vm_order_name: "{{ vm_order.metadata.name }}"
+
+    - name: Set lock name and holder identity
+      ansible.builtin.set_fact:
+        vm_order_lock_name: "{{ vm_order_name }}-lock"
+        vm_order_holder_id: >-
+          {{ vm_order_name }}-{{ awx_job_id | default("unknown") }}-{{
+            lookup('community.general.random_string', special=false, numbers=false, upper=false)
+          }}
+
+    - name: VM order before defaults
+      ansible.builtin.debug:
+        var: vm_order
+
+    - name: Apply default VM settings
+      ansible.builtin.include_role:
+        name: cloudkit.service.vm_settings
+
+    - name: VM order after defaults
+      ansible.builtin.debug:
+        var: vm_order
+
+    - name: Extract VM template info
+      ansible.builtin.include_role:
+        name: cloudkit.service.extract_vm_template_info
+
+    - name: Determine working namespace
+      ansible.builtin.include_role:
+        name: cloudkit.service.vm_working_namespace
+
+  tasks:
+    - name: Create virtual machine
+      block:
+        - name: Display VM order information
+          ansible.builtin.debug:
+            msg:
+              - "VM order: {{ vm_order_name }}"
+              - "VM target namespace: {{ vm_target_namespace }}"
+              - "Template ID: {{ template_id }}"
+
+        - name: Register VM Namespace Object
+          kubernetes.core.k8s_info:
+            api_version: v1
+            kind: Namespace
+            name: "{{ vm_target_namespace }}"
+          register: target_vm_ns_object
+
+        - name: "Set the finalizer on the VM namespace {{ vm_target_namespace }}"
+          kubernetes.core.k8s:
+            state: present
+            definition:
+              apiVersion: v1
+              kind: Namespace
+              metadata:
+                name: "{{ vm_target_namespace }}"
+                finalizers: "{{ (target_vm_ns_object.resources[0].metadata.finalizers | default([])) | union([vm_settings_cloudkit_finalizer]) }}"
+
+        - name: Call the selected VM template
+          ansible.builtin.include_role:
+            name: "{{ template_id }}"
+            tasks_from: "create"
+
+      rescue:
+        - name: Propagate failure
+          ansible.builtin.fail:
+            msg: Propagating earlier failure from rescue block

playbook_cloudkit_delete_vm.yml

@@ -0,0 +1,112 @@
+---
+- name: Delete a virtual machine from a VMOrder
+  hosts: localhost
+  gather_facts: false
+
+  vars:
+    vm_order: "{{ ansible_eda.event.payload }}"
+    vm_settings_cloudkit_finalizer: "cloudkit.openshift.io/finalizer"
+
+  pre_tasks:
+    - name: Set VM order name
+      ansible.builtin.set_fact:
+        vm_order_name: "{{ vm_order.metadata.name }}"
+
+    - name: Set lock name and holder identity
+      ansible.builtin.set_fact:
+        vm_order_lock_name: "{{ vm_order_name }}-lock"
+        vm_order_holder_id: >-
+          {{ vm_order_name }}-{{ awx_job_id | default("unknown") }}-{{
+            lookup('community.general.random_string', special=false, numbers=false, upper=false)
+          }}
+
+    - name: VM order for deletion
+      ansible.builtin.debug:
+        var: vm_order
+
+    - name: Apply default VM settings
+      ansible.builtin.include_role:
+        name: cloudkit.service.vm_settings
+
+    - name: Extract VM template info
+      ansible.builtin.include_role:
+        name: cloudkit.service.extract_vm_template_info
+
+    - name: Determine target namespace
+      ansible.builtin.include_role:
+        name: cloudkit.service.vm_working_namespace
+
+    - name: Register VM Namespace Object
+      kubernetes.core.k8s_info:
+        api_version: v1
+        kind: Namespace
+        name: "{{ vm_target_namespace }}"
+      register: target_vm_ns_object
+
+  tasks:
+    - name: Delete virtual machine
+      block:
+        - name: Acquire VM lock for deletion
+          kubernetes.core.k8s:
+            state: present
+            apply: true
+            server_side_apply:
+              field_manager: "{{ vm_order_holder_id }}"
+            definition:
+              apiVersion: coordination.k8s.io/v1
+              kind: Lease
+              metadata:
+                name: "{{ vm_order_lock_name }}"
+                namespace: "{{ vm_target_namespace }}"
+                labels:
+                  cloudkit.openshift.io/aap-job-id: "job-{{ awx_job_id | default('unknown') }}"
+                  cloudkit.openshift.io/resource-type: "vm"
+                  cloudkit.openshift.io/operation: "delete"
+              spec:
+                holderIdentity: "{{ vm_order_holder_id }}"
+
+        - name: Set lock_acquired
+          ansible.builtin.set_fact:
+            lock_acquired: true
+
+        - name: Display VM deletion information
+          ansible.builtin.debug:
+            msg:
+              - "Deleting VM order: {{ vm_order_name }}"
+              - "VM target namespace: {{ vm_target_namespace }}"
+              - "Template ID: {{ template_id }}"
+
+        - name: Call the selected VM template for deletion
+          ansible.builtin.include_role:
+            name: "{{ template_id }}"
+            tasks_from: "delete"
+          vars:
+            template_parameters:
+              vm_name: "{{ vm_order_name }}"
+              vm_namespace: "{{ vm_target_namespace }}"
+
+        - name: "Remove the finalizer on the VM namespace {{ vm_target_namespace }}"
+          kubernetes.core.k8s:
+            state: present
+            definition:
+              apiVersion: v1
+              kind: Namespace
+              metadata:
+                name: "{{ vm_target_namespace }}"
+                finalizers: "{{ (target_vm_ns_object.resources[0].metadata.finalizers | default([])) | difference([vm_settings_cloudkit_finalizer]) }}"
+      rescue:
+        - name: Propagate failure
+          ansible.builtin.fail:
+            msg: Propagating earlier failure from rescue block
+
+      always:
+        - name: Delete VM lock
+          when: lock_acquired | default(false)
+          kubernetes.core.k8s:
+            state: absent
+            definition:
+              apiVersion: coordination.k8s.io/v1
+              kind: Lease
+              metadata:
+                name: "{{ vm_order_lock_name }}"
+                namespace: "{{ vm_target_namespace }}"

rulebooks/cluster_fulfillment.yml

@@ -20,3 +20,17 @@
         run_job_template:
           name: "{{ job_template_prefix }}-delete-hosted-cluster"
           organization: "{{ job_template_organization }}"
+
+    - name: Create virtual machine
+      condition: event.meta.endpoint == "create-vm"
+      action:
+        run_job_template:
+          name: "{{ job_template_prefix }}-create-vm"
+          organization: "{{ job_template_organization }}"
+
+    - name: Delete virtual machine
+      condition: event.meta.endpoint == "delete-vm"
+      action:
+        run_job_template:
+          name: "{{ job_template_prefix }}-delete-vm"
+          organization: "{{ job_template_organization }}"